anph
/
hugegraph-sync
forked from hugegraph/hugegraph-sync
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(store): integrate rest of store-test submodule (#2563) 

Co-authored-by: imbajin <jin@apache.org>
This commit is contained in:
Peng Junzhi 2024-07-20 05:10:38 -05:00 committed by GitHub
parent 73a4c9d028
commit edb59ebaeb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
65 changed files with 9953 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
.github/workflows/pd-store-ci.yml vendored
View File

@ -1,4 +1,4 @@
name: "hugegraph-pd-store-ci"
name: "pd-store-ci"
on:
push:
@ -109,10 +109,39 @@ jobs:
cp $HOME/.m2/settings.xml /tmp/settings.xml
mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml
- name: Package
run: |
mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp
- name: Prepare env and service
run: |
$TRAVIS_DIR/start-pd.sh
$TRAVIS_DIR/start-store.sh
- name: Run common test
run: |
mvn test -pl hugegraph-store/hg-store-test -am -P store-common-test
- name: Run client test
run: |
mvn test -pl hugegraph-store/hg-store-test -am -P store-client-test
- name: Run core test
run: |
mvn test -pl hugegraph-store/hg-store-test -am -P store-core-test
- name: Run rocksdb test
run: |
mvn test -pl hugegraph-store/hg-store-test -am -P store-rocksdb-test
- name: Run server test
run: |
mvn test -pl hugegraph-store/hg-store-test -am -P store-server-test
- name: Run raft-core test
run: |
mvn test -pl hugegraph-store/hg-store-test -am -P store-raftcore-test
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3.0.0
with:

5
hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeSessionImpl.java
View File

@ -220,7 +220,10 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession {
private boolean prepareBatchEntry(OpType opType, String table
, HgOwnerKey startKey, HgOwnerKey endKey, byte[] value) {
this.batchEntryBuilder.clear().setOpType(opType);
this.batchEntryBuilder.setTable(tables.get(table));
Integer tableCode = tables.get(table);
if (tableCode != null) {
this.batchEntryBuilder.setTable(tableCode);
}
if (startKey != null) {
this.batchEntryBuilder.setStartKey(toKey(startKey));
}

147
hugegraph-store/hg-store-test/pom.xml
View File

@ -98,10 +98,10 @@
</exclusion>
</exclusions>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.apache.hugegraph</groupId>-->
<!-- <artifactId>hg-store-core</artifactId>-->
<!-- </dependency>-->
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-pd-grpc</artifactId>
@ -203,12 +203,12 @@
</exclusion>
</exclusions>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.apache.hugegraph</groupId>-->
<!-- <artifactId>hg-store-node</artifactId>-->
<!-- <version>${revision}</version>-->
<!-- <scope>compile</scope>-->
<!-- </dependency>-->
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-node</artifactId>
<version>${revision}</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
@ -217,31 +217,30 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<!-- TODO：uncomment later-->
<!-- <execution>-->
<!-- <id>store-client-test</id>-->
<!-- <configuration>-->
<!-- <testSourceDirectory>${basedir}/src/main/java/-->
<!-- </testSourceDirectory>-->
<!-- <testClassesDirectory>${basedir}/target/classes/-->
<!-- </testClassesDirectory>-->
<!-- <includes>-->
<!-- <include>**/ClientSuiteTest.java</include>-->
<!-- </includes>-->
<!-- </configuration>-->
<!-- </execution>-->
<!-- <execution>-->
<!-- <id>store-core-test</id>-->
<!-- <configuration>-->
<!-- <testSourceDirectory>${basedir}/src/main/java/-->
<!-- </testSourceDirectory>-->
<!-- <testClassesDirectory>${basedir}/target/classes/-->
<!-- </testClassesDirectory>-->
<!-- <includes>-->
<!-- <include>**/CoreSuiteTest.java</include>-->
<!-- </includes>-->
<!-- </configuration>-->
<!-- </execution>-->
<execution>
<id>store-client-test</id>
<configuration>
<testSourceDirectory>${basedir}/src/main/java/
</testSourceDirectory>
<testClassesDirectory>${basedir}/target/classes/
</testClassesDirectory>
<includes>
<include>**/ClientSuiteTest.java</include>
</includes>
</configuration>
</execution>
<execution>
<id>store-core-test</id>
<configuration>
<testSourceDirectory>${basedir}/src/main/java/
</testSourceDirectory>
<testClassesDirectory>${basedir}/target/classes/
</testClassesDirectory>
<includes>
<include>**/CoreSuiteTest.java</include>
</includes>
</configuration>
</execution>
<execution>
<id>store-common-test</id>
<configuration>
@ -254,45 +253,45 @@
</includes>
</configuration>
</execution>
<!-- <execution>-->
<!-- <id>store-rocksdb-test</id>-->
<!-- <configuration>-->
<!-- <testSourceDirectory>${basedir}/src/main/java/-->
<!-- </testSourceDirectory>-->
<!-- <testClassesDirectory>${basedir}/target/classes/-->
<!-- </testClassesDirectory>-->
<!-- <includes>-->
<!-- <include>**/RocksDbSuiteTest.java</include>-->
<!-- </includes>-->
<!-- </configuration>-->
<!-- </execution>-->
<!-- <execution>-->
<!-- <id>store-server-test</id>-->
<!-- <configuration>-->
<!-- <testSourceDirectory>${basedir}/src/main/java/-->
<!-- </testSourceDirectory>-->
<!-- <testClassesDirectory>${basedir}/target/classes/-->
<!-- </testClassesDirectory>-->
<!-- <includes>-->
<!-- <include>**/ServerSuiteTest.java</include>-->
<!-- </includes>-->
<!-- <excludes>-->
<!-- <exclude>**/node/**/*</exclude>-->
<!-- </excludes>-->
<!-- </configuration>-->
<!-- </execution>-->
<!-- <execution>-->
<!-- <id>store-raftcore-test</id>-->
<!-- <configuration>-->
<!-- <testSourceDirectory>${basedir}/src/main/java/-->
<!-- </testSourceDirectory>-->
<!-- <testClassesDirectory>${basedir}/target/classes/-->
<!-- </testClassesDirectory>-->
<!-- <includes>-->
<!-- <include>**/RaftSuiteTest.java</include>-->
<!-- </includes>-->
<!-- </configuration>-->
<!-- </execution>-->
<execution>
<id>store-rocksdb-test</id>
<configuration>
<testSourceDirectory>${basedir}/src/main/java/
</testSourceDirectory>
<testClassesDirectory>${basedir}/target/classes/
</testClassesDirectory>
<includes>
<include>**/RocksDbSuiteTest.java</include>
</includes>
</configuration>
</execution>
<execution>
<id>store-server-test</id>
<configuration>
<testSourceDirectory>${basedir}/src/main/java/
</testSourceDirectory>
<testClassesDirectory>${basedir}/target/classes/
</testClassesDirectory>
<includes>
<include>**/ServerSuiteTest.java</include>
</includes>
<excludes>
<exclude>**/node/**/*</exclude>
</excludes>
</configuration>
</execution>
<execution>
<id>store-raftcore-test</id>
<configuration>
<testSourceDirectory>${basedir}/src/main/java/
</testSourceDirectory>
<testClassesDirectory>${basedir}/target/classes/
</testClassesDirectory>
<includes>
<include>**/RaftSuiteTest.java</include>
</includes>
</configuration>
</execution>
</executions>
</plugin>
<plugin>

210
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgBusinessImplTest.java Normal file
View File

@ -0,0 +1,210 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.business.BusinessHandler;
import org.apache.hugegraph.store.business.BusinessHandlerImpl;
import org.apache.hugegraph.store.meta.PartitionManager;
import org.apache.hugegraph.store.options.HgStoreEngineOptions;
import org.apache.hugegraph.store.options.RaftRocksdbOptions;
import org.apache.hugegraph.store.pd.FakePdServiceProvider;
import org.apache.hugegraph.store.pd.PdProvider;
import org.apache.hugegraph.util.Bytes;
import org.junit.Assert;
public class HgBusinessImplTest {
static String GRAPH_NAME = "graph_test";
static String[] GRAPH_TABLE = {"table1", "table2"};
static String dbPath = "tmp/junit";
// @BeforeClass
public static void init() {
UnitTestBase.deleteDir(new File(dbPath));
Map<String, Object> configMap = new HashMap<>();
configMap.put("rocksdb.write_buffer_size", "1048576");
RaftRocksdbOptions.initRocksdbGlobalConfig(configMap);
BusinessHandlerImpl.initRocksdb(configMap, null);
}
public BusinessHandler getBusinessHandler() {
int partitionCount = 2;
HgStoreEngineOptions options = new HgStoreEngineOptions() {{
setDataPath(dbPath);
setFakePdOptions(new HgStoreEngineOptions.FakePdOptions() {{
setPartitionCount(partitionCount);
setPeersList("127.0.0.1");
setStoreList("127.0.0.1");
}});
}};
PdProvider pdProvider = new FakePdServiceProvider(options.getFakePdOptions());
PartitionManager partitionManager = new PartitionManager(pdProvider, options) {
@Override
public List<Integer> getLeaderPartitionIds(String graph) {
List<Integer> ids = new ArrayList<>();
for (int i = 0; i < partitionCount; i++) {
ids.add(i);
}
return ids;
}
};
BusinessHandler handler = new BusinessHandlerImpl(partitionManager);
return handler;
}
// @Test
public void testPut() {
String graph1 = "test-graph11";
String graph2 = "test-graph12";
String table = "test";
BusinessHandler handler = getBusinessHandler();
handler.doPut(graph1, 0, table, "key1".getBytes(), "value1".getBytes());
handler.doPut(graph1, 0xF, table, "key2".getBytes(), "value1".getBytes());
handler.doPut(graph1, 0xFF, table, "key3".getBytes(), "value1".getBytes());
handler.doPut(graph1, 0xFFF, table, "key4".getBytes(), "value1".getBytes());
handler.doPut(graph2, 0, table, "key21".getBytes(), "value1".getBytes());
handler.doPut(graph2, 0xF, table, "key22".getBytes(), "value1".getBytes());
handler.doPut(graph2, 0xFF, table, "key23".getBytes(), "value1".getBytes());
handler.doPut(graph2, 0xFFF, table, "key24".getBytes(), "value1".getBytes());
System.out.println("--------------------dump all -------");
dump(handler, graph1, 0);
System.out.println("--------------------dump scan 0 0xff -------");
ScanIterator iterator = handler.scan(graph1, table, 0, 0xff);
int count = 0;
while (iterator.hasNext()) {
RocksDBSession.BackendColumn entry = iterator.next();
System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name));
count++;
}
Assert.assertEquals(2, count);
System.out.println("--------------------dump scan prefix -------");
iterator = handler.scanPrefix(graph1, 0, table, "key".getBytes());
count = 0;
while (iterator.hasNext()) {
RocksDBSession.BackendColumn entry = iterator.next();
System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name));
count++;
}
Assert.assertEquals(4, count);
System.out.println("--------------------dump scan range key1 key4 -------");
iterator = handler.scan(graph1, 0, table, "key1".getBytes(), "key4".getBytes(),
ScanIterator.Trait.SCAN_LT_END);
count = 0;
while (iterator.hasNext()) {
RocksDBSession.BackendColumn entry = iterator.next();
System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name));
count++;
}
Assert.assertEquals(3, count);
}
// @Test
public void testLoadSnapshot() throws InterruptedException {
BusinessHandler handler = getBusinessHandler();
String graph1 = "test-graph1";
String graph2 = "test-graph2";
String table = "test";
for (int i = 0; i < 256; i++) {
handler.doPut(graph1, i, table, ("key" + i).getBytes(), "value1".getBytes());
}
ScanIterator iterator = handler.scanAll(graph1, table);
System.out.println(iterator.count());
String snapshotPath;
try (RocksDBSession session = handler.getSession(0)) {
snapshotPath = session.getDbPath();
}
handler.closeAll();
System.out.println("start loadSnapshot");
handler.loadSnapshot(snapshotPath, graph1, 0, 10);
iterator = handler.scanAll(graph1, table);
Assert.assertEquals(255, iterator.count());
try (RocksDBSession session = handler.getSession(0)) {
System.out.println(session.getDbPath());
}
CountDownLatch latch = new CountDownLatch(1);
RocksDBFactory.getInstance()
.addRocksdbChangedListener(new RocksDBFactory.RocksdbChangedListener() {
@Override
public void onCompacted(String dbName) {
RocksDBFactory.RocksdbChangedListener.super.onCompacted(dbName);
}
@Override
public void onDBDeleteBegin(String dbName, String filePath) {
RocksDBFactory.RocksdbChangedListener.super.onDBDeleteBegin(dbName,
filePath);
}
@Override
public void onDBDeleted(String dbName, String filePath) {
latch.countDown();
}
@Override
public void onDBSessionReleased(RocksDBSession dbSession) {
RocksDBFactory.RocksdbChangedListener.super.onDBSessionReleased(
dbSession);
}
});
latch.await();
}
public void dump(BusinessHandler handler, String graph, int partId) {
ScanIterator cfIterator = handler.scanRaw(graph, partId, 0);
while (cfIterator.hasNext()) {
try (ScanIterator iterator = cfIterator.next()) {
byte[] cfName = cfIterator.position();
System.out.println(graph + "-" + +partId + "-" + new String(cfName) + "--------");
while (iterator.hasNext()) {
RocksDBSession.BackendColumn col = iterator.next();
System.out.println(new String(col.name) + " -- " + Bytes.toHex(col.name));
}
}
}
cfIterator.close();
}
}

481
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerOneRaftFakePDTest.java Normal file
View File

@ -0,0 +1,481 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store;
import static org.apache.hugegraph.store.client.util.HgAssert.isInvalid;
import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER;
import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import org.apache.hugegraph.pd.common.PartitionUtils;
import org.apache.hugegraph.store.client.HgStoreNodeManager;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.util.HgStoreTestUtil;
import org.junit.Assert;
/**
* 使用fake-pd，支持raft的单元测试
*/
public class HgSessionManagerOneRaftFakePDTest {
private static final Map<Integer, Long> leaderMap = new ConcurrentHashMap<>();
private static final Map<Long, String> storeMap = new ConcurrentHashMap<>();
private static final int partitionCount = 3;
// 需要与store的application.yml的fake-pd.partition-count保持一致
private static final String[] storeAddress = {
"127.0.0.1:8500"
};
// @BeforeClass
public static void init() {
for (String address : storeAddress) {
storeMap.put((long) address.hashCode(), address);
}
for (int i = 0; i < partitionCount; i++) {
leaderMap.put(i, storeMap.keySet().iterator().next());
}
HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance();
nodeManager.setNodePartitioner((builder, graphName, startKey, endKey) -> {
int startCode = PartitionUtils.calcHashcode(startKey);
int endCode = PartitionUtils.calcHashcode(endKey);
if (ALL_PARTITION_OWNER == startKey) {
storeMap.forEach((k, v) -> {
builder.add(k, -1);
});
} else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey ||
Arrays.equals(startKey, endKey)) {
builder.add(leaderMap.get(startCode % partitionCount), startCode);
} else {
Assert.fail("OwnerKey转成HashCode后已经无序了， 按照OwnerKey范围查询没意义");
builder.add(leaderMap.get(startCode % partitionCount), startCode);
builder.add(leaderMap.get(endCode % partitionCount), endCode);
}
return 0;
});
nodeManager.setNodeProvider((graphName, nodeId) -> {
System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + " " +
storeMap.get(nodeId));
return nodeManager.getNodeBuilder().setNodeId(nodeId)
.setAddress(storeMap.get(nodeId)).build();
});
nodeManager.setNodeNotifier((graphName, storeNotice) -> {
System.out.println("recv node notifier " + storeNotice);
if (storeNotice.getPartitionLeaders().size() > 0) {
leaderMap.putAll(storeNotice.getPartitionLeaders());
System.out.println("leader changed ");
leaderMap.forEach((k, v) -> {
System.out.print(" " + k + " " + v + ",");
});
System.out.println();
}
return 0;
});
}
private static HgStoreSession getStoreSession() {
return HgSessionManager.getInstance().openSession(HgStoreTestUtil.GRAPH_NAME);
}
private static HgStoreSession getStoreSession(String graph) {
return HgSessionManager.getInstance().openSession(graph);
}
// @Test
public void put_get() {
System.out.println("--- test put & get ---");
HgStoreSession session = getStoreSession();
HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY");
//HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY");
byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE");
Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value));
Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get(
HgStoreTestUtil.TABLE_NAME, key)));
}
// @Test
public void put_get2() {
System.out.println("--- test put & get 2---");
String GRAPH_NAME = "default/hugegraph/g2";
String TABLE_NAME = "put_get2";
HgStoreSession session = getStoreSession(GRAPH_NAME);
HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY");
//HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY");
byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE");
Assert.assertTrue(session.put(TABLE_NAME, key, value));
Assert.assertEquals(HgStoreTestUtil.toStr(value),
HgStoreTestUtil.toStr(session.get(TABLE_NAME, key)));
HgKvIterator<HgKvEntry> iterator = session.scanIterator(TABLE_NAME);
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
System.out.println(new String(entry.key()));
Assert.assertEquals(HgStoreTestUtil.toStr(key.getKey()),
HgStoreTestUtil.toStr(entry.key()));
}
}
// @Test
public void batchGet() {
System.out.println("--- test batchGet ---");
HgStoreSession session = getStoreSession();
String keyPrefix = "BATCH-GET-UNIT";
Map<HgOwnerKey, byte[]> map = HgStoreTestUtil.batchPut(session, keyPrefix);
List<HgOwnerKey> keyList =
map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList());
//printOwner(keyList);
List<HgKvEntry> resList = session.batchGetOwner(HgStoreTestUtil.TABLE_NAME, keyList);
Assert.assertFalse(isInvalid(resList));
Assert.assertEquals(resList.size(), keyList.size());
// println(list);
HgStoreTestUtil.println("--- batch-get result ---");
Assert.assertTrue((resList.stream()
.map(e -> map.containsKey(HgStoreTestUtil.toOwnerKey(e.key())))
.allMatch(Boolean::booleanValue))
);
}
// @Test
//CAUTION: ONLY FOR LONG！
//注意：目前只可以对long类型value进行Merge操作。
public void merge() {
System.out.println("--- test merge (1+1=2)---");
HgStoreSession session = getStoreSession();
String mergeKey = "merge-key";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey);
byte[] value = HgStoreTestUtil.toBytes(1L);
System.out.println("- put " + mergeKey + ":1 -");
session.put(HgStoreTestUtil.TABLE_NAME, key, value);
System.out.println("- merge " + mergeKey + ":1 -");
session.merge(HgStoreTestUtil.TABLE_NAME, key, value);
long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key));
System.out.printf("after merge " + mergeKey + "=%s%n", res);
Assert.assertEquals(2L, res);
String putStr = "19";
session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr));
byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key);
Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1));
}
// @Test
public void delete() {
System.out.println("--- test delete ---");
HgStoreSession session = getStoreSession();
String delKey = "del-key";
String delValue = "del-value";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey);
byte[] value = HgStoreTestUtil.toBytes(delValue);
HgStoreTestUtil.println("- put " + delKey + ": " + delValue);
session.put(HgStoreTestUtil.TABLE_NAME, key, value);
HgStoreTestUtil.println("- delete " + delKey);
session.delete(HgStoreTestUtil.TABLE_NAME, key);
value = session.get(HgStoreTestUtil.TABLE_NAME, key);
HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value));
Assert.assertEquals(EMPTY_BYTES, value);
}
// @Test
public void deleteSingle() {
System.out.println("--- test deleteSingle ---");
HgStoreSession session = getStoreSession();
String delKey = "del-single-key";
String delValue = "del-single-value";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey);
byte[] value = HgStoreTestUtil.toBytes(delValue);
HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue);
session.put(HgStoreTestUtil.TABLE_NAME, key, value);
value = session.get(HgStoreTestUtil.TABLE_NAME, key);
HgStoreTestUtil.println(
"- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value));
Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value));
HgStoreTestUtil.println("- delete-single : [" + delKey + "]");
session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key);
value = session.get(HgStoreTestUtil.TABLE_NAME, key);
HgStoreTestUtil.println(
"- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value));
Assert.assertEquals("", HgStoreTestUtil.toStr(value));
}
// @Test
public void deleteRange() {
HgStoreTestUtil.println("--- test deleteRange ---");
HgStoreSession session = getStoreSession();
String rangePrefix = "DEL-RANGE-KEY";
String owner = "batch-delete-owner";
Map<HgOwnerKey, byte[]> map =
HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10,
key -> {
return HgStoreTestUtil.toOwnerKey(owner, key);
});
HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00");
HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05");
Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey));
HgStoreTestUtil.println("- after delete range from ["
+ HgStoreTestUtil.toStr(startKey.getKey())
+ "] to ["
+ HgStoreTestUtil.toStr(endKey.getKey()) + "]");
for (int i = 0; i < 10; i++) {
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" +
HgStoreTestUtil.padLeftZeros(
String.valueOf(i), 2));
String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key));
HgStoreTestUtil.println(
"- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value);
// TODO: [start,end)?
if (i < 5) {
Assert.assertEquals("", value);
} else {
//println(value);
Assert.assertNotEquals("", value);
}
}
}
// @Test
public void deletePrefix() {
System.out.println("--- test deletePrefix ---");
HgStoreSession session = getStoreSession();
String prefixStr = "DEL-PREFIX-KEY";
String owner = "batch-delete-owner";
Map<HgOwnerKey, byte[]> map =
HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10,
key -> {
return HgStoreTestUtil.toOwnerKey(owner, key);
});
//printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()));
HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr);
Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator(
HgStoreTestUtil.TABLE_NAME, prefix)));
session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix);
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(
HgStoreTestUtil.TABLE_NAME, prefix)));
HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]");
for (int i = 0; i < 10; i++) {
HgOwnerKey key =
HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2));
String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key));
System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value);
Assert.assertEquals("", value);
}
}
// @Test
public void scanIterator() {
HgStoreTestUtil.println("--- test scanIterator ---");
String tableName = "UNIT_SCAN";
String keyName = "SCAN-ITER";
HgStoreSession session = getStoreSession();
HgStoreTestUtil.batchPut(session, tableName, keyName, 10000);
int count = 0;
int limit = 0;
int max = 99999;
HgKvIterator<HgKvEntry> iterator = null;
HgStoreTestUtil.println("-- test 0 element --");
iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"),
HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0);
Assert.assertFalse(iterator.hasNext());
try {
iterator.next();
Assert.fail();
} catch (Throwable t) {
HgStoreTestUtil.println("-- test NoSuchElementException --");
Assert.assertTrue(t instanceof NoSuchElementException);
}
HgStoreTestUtil.println("-- test limit 1 to 10 --");
for (int i = 1; i <= 10; i++) {
HgStoreTestUtil.println("- limit " + i + " -");
limit = i;
iterator = session.scanIterator(tableName,
HgStoreTestUtil.toAllPartitionKey(keyName + "-0"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-1"),
limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
HgStoreTestUtil.println(entry);
}
Assert.assertEquals(limit, count);
}
HgStoreTestUtil.println("-- test limit 1 to 10 not enough --");
for (int i = 1; i <= 10; i++) {
HgStoreTestUtil.println("- limit " + i + " -");
limit = i;
iterator = session.scanIterator(tableName,
HgStoreTestUtil.toAllPartitionKey(keyName + "-00001"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-00005"),
limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
HgStoreTestUtil.println(entry);
}
if (i <= 5) {
Assert.assertEquals(limit, count);
} else {
Assert.assertEquals(5, count);
}
}
HgStoreTestUtil.println("-- test limit 0 (no limit) --");
limit = 0;
iterator =
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 1000 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(10000, count);
HgStoreTestUtil.println("-- test scan all --");
iterator = session.scanIterator(tableName);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 1000 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(10000, count);
HgStoreTestUtil.println("-- test scan prefix --");
iterator =
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-01"));
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 100 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(1000, count);
}
// @Test
public void truncate() {
HgStoreTestUtil.println("--- test truncate ---");
String graph = "graph_truncate";
String tableName = "UNIT_TRUNCATE_1";
String keyName = "KEY_TRUNCATE";
HgStoreSession session = getStoreSession(graph);
HgStoreTestUtil.batchPut(session, tableName, keyName, 100);
Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName)));
String tableName2 = "UNIT_TRUNCATE_2";
HgStoreTestUtil.batchPut(session, tableName2, keyName, 100);
Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2)));
session.truncate();
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName)));
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2)));
}
//// @Test
public void scanIteratorHuge() {
/*************** test no limit, with 10 millions **************/
String tableName = "UNIT_HUGE";
String keyName = "SCAN-HUGE";
int amount = 10_000_000;
HgStoreSession session = getStoreSession();
if (HgStoreTestUtil.amountOf(
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) <
10) {
HgStoreTestUtil.batchPut(session, tableName, keyName, amount);
}
int count = 0;
HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName);
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % (amount / 10) == 0) {
HgStoreTestUtil.println(entry);
}
}
Assert.assertEquals(amount, count);
}
}

648
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerRaftFakePDTest.java Normal file
View File

@ -0,0 +1,648 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store;
import static org.apache.hugegraph.store.client.util.HgAssert.isInvalid;
import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER;
import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import org.apache.hugegraph.pd.common.PartitionUtils;
import org.apache.hugegraph.store.client.HgStoreNodeManager;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.util.HgStoreTestUtil;
import org.junit.Assert;
/**
* 使用fake-pd，支持raft的单元测试
*/
public class HgSessionManagerRaftFakePDTest {
private static final Map<Integer, Long> leaderMap = new ConcurrentHashMap<>();
private static final Map<Long, String> storeMap = new ConcurrentHashMap<>();
private static final int partitionCount = 3;
// 需要与store的application.yml的fake-pd.partition-count保持一致
private static final String[] storeAddress =
{ // 需要与store的application.yml的fake-pd.store-list保持一致
"127.0.0.1:8501", "127.0.0.1:8502", "127.0.0.1:8503"
};
/*private static String[] storeAddress = {
"127.0.0.1:9080"
};*/
// @BeforeClass
public static void init() {
for (String address : storeAddress) {
storeMap.put((long) address.hashCode(), address);
}
for (int i = 0; i < partitionCount; i++) {
leaderMap.put(i, storeMap.keySet().iterator().next());
}
HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance();
nodeManager.setNodePartitioner((builder, graphName, startKey, endKey) -> {
int startCode = PartitionUtils.calcHashcode(startKey);
int endCode = PartitionUtils.calcHashcode(endKey);
if (ALL_PARTITION_OWNER == startKey) {
storeMap.forEach((k, v) -> {
builder.add(k, -1);
});
} else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey ||
Arrays.equals(startKey, endKey)) {
builder.add(leaderMap.get(startCode % partitionCount), startCode);
} else {
Assert.fail("OwnerKey转成HashCode后已经无序了， 按照OwnerKey范围查询没意义");
builder.add(leaderMap.get(startCode % partitionCount), startCode);
builder.add(leaderMap.get(endCode % partitionCount), endCode);
}
return 0;
});
nodeManager.setNodeProvider((graphName, nodeId) -> {
System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + " " +
storeMap.get(nodeId));
return nodeManager.getNodeBuilder().setNodeId(nodeId)
.setAddress(storeMap.get(nodeId)).build();
});
nodeManager.setNodeNotifier((graphName, storeNotice) -> {
System.out.println("recv node notifier " + storeNotice);
if (storeNotice.getPartitionLeaders().size() > 0) {
leaderMap.putAll(storeNotice.getPartitionLeaders());
System.out.println("leader changed ");
leaderMap.forEach((k, v) -> {
System.out.print(" " + k + " " + v + ",");
});
System.out.println();
}
return 0;
});
}
private static HgStoreSession getStoreSession() {
return HgSessionManager.getInstance().openSession(HgStoreTestUtil.GRAPH_NAME);
}
private static HgStoreSession getStoreSession(String graph) {
return HgSessionManager.getInstance().openSession(graph);
}
// @Test
public void put_get() {
System.out.println("--- test put & get ---");
HgStoreSession session = getStoreSession();
HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY");
//HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY");
byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE");
Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value));
Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get(
HgStoreTestUtil.TABLE_NAME, key)));
}
// // @Test
// public void batchPutOwner() {
// Map<String, Map<HgOwnerKey, byte[]>> entries = new HashMap<>(2);
//
// int ownerNum = 10;
// int keyNum = 10;
// String keyPrefix = "BATCH-UNIT-TEST-";
// long amount = ownerNum * keyNum;
// Map<HgOwnerKey, byte[]> kv = new HashMap<>(keyNum, 1);
//
// for (int i = 0; i < ownerNum; i++) {
//
// for (int ii = 0; ii < keyNum; ii++) {
// HgOwnerKey ownerKey = new HgOwnerKey(toBytes("OWNER-" + i)
// , toBytes(keyPrefix + i + "-" + ii));
// byte[] value = toBytes(keyPrefix + "VALUE-" + ownerNum + "-" + ii);
// kv.put(ownerKey, value);
// }
//
// }
//
// entries.put(TABLE_NAME, kv);
// HgStoreSession session = getStoreSession();
// Assert.assertTrue(session.batchPutOwner(entries));
// System.out.println("put ok");
// Assert.assertEquals(amount, amountOf(session.scanIterator(TABLE_NAME, toAllPartitionKey
// (keyPrefix))));
//
// }
// // @Test
//
// public void batchPutWithoutStop() {
// final int loops = 6 * 10;
// final int interval = 1000 * 10;
// for(int j=0;j<loops;j++) {
// Map<String, Map<HgOwnerKey, byte[]>> entries = new HashMap<>(2);
//
// int ownerNum = 10;
// int keyNum = 10;
// String keyPrefix = "BATCH-UNIT-TEST-";
// long amount = ownerNum * keyNum;
// Map<HgOwnerKey, byte[]> kv = new HashMap<>(keyNum, 1);
//
// for (int i = 0; i < ownerNum; i++) {
//
// for (int ii = 0; ii < keyNum; ii++) {
// HgOwnerKey ownerKey = new HgOwnerKey(toBytes("OWNER-" + i * j)
// , toBytes(keyPrefix + i + "-" + ii));
// byte[] value = toBytes(keyPrefix + "VALUE-" + ownerNum + "-" + ii*j);
// kv.put(ownerKey, value);
// }
// }
//
// entries.put(TABLE_NAME, kv);
// HgStoreSession session = getStoreSession();
// Assert.assertTrue(session.batchPutOwner(entries));
// System.out.println("put ok");
//
// try {
// Thread.sleep(interval);
// } catch (InterruptedException e) {
// System.out.println(e.getMessage());
// }
// }
// }
// @Test
public void batchGet() {
System.out.println("--- test batchGet ---");
HgStoreSession session = getStoreSession();
String keyPrefix = "BATCH-GET-UNIT";
Map<HgOwnerKey, byte[]> map = HgStoreTestUtil.batchPut(session, keyPrefix);
List<HgOwnerKey> keyList =
map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList());
//printOwner(keyList);
List<HgKvEntry> resList = session.batchGetOwner(HgStoreTestUtil.TABLE_NAME, keyList);
Assert.assertFalse(isInvalid(resList));
Assert.assertEquals(resList.size(), keyList.size());
// println(list);
HgStoreTestUtil.println("--- batch-get result ---");
Assert.assertTrue((resList.stream()
.map(e -> map.containsKey(HgStoreTestUtil.toOwnerKey(e.key())))
.allMatch(Boolean::booleanValue))
);
}
// @Test
//CAUTION: ONLY FOR LONG！
//注意：目前只可以对long类型value进行Merge操作。
public void merge() {
System.out.println("--- test merge (1+1=2)---");
HgStoreSession session = getStoreSession();
String mergeKey = "merge-key";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey);
byte[] value = HgStoreTestUtil.toBytes(1L);
System.out.println("- put " + mergeKey + ":1 -");
session.put(HgStoreTestUtil.TABLE_NAME, key, value);
System.out.println("- merge " + mergeKey + ":1 -");
session.merge(HgStoreTestUtil.TABLE_NAME, key, value);
long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key));
System.out.printf("after merge " + mergeKey + "=%s%n", res);
Assert.assertEquals(2L, res);
String putStr = "19";
session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr));
byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key);
Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1));
}
// @Test
public void delete() {
System.out.println("--- test delete ---");
HgStoreSession session = getStoreSession();
String delKey = "del-key";
String delValue = "del-value";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey);
byte[] value = HgStoreTestUtil.toBytes(delValue);
HgStoreTestUtil.println("- put " + delKey + ": " + delValue);
session.put(HgStoreTestUtil.TABLE_NAME, key, value);
HgStoreTestUtil.println("- delete " + delKey);
session.delete(HgStoreTestUtil.TABLE_NAME, key);
value = session.get(HgStoreTestUtil.TABLE_NAME, key);
HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value));
Assert.assertEquals(EMPTY_BYTES, value);
}
// @Test
public void deleteSingle() {
System.out.println("--- test deleteSingle ---");
HgStoreSession session = getStoreSession();
String delKey = "del-single-key";
String delValue = "del-single-value";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey);
byte[] value = HgStoreTestUtil.toBytes(delValue);
HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue);
session.put(HgStoreTestUtil.TABLE_NAME, key, value);
value = session.get(HgStoreTestUtil.TABLE_NAME, key);
HgStoreTestUtil.println(
"- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value));
Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value));
HgStoreTestUtil.println("- delete-single : [" + delKey + "]");
session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key);
value = session.get(HgStoreTestUtil.TABLE_NAME, key);
HgStoreTestUtil.println(
"- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value));
Assert.assertEquals("", HgStoreTestUtil.toStr(value));
}
// @Test
public void deleteRange() {
HgStoreTestUtil.println("--- test deleteRange ---");
HgStoreSession session = getStoreSession();
String rangePrefix = "DEL-RANGE-KEY";
String owner = "batch-delete-owner";
Map<HgOwnerKey, byte[]> map =
HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10,
key -> {
return HgStoreTestUtil.toOwnerKey(owner, key);
});
HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00");
HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05");
Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey));
HgStoreTestUtil.println("- after delete range from ["
+ HgStoreTestUtil.toStr(startKey.getKey())
+ "] to ["
+ HgStoreTestUtil.toStr(endKey.getKey()) + "]");
for (int i = 0; i < 10; i++) {
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" +
HgStoreTestUtil.padLeftZeros(
String.valueOf(i), 2));
String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key));
HgStoreTestUtil.println(
"- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value);
// TODO: [start,end)?
if (i < 5) {
Assert.assertEquals("", value);
} else {
//println(value);
Assert.assertNotEquals("", value);
}
}
}
// @Test
public void deletePrefix() {
System.out.println("--- test deletePrefix ---");
HgStoreSession session = getStoreSession();
String prefixStr = "DEL-PREFIX-KEY";
String owner = "batch-delete-owner";
Map<HgOwnerKey, byte[]> map =
HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10,
key -> {
return HgStoreTestUtil.toOwnerKey(owner, key);
});
//printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()));
HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr);
Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator(
HgStoreTestUtil.TABLE_NAME, prefix)));
session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix);
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(
HgStoreTestUtil.TABLE_NAME, prefix)));
HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]");
for (int i = 0; i < 10; i++) {
HgOwnerKey key =
HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2));
String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key));
System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value);
Assert.assertEquals("", value);
}
}
// // @Test
// public void batchDeleteOwner() {
// System.out.println("--- test batchDelete ---");
// HgStoreSession session = getStoreSession();
// String batchPrefix = "DEL-BATCH-KEY";
// batchPut(session, batchPrefix, 10);
//
// for (int i = 0; i < 10; i++) {
// HgOwnerKey key = toOwnerKey(batchPrefix + toSuffix(i, 2));
// String value = toStr(session.get(TABLE_NAME, key));
// println("- get [" + toStr(key.getKey()) + "] = " + value);
// Assert.assertNotEquals("", value);
// }
//
// Set<HgOwnerKey> keys = new HashSet<>();
// for (int i = 0; i < 5; i++) {
// keys.add(toOwnerKey(batchPrefix + toSuffix(i, 2)));
// }
//
// Map<String, Set<HgOwnerKey>> map = new HashMap<>(1);
// map.put(TABLE_NAME, keys);
// session.batchDeleteOwner(map);
//
// for (int i = 0; i < 10; i++) {
// HgOwnerKey key = toOwnerKey(batchPrefix + toSuffix(i, 2));
// String value = toStr(session.get(TABLE_NAME, key));
// println("- get [" + toStr(key.getKey()) + "] = " + value);
// // TODO: [,)?
// if (i < 5) {
// Assert.assertEquals("", value);
// } else {
// Assert.assertNotEquals("", value);
// }
//
// }
// }
// // @Test
// public void batchDeleteRangeOwner() {
// System.out.println("--- test batchDeleteRange ---");
// HgStoreSession session = getStoreSession();
// String rangePrefix = "DEL-RANGE-KEY";
// String owner="batch-delete-owner";
//
// batchPut(session, TABLE_NAME,rangePrefix, 10,key->{
// return toOwnerKey(owner,key);
// });
// batchPut(session, TABLE_NAME2, rangePrefix, 10,key->{
// return toOwnerKey(owner,key);
// });
//
// HgOwnerKey startKey1 = toOwnerKey(owner,rangePrefix + "-03");
// HgOwnerKey endKey1 = toOwnerKey(owner,rangePrefix + "-05");
//
// HgOwnerKey startKey2 = toOwnerKey(owner,rangePrefix + "-06");
// HgOwnerKey endKey2 = toOwnerKey(owner,rangePrefix + "-09");
//
// Map<String, HgPair<HgOwnerKey, HgOwnerKey>> map = new HashMap<>();
// map.put(TABLE_NAME, new HgPair<>(startKey1, endKey1));
// map.put(TABLE_NAME2, new HgPair<>(startKey2, endKey2));
//
// session.batchDeleteRangeOwner(map);
//
// for (int i = 0; i < 10; i++) {
// HgOwnerKey key = toOwnerKey(owner,rangePrefix + toSuffix(i, 2));
// String value = toStr(session.get(TABLE_NAME, key));
// println("- get [" + toStr(key.getKey()) + "] = " + value);
//
// if (i >= 3 && i < 5) {
// Assert.assertEquals("", value);
// } else {
// Assert.assertNotEquals("", value);
// }
//
// }
//
// for (int i = 0; i < 10; i++) {
// HgOwnerKey key = toOwnerKey(owner,rangePrefix + toSuffix(i, 2));
// String value = toStr(session.get(TABLE_NAME2, key));
// println("- get [" + toStr(key.getKey()) + "] = " + value);
// if (i >= 6 && i < 9) {
// Assert.assertEquals("", value);
// } else {
// Assert.assertNotEquals("", value);
// }
//
// }
// }
// // @Test
// public void batchDeletePrefix() {
// System.out.println("--- test batchDeletePrefix ---");
// HgStoreSession session = getStoreSession();
// String batchKey1 = "DEL-PREFIX-BATCH-1-KEY";
// String batchKey2 = "DEL-PREFIX-BATCH-2-KEY";
// String owner="batch-delete-owner";
//
// batchPut(session, TABLE_NAME, batchKey1, 10,key->{
// return toOwnerKey(owner,key);
// });
// batchPut(session, TABLE_NAME2, batchKey2, 10,key->{
// return toOwnerKey(owner,key);
// });
//
// HgOwnerKey prefix1 = toOwnerKey(owner,batchKey1);
// HgOwnerKey prefix2 = toOwnerKey(owner,batchKey2);
//
// Set<HgOwnerKey> set = new HashSet<>();
//
// set.add(prefix1);
// set.add(prefix2);
//
// Map<String, Set<HgOwnerKey>> map = new HashMap<>();
// map.put(TABLE_NAME, set);
// map.put(TABLE_NAME2, set);
//
// Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME, prefix1)));
// Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME2, prefix2)));
//
// session.batchDeletePrefixOwner(map);
//
// Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME, prefix1)));
// Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME2, prefix2)));
//
// }
// @Test
public void scanIterator() {
HgStoreTestUtil.println("--- test scanIterator ---");
String tableName = "UNIT_SCAN";
String keyName = "SCAN-ITER";
HgStoreSession session = getStoreSession();
HgStoreTestUtil.batchPut(session, tableName, keyName, 10000);
int count = 0;
int limit = 0;
int max = 99999;
HgKvIterator<HgKvEntry> iterator = null;
HgStoreTestUtil.println("-- test 0 element --");
iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"),
HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0);
Assert.assertFalse(iterator.hasNext());
try {
iterator.next();
Assert.fail();
} catch (Throwable t) {
HgStoreTestUtil.println("-- test NoSuchElementException --");
Assert.assertTrue(t instanceof NoSuchElementException);
}
HgStoreTestUtil.println("-- test limit 1 to 10 --");
for (int i = 1; i <= 10; i++) {
HgStoreTestUtil.println("- limit " + i + " -");
limit = i;
iterator = session.scanIterator(tableName,
HgStoreTestUtil.toAllPartitionKey(keyName + "-0"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-1"),
limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
HgStoreTestUtil.println(entry);
}
Assert.assertEquals(limit, count);
}
HgStoreTestUtil.println("-- test limit 1 to 10 not enough --");
for (int i = 1; i <= 10; i++) {
HgStoreTestUtil.println("- limit " + i + " -");
limit = i;
iterator = session.scanIterator(tableName,
HgStoreTestUtil.toAllPartitionKey(keyName + "-00001"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-00005"),
limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
HgStoreTestUtil.println(entry);
}
if (i <= 5) {
Assert.assertEquals(limit, count);
} else {
Assert.assertEquals(5, count);
}
}
HgStoreTestUtil.println("-- test limit 0 (no limit) --");
limit = 0;
iterator =
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 1000 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(10000, count);
HgStoreTestUtil.println("-- test scan all --");
iterator = session.scanIterator(tableName);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 1000 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(10000, count);
HgStoreTestUtil.println("-- test scan prefix --");
iterator =
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-01"));
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 100 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(1000, count);
}
// @Test
public void truncate() {
HgStoreTestUtil.println("--- test truncate ---");
String graph = "graph_truncate";
String tableName = "UNIT_TRUNCATE_1";
String keyName = "KEY_TRUNCATE";
HgStoreSession session = getStoreSession(graph);
HgStoreTestUtil.batchPut(session, tableName, keyName, 100);
Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName)));
String tableName2 = "UNIT_TRUNCATE_2";
HgStoreTestUtil.batchPut(session, tableName2, keyName, 100);
Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2)));
session.truncate();
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName)));
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2)));
}
//// @Test
public void scanIteratorHuge() {
/*************** test no limit, with 10 millions **************/
String tableName = "UNIT_HUGE";
String keyName = "SCAN-HUGE";
int amount = 10_000_000;
HgStoreSession session = getStoreSession();
if (HgStoreTestUtil.amountOf(
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) <
10) {
HgStoreTestUtil.batchPut(session, tableName, keyName, amount);
}
int count = 0;
HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName);
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % (amount / 10) == 0) {
HgStoreTestUtil.println(entry);
}
}
Assert.assertEquals(amount, count);
}
}

981
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerRaftPDTest.java Normal file
View File

@ -0,0 +1,981 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store;
import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.client.PDConfig;
import org.apache.hugegraph.store.client.grpc.KvCloseableIterator;
import org.apache.hugegraph.store.client.util.HgStoreClientConfig;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.client.util.MetricX;
import org.apache.hugegraph.store.grpc.common.ScanOrderType;
import org.apache.hugegraph.store.util.HgStoreTestUtil;
import org.junit.Assert;
import org.junit.Test;
import lombok.extern.slf4j.Slf4j;
/**
* 使用pd，支持raft的单元测试
*/
@Slf4j
public class HgSessionManagerRaftPDTest {
public final static String GRAPH_NAME_X = "default/hugegraph/x";
public final static String GRAPH_NAME_Y = "default/hugegraph/y";
public final static String GRAPH_NAME_Z = "default/hugegraph/z";
public final static String TABLE_NAME_1 = "table1";
public final static String TABLE_NAME_2 = "table2";
public final static String TABLE_NAME_3 = "table3";
private static final String pdAddress = "localhost:8686";
public static HgStoreClient storeClient;
private static PDClient pdClient;
// @BeforeClass
public static void init() {
pdClient = PDClient.create(PDConfig.of(pdAddress).setEnableCache(true));
storeClient = HgStoreClient.create(pdClient);
}
private static HgStoreSession getStoreSession() {
return storeClient.openSession(HgStoreTestUtil.GRAPH_NAME);
}
private static HgStoreSession getStoreSession(String graph) {
return storeClient.openSession(graph);
}
// @Test
public void put_get() {
System.out.println("--- test put & get ---");
HgStoreSession session = getStoreSession(HgStoreTestUtil.GRAPH_NAME);
HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY");
//HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY");
byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE");
// Assert.assertTrue(session.put(TABLE_NAME, key, value));
// Assert.assertEquals(toStr(value), toStr(session.get(TABLE_NAME, key)));
long stime = System.currentTimeMillis();
HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, "BATCH-PUT-TEST", 30000);
System.out.println("Time is " + (System.currentTimeMillis() - stime));
}
public void put_get2() {
System.out.println("--- test put & get ---");
HgStoreSession session = getStoreSession("testGraph");
HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY");
byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE");
long stime = System.currentTimeMillis();
HgStoreTestUtil.batchPut(session, "testTable", "BATCH-PUT-TEST", 30000);
System.out.println("Time is " + (System.currentTimeMillis() - stime));
}
// @Test
public void scanPartition() {
HgStoreTestUtil.println("--- test scanPartition ---");
HgStoreSession session = getStoreSession();
// System.out.println(amountOf(session.scanIterator(TABLE_NAME)));
Iterator iterator = session.scanIterator(HgStoreTestUtil.TABLE_NAME, 0, 65535,
HgKvStore.SCAN_HASHCODE, EMPTY_BYTES);
System.out.println(HgStoreTestUtil.amountOf(iterator));
}
// @Test
public void check() {
System.out.println("--- test check ---");
HgStoreSession session = getStoreSession();
HgKvIterator<HgKvEntry> iterator = session.scanIterator(HgStoreTestUtil.TABLE_NAME);
int count = 0;
while (iterator.hasNext()) {
iterator.next();
count++;
}
System.out.println(count);
}
// @Test
public void put_get_unique() {
System.out.println("--- test put & get ---");
HgStoreSession session = getStoreSession();
// add timestamp into key to avoid key duplication
Date date = new Date();
SimpleDateFormat formatter = new SimpleDateFormat("HHmmss");
String timestamp = formatter.format(date);
HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY-" + timestamp);
//HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY");
byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE");
Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value));
Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get(
HgStoreTestUtil.TABLE_NAME, key)));
}
@Test
public void testBatchPutExt() throws IOException {
System.out.println("--- test batchPut ---");
HgStoreSession session = getStoreSession();
String keyPrefix = "BATCH-GET-UNIT";
Map<HgOwnerKey, byte[]> map = HgStoreTestUtil.batchPut(session, keyPrefix, 1000);
File outputFile = new File("tmp/batch_put_list");
if (outputFile.exists()) {
FileUtils.forceDelete(outputFile);
}
FileUtils.forceMkdir(new File("tmp/"));
ObjectOutputStream oo = new ObjectOutputStream(new FileOutputStream(outputFile));
oo.writeObject(map);
oo.close();
System.out.printf("%d entries have be put into graph %s\n", map.size(),
HgStoreTestUtil.GRAPH_NAME);
int count = 0;
HgKvIterator<HgKvEntry> iterator = null;
iterator = session.scanIterator(HgStoreTestUtil.TABLE_NAME);
while (iterator.hasNext()) {
iterator.next();
count++;
}
Assert.assertEquals(1000, count);
}
// @Test
public void testBatchGetExt() throws IOException, ClassNotFoundException {
File outputFile = new File("tmp/batch_put_list");
ObjectInputStream ois = new ObjectInputStream(new FileInputStream(outputFile));
Map<HgOwnerKey, byte[]> map = (Map<HgOwnerKey, byte[]>) ois.readObject();
ois.close();
System.out.printf("%d entries get from %s\n", map.size(), outputFile.getPath());
HgStoreSession session = getStoreSession();
List<HgOwnerKey> keyList =
map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList());
List<HgKvEntry> resList = session.batchGetOwner(HgStoreTestUtil.TABLE_NAME, keyList);
Assert.assertTrue((resList.stream()
.map(e -> map.containsKey(HgStoreTestUtil.toOwnerKey(e.key())))
.allMatch(Boolean::booleanValue))
);
}
@Test
public void testBatchPutUniqueExt() throws IOException {
System.out.println("--- test batchPut ---");
HgStoreSession session = getStoreSession();
// add timestamp into key to avoid key duplication
Date date = new Date();
SimpleDateFormat formatter = new SimpleDateFormat("HHmmss");
String timestamp = formatter.format(date);
String keyPrefix = "BATCH-GET-UNIT-" + timestamp;
Map<HgOwnerKey, byte[]> map = HgStoreTestUtil.batchPut(session, keyPrefix, 300000);
File outputFile = new File("tmp/batch_put_list");
if (outputFile.exists()) {
FileUtils.forceDelete(outputFile);
}
FileUtils.forceMkdir(new File("tmp/"));
ObjectOutputStream oo = new ObjectOutputStream(new FileOutputStream(outputFile));
oo.writeObject(map);
oo.close();
System.out.printf("%d entries have be put into graph %s\n", map.size(),
HgStoreTestUtil.GRAPH_NAME);
}
// @Test
public void testBatchPutMultiGraph() throws IOException {
System.out.println("--- test testBatchPutMultiGraph ---");
Date date = new Date();
SimpleDateFormat formatter = new SimpleDateFormat("HHmmss");
String timestamp = formatter.format(date);
HgStoreSession session1 = getStoreSession(GRAPH_NAME_X);
HgStoreSession session2 = getStoreSession(GRAPH_NAME_Y);
HgStoreSession session3 = getStoreSession(GRAPH_NAME_Z);
String keyPrefix1 = "BATCH-PUT-UNIT-X-" + timestamp;
String keyPrefix2 = "BATCH-PUT-UNIT-Y-" + timestamp;
String keyPrefix3 = "BATCH-PUT-UNIT-Z-" + timestamp;
HgStoreTestUtil.batchPut(session1, TABLE_NAME_1, keyPrefix1, 1000);
HgStoreTestUtil.batchPut(session1, TABLE_NAME_2, keyPrefix1, 1000);
HgStoreTestUtil.batchPut(session1, TABLE_NAME_3, keyPrefix1, 1000);
HgStoreTestUtil.batchPut(session2, TABLE_NAME_1, keyPrefix2, 1000);
HgStoreTestUtil.batchPut(session2, TABLE_NAME_2, keyPrefix2, 1000);
HgStoreTestUtil.batchPut(session2, TABLE_NAME_3, keyPrefix2, 1000);
HgStoreTestUtil.batchPut(session3, TABLE_NAME_1, keyPrefix3, 1000);
HgStoreTestUtil.batchPut(session3, TABLE_NAME_2, keyPrefix3, 1000);
HgStoreTestUtil.batchPut(session3, TABLE_NAME_3, keyPrefix3, 1000);
}
// @Test
//CAUTION: ONLY FOR LONG！
//注意：目前只可以对long类型value进行Merge操作。
public void merge() {
System.out.println("--- test merge (1+1=2)---");
HgStoreSession session = getStoreSession();
String mergeKey = "merge-key";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey);
byte[] value = HgStoreTestUtil.toBytes(1L);
System.out.println("- put " + mergeKey + ":1 -");
session.put(HgStoreTestUtil.TABLE_NAME, key, value);
System.out.println("- merge " + mergeKey + ":1 -");
session.merge(HgStoreTestUtil.TABLE_NAME, key, value);
long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key));
System.out.printf("after merge " + mergeKey + "=%s%n", res);
Assert.assertEquals(2L, res);
String putStr = "19";
session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr));
byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key);
Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1));
}
// @Test
public void delete() {
System.out.println("--- test delete ---");
HgStoreSession session = getStoreSession();
String delKey = "del-key";
String delValue = "del-value";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey);
byte[] value = HgStoreTestUtil.toBytes(delValue);
HgStoreTestUtil.println("- put " + delKey + ": " + delValue);
session.put(HgStoreTestUtil.TABLE_NAME, key, value);
HgStoreTestUtil.println("- delete " + delKey);
session.delete(HgStoreTestUtil.TABLE_NAME, key);
value = session.get(HgStoreTestUtil.TABLE_NAME, key);
HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value));
Assert.assertEquals(EMPTY_BYTES, value);
}
// @Test
public void deleteSingle() {
System.out.println("--- test deleteSingle ---");
HgStoreSession session = getStoreSession();
String delKey = "del-single-key";
String delValue = "del-single-value";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey);
byte[] value = HgStoreTestUtil.toBytes(delValue);
HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue);
session.put(HgStoreTestUtil.TABLE_NAME, key, value);
value = session.get(HgStoreTestUtil.TABLE_NAME, key);
HgStoreTestUtil.println(
"- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value));
Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value));
HgStoreTestUtil.println("- delete-single : [" + delKey + "]");
session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key);
value = session.get(HgStoreTestUtil.TABLE_NAME, key);
HgStoreTestUtil.println(
"- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value));
Assert.assertEquals("", HgStoreTestUtil.toStr(value));
}
// @Test
public void deleteRange() {
HgStoreTestUtil.println("--- test deleteRange ---");
HgStoreSession session = getStoreSession();
String rangePrefix = "DEL-RANGE-KEY";
String owner = "batch-delete-owner";
Map<HgOwnerKey, byte[]> map =
HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10,
key -> {
return HgStoreTestUtil.toOwnerKey(owner, key);
});
HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00");
HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05");
Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey));
HgStoreTestUtil.println("- after delete range from ["
+ HgStoreTestUtil.toStr(startKey.getKey())
+ "] to ["
+ HgStoreTestUtil.toStr(endKey.getKey()) + "]");
for (int i = 0; i < 10; i++) {
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" +
HgStoreTestUtil.padLeftZeros(
String.valueOf(i), 2));
String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key));
HgStoreTestUtil.println(
"- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value);
// TODO: [start,end)?
if (i < 5) {
Assert.assertEquals("", value);
} else {
//println(value);
Assert.assertNotEquals("", value);
}
}
}
// @Test
public void deletePrefix() {
System.out.println("--- test deletePrefix ---");
HgStoreSession session = getStoreSession();
String prefixStr = "DEL-PREFIX-KEY";
String owner = "batch-delete-owner";
Map<HgOwnerKey, byte[]> map =
HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10,
key -> {
return HgStoreTestUtil.toOwnerKey(owner, key);
});
//printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()));
HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr);
Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator(
HgStoreTestUtil.TABLE_NAME, prefix)));
session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix);
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(
HgStoreTestUtil.TABLE_NAME, prefix)));
HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]");
for (int i = 0; i < 10; i++) {
HgOwnerKey key =
HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2));
String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key));
System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value);
Assert.assertEquals("", value);
}
}
// // @Test
// public void batchDeleteOwner() {
// System.out.println("--- test batchDelete ---");
// HgStoreSession session = getStoreSession();
// String batchPrefix = "DEL-BATCH-KEY";
// batchPut(session, batchPrefix, 10);
//
// for (int i = 0; i < 10; i++) {
// HgOwnerKey key = toOwnerKey(batchPrefix + toSuffix(i, 2));
// String value = toStr(session.get(TABLE_NAME, key));
// println("- get [" + toStr(key.getKey()) + "] = " + value);
// Assert.assertNotEquals("", value);
// }
//
// Set<HgOwnerKey> keys = new HashSet<>();
// for (int i = 0; i < 5; i++) {
// keys.add(toOwnerKey(batchPrefix + toSuffix(i, 2)));
// }
//
// Map<String, Set<HgOwnerKey>> map = new HashMap<>(1);
// map.put(TABLE_NAME, keys);
// session.batchDeleteOwner(map);
//
// for (int i = 0; i < 10; i++) {
// HgOwnerKey key = toOwnerKey(batchPrefix + toSuffix(i, 2));
// String value = toStr(session.get(TABLE_NAME, key));
// println("- get [" + toStr(key.getKey()) + "] = " + value);
// // TODO: [,)?
// if (i < 5) {
// Assert.assertEquals("", value);
// } else {
// Assert.assertNotEquals("", value);
// }
//
// }
// }
// // @Test
// public void batchDeleteRangeOwner() {
// System.out.println("--- test batchDeleteRange ---");
// HgStoreSession session = getStoreSession();
// String rangePrefix = "DEL-RANGE-KEY";
// String owner="batch-delete-owner";
//
// batchPut(session, TABLE_NAME,rangePrefix, 10,key->{
// return toOwnerKey(owner,key);
// });
// batchPut(session, TABLE_NAME2, rangePrefix, 10,key->{
// return toOwnerKey(owner,key);
// });
//
// HgOwnerKey startKey1 = toOwnerKey(owner,rangePrefix + "-03");
// HgOwnerKey endKey1 = toOwnerKey(owner,rangePrefix + "-05");
//
// HgOwnerKey startKey2 = toOwnerKey(owner,rangePrefix + "-06");
// HgOwnerKey endKey2 = toOwnerKey(owner,rangePrefix + "-09");
//
// Map<String, HgPair<HgOwnerKey, HgOwnerKey>> map = new HashMap<>();
// map.put(TABLE_NAME, new HgPair<>(startKey1, endKey1));
// map.put(TABLE_NAME2, new HgPair<>(startKey2, endKey2));
//
// session.batchDeleteRangeOwner(map);
//
// for (int i = 0; i < 10; i++) {
// HgOwnerKey key = toOwnerKey(owner,rangePrefix + toSuffix(i, 2));
// String value = toStr(session.get(TABLE_NAME, key));
// println("- get [" + toStr(key.getKey()) + "] = " + value);
//
// if (i >= 3 && i < 5) {
// Assert.assertEquals("", value);
// } else {
// Assert.assertNotEquals("", value);
// }
//
// }
//
// for (int i = 0; i < 10; i++) {
// HgOwnerKey key = toOwnerKey(owner,rangePrefix + toSuffix(i, 2));
// String value = toStr(session.get(TABLE_NAME2, key));
// println("- get [" + toStr(key.getKey()) + "] = " + value);
// if (i >= 6 && i < 9) {
// Assert.assertEquals("", value);
// } else {
// Assert.assertNotEquals("", value);
// }
//
// }
// }
// // @Test
// public void batchDeletePrefix() {
// System.out.println("--- test batchDeletePrefix ---");
// HgStoreSession session = getStoreSession();
// String batchKey1 = "DEL-PREFIX-BATCH-1-KEY";
// String batchKey2 = "DEL-PREFIX-BATCH-2-KEY";
// String owner="batch-delete-owner";
//
// batchPut(session, TABLE_NAME, batchKey1, 10,key->{
// return toOwnerKey(owner,key);
// });
// batchPut(session, TABLE_NAME2, batchKey2, 10,key->{
// return toOwnerKey(owner,key);
// });
//
// HgOwnerKey prefix1 = toOwnerKey(owner,batchKey1);
// HgOwnerKey prefix2 = toOwnerKey(owner,batchKey2);
//
// Set<HgOwnerKey> set = new HashSet<>();
//
// set.add(prefix1);
// set.add(prefix2);
//
// Map<String, Set<HgOwnerKey>> map = new HashMap<>();
// map.put(TABLE_NAME, set);
// map.put(TABLE_NAME2, set);
//
// Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME, prefix1)));
// Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME2, prefix2)));
//
// session.batchDeletePrefixOwner(map);
//
// Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME, prefix1)));
// Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME2, prefix2)));
//
// }
// @Test
public void scanIterator() {
HgStoreTestUtil.println("--- test scanIterator ---");
String tableName = "UNIT_SCAN";
String keyName = "SCAN-ITER";
HgStoreSession session = getStoreSession();
HgStoreTestUtil.batchPut(session, tableName, keyName, 10000);
int count = 0;
int limit = 0;
int max = 99999;
HgKvIterator<HgKvEntry> iterator = null;
HgStoreTestUtil.println("-- test 0 element --");
iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"),
HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0);
Assert.assertFalse(iterator.hasNext());
try {
iterator.next();
Assert.fail();
} catch (Throwable t) {
HgStoreTestUtil.println("-- test NoSuchElementException --");
Assert.assertTrue(t instanceof NoSuchElementException);
}
HgStoreTestUtil.println("-- test limit 1 to 10 --");
for (int i = 1; i <= 10; i++) {
HgStoreTestUtil.println("- limit " + i + " -");
limit = i;
iterator = session.scanIterator(tableName,
HgStoreTestUtil.toAllPartitionKey(keyName + "-0"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-1"),
limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
HgStoreTestUtil.println(entry);
}
Assert.assertEquals(limit, count);
}
HgStoreTestUtil.println("-- test limit 1 to 10 not enough --");
for (int i = 1; i <= 10; i++) {
HgStoreTestUtil.println("- limit " + i + " -");
limit = i;
iterator = session.scanIterator(tableName,
HgStoreTestUtil.toAllPartitionKey(keyName + "-00001"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-00005"),
limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
HgStoreTestUtil.println(entry);
}
if (i <= 5) {
Assert.assertEquals(limit, count);
} else {
Assert.assertEquals(5, count);
}
}
HgStoreTestUtil.println("-- test limit 0 (no limit) --");
limit = 0;
iterator =
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 1000 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(10000, count);
HgStoreTestUtil.println("-- test scan all --");
iterator = session.scanIterator(tableName);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 1000 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(10000, count);
HgStoreTestUtil.println("-- test scan prefix --");
iterator =
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-01"));
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 100 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(1000, count);
}
// @Test
public void paging() {
HgStoreTestUtil.println("--- test scanIterator_range ---");
String graph = "UNIT/paging";
String tableName = "UNIT_SCAN_PAGING";
String keyName = "SCAN-PAGING";
int keyAmt = 100;
HgStoreSession session = getStoreSession(graph);
HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt);
int count = 0;
int limit = 0;
int max = 99999;
HgKvIterator<HgKvEntry> iterator = null;
count = 0;
iterator = session.scanIterator(tableName
, HgStoreTestUtil.toAllPartitionKey(keyName + "-000")
, HgStoreClientConst.EMPTY_OWNER_KEY
, 0, HgKvStore.SCAN_ANY, EMPTY_BYTES
);
//HgStoreTestUtil.println(iterator);
// Assert.assertEquals(100, HgStoreTestUtil.amountOf(iterator));
List<byte[]> positionList = new LinkedList<>();
while (iterator.hasNext()) {
HgStoreTestUtil.print((count++) + " ");
HgKvEntry entry = iterator.next();
HgStoreTestUtil.print(entry);
HgStoreTestUtil.print(" " + Arrays.toString(iterator.position()) + "\n");
positionList.add(iterator.position());
if (count >= max) break;
}
/* iterator = session.scanIterator(tableName
, toAllPartitionKey(keyName + "-000")
, HgStoreClientConst.EMPTY_OWNER_KEY
, 0, HgKvStore.SCAN_ANY, EMPTY_BYTES
);
byte[] position=positionList.get(50);
println("seek: "+Arrays.toString(position));
iterator.seek(position);
//HgStoreTestUtil.println(iterator);
Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator));*/
iterator = session.scanIterator(tableName, 100);
byte[] position = positionList.get(50);
HgStoreTestUtil.println("seek: " + Arrays.toString(position));
iterator.seek(position);
HgStoreTestUtil.println(iterator);
//Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator));
}
// @Test
public void truncate() {
HgStoreTestUtil.println("--- test truncate ---");
String graph = "graph_truncate";
String tableName = "UNIT_TRUNCATE_1";
String keyName = "KEY_TRUNCATE";
HgStoreSession session = getStoreSession(graph);
HgStoreTestUtil.batchPut(session, tableName, keyName, 100);
Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName)));
String tableName2 = "UNIT_TRUNCATE_2";
HgStoreTestUtil.batchPut(session, tableName2, keyName, 100);
Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2)));
session.truncate();
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName)));
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2)));
}
//// @Test
public void scanIteratorHuge() {
/*************** test no limit, with 10 millions **************/
String tableName = "UNIT_HUGE";
String keyName = "SCAN-HUGE";
int amount = 10_000_000;
HgStoreSession session = getStoreSession();
if (HgStoreTestUtil.amountOf(
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) <
10) {
HgStoreTestUtil.batchPut(session, tableName, keyName, amount);
}
int count = 0;
HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName);
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % (amount / 10) == 0) {
HgStoreTestUtil.println(entry);
}
}
Assert.assertEquals(amount, count);
}
// @Test
public void scanTable() {
HgStoreSession session = getStoreSession("DEFAULT/hg1/g");
HgStoreTestUtil.println(session.scanIterator("g+v", 10));
}
// @Test
public void testDelGraph() {
HgStoreSession session = getStoreSession();
session.deleteGraph(HgStoreTestUtil.GRAPH_NAME);
}
// @Test
public void benchmark_scanBatch() {
HgStoreTestUtil.println("--- Benchmark scanBatch ---");
String tableName = "Benchmark_SCAN_BATCH";
String keyName = "SCAN-BATCH";
int keyAmt = 30001;
HgStoreSession session = getStoreSession();
/*if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 1)) < 1) {
batchPut(session, tableName, keyName, keyAmt);
}*/
HgStoreTestUtil.println("-- Starting scan --");
MetricX metrics = MetricX.ofStart();
//HgStoreTestUtil.println(session.scanIterator(tableName));
List<HgKvIterator<HgKvEntry>> iterators = session.scanBatch(HgScanQuery.tableOf(tableName));
Assert.assertEquals(keyAmt, HgStoreTestUtil.amountIn(iterators));
metrics.end();
log.info("*************************************************");
log.info("*********** Batch Scan Completed **************");
log.info("Total : {} (sec)", metrics.past() / 1000);
log.info(" Sum : {} (sec)", MetricX.getIteratorWait() / 1000);
log.info(" Amt : {} (scans).", MetricX.getIteratorCount());
log.info(" Avg : {} (ms)", MetricX.getIteratorWaitAvg());
log.info(" Max : {} (ms)", MetricX.getIteratorWaitMax());
log.info(" Fail : {} (times)", metrics.getFailureCount());
log.info(" Page : {} (KVs)", HgStoreClientConfig.of().getNetKvScannerPageSize());
log.info("Iters : {}", iterators.size());
log.info("*************************************************");
}
@Test
public void benchmark_scanBatch2() throws IOException {
HgStoreTestUtil.println("--- Benchmark scanBatch2 ---");
//String tableName = TABLE_NAME;
HgStoreSession session = getStoreSession();
String keyName = "SCAN-BATCH";
int keyAmt = 3000;
Map<HgOwnerKey, byte[]> data =
HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, keyName, keyAmt,
//(key) -> toOwnerKey(key.substring(0, keyName.length()
// + 2), key)
(key) -> HgStoreTestUtil.toOwnerKey(0, key)
);
HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME2, keyName, keyAmt,
//(key) -> toOwnerKey(key.substring(0, keyName.length() + 2), key)
(key) -> HgStoreTestUtil.toOwnerKey(0, key)
);
MetricX metrics = MetricX.ofStart();
long t = System.currentTimeMillis();
int count = 0;
String queryTable = null;
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterators = null;
List<HgOwnerKey> queryKeys = null;
List<HgOwnerKey> keys = new ArrayList<>();
data.forEach((k, v) -> keys.add(k));
List<HgOwnerKey> prefixKeys = new ArrayList<>();
prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-0"));
prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-1"));
prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-2"));
prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-3"));
HgStoreTestUtil.println("-- test every key, one table --");
count = 0;
queryTable = HgStoreTestUtil.TABLE_NAME;
queryKeys = keys;
iterators = session.scanBatch2(
HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(),
ScanOrderType.ORDER_NONE)
.builder()
.setScanType(0x40)
.build());
while (iterators.hasNext()) {
HgKvIterator<HgKvEntry> iterator = iterators.next();
count += HgStoreTestUtil.amountOf(iterator);
}
iterators.close();
Assert.assertEquals(keyAmt, count);
log.info(" size is {}", count);
HgStoreTestUtil.println("-- test prefix key, one table --");
count = 0;
queryTable = HgStoreTestUtil.TABLE_NAME;
queryKeys = prefixKeys;
iterators = session.scanBatch2(
HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(),
ScanOrderType.ORDER_STRICT)
.builder()
.setScanType(0x40)
.build());
while (iterators.hasNext()) {
HgKvIterator<HgKvEntry> iterator = iterators.next();
count += HgStoreTestUtil.amountOf(iterator);
}
iterators.close();
Assert.assertEquals(keyAmt, count);
log.info(" size is {}", count);
HgStoreTestUtil.println("-- test prefix key, two table --");
count = 0;
queryTable = HgStoreTestUtil.TABLE_NAME + "," + HgStoreTestUtil.TABLE_NAME2;
queryKeys = prefixKeys;
iterators = session.scanBatch2(
HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(),
ScanOrderType.ORDER_NONE)
.builder()
.setScanType(0x40)
.build());
while (iterators.hasNext()) {
HgKvIterator<HgKvEntry> iterator = iterators.next();
count += HgStoreTestUtil.amountOf(iterator);
}
iterators.close();
Assert.assertEquals(keyAmt * 2, count);
log.info(" size is {}", count);
HgStoreTestUtil.println("-- test prefix key, two table, perKeyMax --");
count = 0;
queryTable = HgStoreTestUtil.TABLE_NAME + "," + HgStoreTestUtil.TABLE_NAME2;
queryKeys = prefixKeys;
queryKeys.remove(queryKeys.size() - 1);//remove the last one.
iterators = session.scanBatch2(
HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(),
ScanOrderType.ORDER_NONE)
.builder()
.setScanType(0x40)
.setPerKeyMax(10)
.build());
while (iterators.hasNext()) {
HgKvIterator<HgKvEntry> iterator = iterators.next();
count += HgStoreTestUtil.amountOf(iterator);
}
iterators.close();
Assert.assertEquals(queryKeys.size() * 10, count);
log.info(" size is {}", count);
keys.clear();
log.info("time is {}", System.currentTimeMillis() - t);
metrics.end();
log.info("*************************************************");
log.info("*********** Batch Scan Completed **************");
log.info("Total : {} (sec)", metrics.past() / 1000);
log.info(" Sum : {} (sec)", MetricX.getIteratorWait() / 1000);
log.info(" Amt : {} (scans).", MetricX.getIteratorCount());
log.info(" Avg : {} (ms)", MetricX.getIteratorWaitAvg());
log.info(" Max : {} (ms)", MetricX.getIteratorWaitMax());
log.info(" Fail : {} (times)", metrics.getFailureCount());
log.info(" Page : {} (KVs)", HgStoreClientConfig.of().getNetKvScannerPageSize());
log.info(" size is {}", count);
log.info("*************************************************");
}
// @Test
public void benchmark_scanBatch_SkipDegree() throws IOException {
HgStoreTestUtil.println("--- Benchmark scanBatch2 1Owner---");
String tableName = HgStoreTestUtil.TABLE_NAME;
HgStoreSession session = getStoreSession();
String keyName = "SCAN-BATCH";
int keyAmt = 300000;
byte[] owner = "Owner".getBytes();
Map<HgOwnerKey, byte[]> data =
HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt, key -> {
return HgStoreTestUtil.toOwnerKey(owner, key);
});
HgStoreTestUtil.println("-- Starting scan --");
MetricX metrics = MetricX.ofStart();
//HgStoreTestUtil.println(session.scanIterator(tableName));
long t = System.currentTimeMillis();
int count = 0;
{
List<HgOwnerKey> keys = new ArrayList<>();
keys.add(HgStoreTestUtil.toOwnerKey(owner, keyName));
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterators =
session.scanBatch2(HgScanQuery.prefixIteratorOf(tableName, keys.iterator(),
ScanOrderType.ORDER_NONE)
.builder().setScanType(0x40).setSkipDegree(1)
.build());
//KvCloseableIterator<HgKvIterator<Kv>> iterators = session.scanBatch2(HgScanQuery
// .tableOf(tableName));
while (iterators.hasNext()) {
HgKvIterator<HgKvEntry> iterator = iterators.next();
// while (iterator.hasNext()){
// System.out.println(new String(iterator.next().key()));
// }
count += HgStoreTestUtil.amountOf(iterator);
// log.info("{} - {}", new String(iterator.key()), iterator.value());
}
keys.clear();
iterators.close();
iterators = null;
}
log.info("time is {}", System.currentTimeMillis() - t);
metrics.end();
log.info("*************************************************");
log.info("*********** Batch Scan Completed **************");
log.info("Total : {} (sec)", metrics.past() / 1000);
log.info(" Sum : {} (sec)", MetricX.getIteratorWait() / 1000);
log.info(" Amt : {} (scans).", MetricX.getIteratorCount());
log.info(" Avg : {} (ms)", MetricX.getIteratorWaitAvg());
log.info(" Max : {} (ms)", MetricX.getIteratorWaitMax());
log.info(" Fail : {} (times)", metrics.getFailureCount());
log.info(" Page : {} (KVs)", HgStoreClientConfig.of().getNetKvScannerPageSize());
log.info(" size is {}", count);
log.info("*************************************************");
}
}

1125
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerTest.java Normal file
View File

File diff suppressed because it is too large Load Diff

236
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/PartitionEngineTest.java Normal file
View File

@ -0,0 +1,236 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.cmd.HgCmdClient;
import org.apache.hugegraph.store.meta.Partition;
import org.junit.Assert;
import com.alipay.sofa.jraft.option.RpcOptions;
public class PartitionEngineTest {
// @Test
public void test() {
String[] peers = new String[]{"1", "2", "6"};
String[] learners = new String[]{"3", "7"};
List<String> oldPeers1 = Arrays.asList("1", "2", "3");
List<String> oldLearner1 = Arrays.asList("4", "5");
List<String> oldPeers =
oldPeers1.stream().map(peer -> peer).collect(Collectors.toList());
oldPeers.addAll(
oldLearner1.stream().map(peer -> peer).collect(Collectors.toList()));
List<String> addedNodes = Arrays.stream(peers).filter(peer ->
!oldPeers.stream()
.map(e -> e)
.collect(
Collectors.toSet())
.contains(peer))
.collect(Collectors.toList());
addedNodes.addAll(Arrays.stream(learners).filter(peer ->
!oldPeers.stream()
.map(e -> e)
.collect(
Collectors.toSet())
.contains(peer))
.collect(Collectors.toList()));
List<String> removedNodes = oldPeers.stream().filter(peer ->
!Arrays.stream(peers)
.map(e -> e).collect(
Collectors.toSet())
.contains(peer))
.collect(Collectors.toList());
removedNodes = removedNodes.stream().filter(peer ->
!Arrays.stream(learners).map(e -> e)
.collect(Collectors.toSet())
.contains(peer))
.collect(Collectors.toList());
List<String> mixedPeer = oldPeers1.stream().filter(peer ->
Arrays.stream(learners)
.map(e -> e).collect(
Collectors.toSet())
.contains(peer))
.collect(Collectors.toList());
// 新增 6、7
Assert.assertEquals(2, addedNodes.size());
addedNodes.clear();
addedNodes.addAll(Arrays.asList(peers));
addedNodes.addAll(Arrays.asList(learners));
addedNodes.removeAll(oldPeers);
Assert.assertEquals(2, addedNodes.size());
addedNodes.forEach(s -> System.out.print(s + " "));
System.out.println();
// 删除 4，5
Assert.assertEquals(2, removedNodes.size());
removedNodes.clear();
removedNodes.addAll(oldPeers);
removedNodes.removeAll(Arrays.asList(peers));
removedNodes.removeAll(Arrays.asList(learners));
Assert.assertEquals(2, removedNodes.size());
removedNodes.forEach(s -> System.out.print(s + " "));
System.out.println();
// 交集 5
Assert.assertEquals(1, mixedPeer.size());
oldPeers1.removeAll(Arrays.asList(learners));
Assert.assertEquals(1, oldPeers1.size());
mixedPeer.forEach(s -> System.out.print(s + " "));
}
// @Test
public void testPartition() {
Partition p = new Partition();
p.setId(1);
List<Metapb.Shard> shards = new ArrayList<>();
shards.add(Metapb.Shard.newBuilder().build());
// p.setShardsList(shards);
Partition p2 = p.clone();
p.setId(2);
Assert.assertNotEquals(p2.getId(), p.getId());
}
// @Test
public void testUpdateShardsList() {
List<Metapb.Shard> curShards = new ArrayList<>();
curShards.add(Metapb.Shard.newBuilder().setStoreId(1001).setRole(Metapb.ShardRole.Follower)
.build());
curShards.add(Metapb.Shard.newBuilder().setStoreId(1002).setRole(Metapb.ShardRole.Leader)
.build());
curShards.add(Metapb.Shard.newBuilder().setStoreId(1003).setRole(Metapb.ShardRole.Follower)
.build());
List<Metapb.Shard> reqShards = new ArrayList<>();
reqShards.add(Metapb.Shard.newBuilder().setStoreId(1001).setRole(Metapb.ShardRole.Leader)
.build());
reqShards.add(Metapb.Shard.newBuilder().setStoreId(1002).setRole(Metapb.ShardRole.Leader)
.build());
reqShards.add(Metapb.Shard.newBuilder().setStoreId(1004).setRole(Metapb.ShardRole.Leader)
.build());
long leaderStoreId = 0;
for (Metapb.Shard shard : curShards) {
if (shard.getRole() == Metapb.ShardRole.Leader) {
leaderStoreId = shard.getStoreId();
break;
}
}
// remove
List<Metapb.Shard> shards = curShards.stream().filter(shard ->
reqShards.stream()
.map(Metapb.Shard::getStoreId)
.collect(
Collectors.toSet())
.contains(
shard.getStoreId()))
.collect(Collectors.toList());
// add
List<Metapb.Shard> addShards = reqShards.stream().filter(shard ->
!curShards.stream()
.map(Metapb.Shard::getStoreId)
.collect(
Collectors.toSet())
.contains(
shard.getStoreId()))
.collect(Collectors.toList());
shards.addAll(addShards);
// change leader
for (Metapb.Shard shard : shards) {
if (shard.getStoreId() == leaderStoreId) {
shard.toBuilder().setRole(Metapb.ShardRole.Leader).build();
} else {
shard.toBuilder().setRole(Metapb.ShardRole.Follower).build();
}
}
}
// @Test
public void testPriority() {
List<String> oldPeers = new ArrayList<>();
oldPeers.add("127.0.0.1:8001::100");
oldPeers.add("127.0.0.1:8002::75");
oldPeers.add("127.0.0.1:8003::50");
List<String> peers = new ArrayList<>();
peers.add("127.0.0.1:8001");
peers.add("127.0.0.1:8003::60");
peers.add("127.0.0.1:8004");
List<String> priPeers = new ArrayList<>();
for (String peer : peers) {
if (peer.contains("::")) {
priPeers.add(peer);
System.out.println(peer);
} else {
boolean find = false;
for (String oldPeer : oldPeers) {
if (oldPeer.contains(peer + "::")) {
find = true;
priPeers.add(oldPeer);
System.out.println(oldPeer);
break;
}
}
if (!find) {
priPeers.add(peer);
System.out.println(peer);
}
}
}
}
// @Test
public void testRpcCall() throws ExecutionException, InterruptedException {
HgCmdClient client = new HgCmdClient();
client.init(new RpcOptions(), null);
ArrayList<Partition> ps = new ArrayList<>();
Metapb.Partition p = Metapb.Partition.newBuilder()
.setGraphName("OK")
.setId(1)
// .addShards(Metapb.Shard.newBuilder().setStoreId
// (1).build())
.build();
ps.add(new Partition(p));
client.createRaftNode("127.0.0.1:8511", ps, status -> {
System.out.println(status);
}).get();
}
}

101
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/UnitTestBase.java Normal file
View File

@ -0,0 +1,101 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBOptions;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.store.business.BusinessHandler;
import org.apache.hugegraph.store.business.BusinessHandlerImpl;
import org.apache.hugegraph.store.meta.PartitionManager;
import org.apache.hugegraph.store.options.HgStoreEngineOptions;
import org.apache.hugegraph.store.options.RaftRocksdbOptions;
import org.apache.hugegraph.store.pd.FakePdServiceProvider;
import org.apache.hugegraph.store.pd.PdProvider;
public class UnitTestBase {
public final static RocksDBFactory factory = RocksDBFactory.getInstance();
public final static String DEFAULT_TEST_TABLE = "unknown";
private String dbPath;
private BusinessHandler handler;
public static boolean deleteDir(File dir) {
if (dir.isDirectory()) {
for (File file : dir.listFiles()) {
deleteDir(file);
}
}
return dir.delete();
}
public void initDB(String dbPath) {
this.dbPath = dbPath;
UnitTestBase.deleteDir(new File(dbPath));
Map<String, Object> configMap = new HashMap<>();
configMap.put("rocksdb.write_buffer_size", "1048576");
configMap.put("rocksdb.bloom_filter_bits_per_key", "10");
RaftRocksdbOptions.initRocksdbGlobalConfig(configMap);
BusinessHandlerImpl.initRocksdb(configMap, null);
}
protected BusinessHandler getBusinessHandler() {
if (handler == null) {
synchronized (this) {
if (handler == null) {
int partitionCount = 2;
HgStoreEngineOptions options = new HgStoreEngineOptions() {{
setDataPath(dbPath);
setFakePdOptions(new HgStoreEngineOptions.FakePdOptions() {{
setPartitionCount(partitionCount);
setPeersList("127.0.0.1");
setStoreList("127.0.0.1");
}});
}};
PdProvider pdProvider = new FakePdServiceProvider(options.getFakePdOptions());
PartitionManager partitionManager = new PartitionManager(pdProvider, options);
BusinessHandler handler = new BusinessHandlerImpl(partitionManager);
}
}
}
return handler;
}
public RocksDBSession getDBSession(String dbName) {
RocksDBSession session = factory.queryGraphDB(dbName);
if (session == null) {
session = factory.createGraphDB(dbPath, dbName);
}
return session;
}
public void close() {
handler.closeAll();
}
}

67
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/ChangeShardNumTest.java Normal file
View File

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.util.HgStoreTestUtil;
import org.junit.Assert;
import org.junit.Test;
/**
* 测试修改副本数
*/
public class ChangeShardNumTest extends HgStoreClientBase {
@Test
public void test3To1() throws PDException {
int number = 10000;
HgStoreSession session = storeClient.openSession(GRAPH_NAME);
HgStoreTestUtil.batchPut(session, TABLE_NAME, "testKey", number);
try (HgKvIterator<HgKvEntry> iterators = session.scanIterator(TABLE_NAME)) {
// Assert.assertEquals(number, HgStoreTestUtil.amountOf(iterators));
}
Metapb.PDConfig pdConfig = pdClient.getPDConfig();
pdConfig = pdConfig.toBuilder().setShardCount(1).build();
pdClient.setPDConfig(pdConfig);
pdClient.balancePartition();
}
// @Test
public void test1To3() throws PDException {
int number = 10000;
HgStoreSession session = storeClient.openSession(GRAPH_NAME);
HgStoreTestUtil.batchPut(session, TABLE_NAME, "testKey", number);
try (HgKvIterator<HgKvEntry> iterators = session.scanIterator(TABLE_NAME)) {
Assert.assertEquals(number, HgStoreTestUtil.amountOf(iterators));
}
Metapb.PDConfig pdConfig = pdClient.getPDConfig();
pdConfig = pdConfig.toBuilder().setShardCount(3).build();
pdClient.setPDConfig(pdConfig);
pdClient.balancePartition();
}
}

34
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/ClientTest.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client;
import org.junit.Assert;
import org.junit.Test;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ClientTest {
@Test
public void testDemo() {
String s = "i am client";
log.info("UT:{}", s);
Assert.assertTrue(s.startsWith("i"));
}
}

167
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgAssertTest.java Normal file
View File

@ -0,0 +1,167 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.List;
import java.util.Map;
import org.apache.hugegraph.store.client.util.HgAssert;
import org.junit.Test;
public class HgAssertTest {
@Test
public void testIsTrue1() {
try {
HgAssert.isTrue(false, "message");
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
}
}
@Test
public void testIsTrue2() {
try {
HgAssert.isTrue(false, () -> "message");
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
}
}
@Test
public void testIsFalse1() {
try {
HgAssert.isFalse(true, "message");
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
}
}
@Test
public void testIsFalse2() {
try {
HgAssert.isFalse(true, () -> "message");
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
}
}
@Test
public void testIsArgumentValid1() {
// Setup
// Run the test
HgAssert.isArgumentValid("content".getBytes(), "parameter");
// Verify the results
}
@Test
public void testIsArgumentValid2() {
// Setup
// Run the test
HgAssert.isArgumentValid("str", "parameter");
// Verify the results
}
@Test
public void testIsArgumentValid3() {
// Setup
// Run the test
HgAssert.isArgumentValid(List.of("value"), "parameter");
// Verify the results
}
@Test
public void testIsArgumentNotNull() {
// Setup
// Run the test
HgAssert.isArgumentNotNull("obj", "parameter");
// Verify the results
}
@Test
public void testIstValid() {
// Setup
// Run the test
HgAssert.istValid("content".getBytes(), "message");
// Verify the results
}
@Test
public void testIsValid() {
// Setup
// Run the test
HgAssert.isValid("str", "message");
// Verify the results
}
@Test
public void testIsNotNull() {
// Setup
// Run the test
HgAssert.isNotNull("obj", "message");
// Verify the results
}
@Test
public void testIsContains1() {
assertTrue(HgAssert.isContains(new Object[]{"obj"}, "obj"));
}
@Test
public void testIsInvalid1() {
assertFalse(HgAssert.isInvalid("strs"));
}
@Test
public void testIsInvalid2() {
assertFalse(HgAssert.isInvalid("content".getBytes()));
}
@Test
public void testIsInvalid3() {
// Setup
final Map<?, ?> map = Map.ofEntries(Map.entry("value", "value"));
// Verify the results
assertFalse(HgAssert.isInvalid(map));
}
@Test
public void testIsInvalid4() {
assertFalse(HgAssert.isInvalid(List.of("value")));
}
@Test
public void testIsContains2() {
assertTrue(HgAssert.isContains(List.of("item"), "item"));
}
@Test
public void testIsNull() {
assertFalse(HgAssert.isNull("objs"));
}
}

190
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgKvStoreTest.java Normal file
View File

@ -0,0 +1,190 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client;
import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toBytes;
import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toOwnerKey;
import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toStr;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgStoreSession;
import org.junit.Assert;
import org.junit.Test;
public class HgKvStoreTest extends HgStoreClientBase {
@Test
public void truncateTest() {
System.out.println("--- test truncateTest ---");
HgStoreSession graph0 = storeClient.openSession("hugegraph0");
HgStoreSession graph1 = storeClient.openSession("hugegraph1");
graph0.truncate();
graph1.truncate();
for (int i = 0; i < 3; i++) {
HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i);
byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i);
graph0.put(TABLE_NAME, key, value0);
byte[] value1 = toBytes("g1 owner-" + i + ";ownerKey-" + i);
graph1.put(TABLE_NAME, key, value1);
}
HgKvIterator<HgKvEntry> iterator = graph0.scanIterator(TABLE_NAME);
Assert.assertTrue(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
// log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value()));
Assert.assertEquals(0, toStr(entry.value()).indexOf("g0"));
}
iterator = graph1.scanIterator(TABLE_NAME);
Assert.assertTrue(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
// log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value()));
Assert.assertEquals(0, toStr(entry.value()).indexOf("g1"));
}
graph0.truncate();
iterator = graph0.scanIterator(TABLE_NAME);
Assert.assertFalse(iterator.hasNext());
iterator = graph1.scanIterator(TABLE_NAME);
Assert.assertTrue(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
// System.out.println("key:" + toStr(entry.key()) + " value:" + toStr(entry.value()));
Assert.assertEquals(0, toStr(entry.value()).indexOf("g1"));
}
}
@Test
public void deleteTableTest() {
System.out.println("--- test deleteTableTest ---");
HgStoreSession graph0 = storeClient.openSession("hugegraph0");
HgStoreSession graph1 = storeClient.openSession("hugegraph1");
graph0.truncate();
graph1.truncate();
for (int i = 0; i < 2; i++) {
HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i);
byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i);
graph0.put(TABLE_NAME, key, value0);
byte[] value1 = toBytes("g1 owner-" + i + ";ownerKey-" + i);
graph1.put(TABLE_NAME, key, value1);
}
HgKvIterator<HgKvEntry> iterator = graph0.scanIterator(TABLE_NAME);
Assert.assertTrue(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
Assert.assertEquals(0, toStr(entry.value()).indexOf("g0"));
}
iterator = graph1.scanIterator(TABLE_NAME);
Assert.assertTrue(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
Assert.assertEquals(0, toStr(entry.value()).indexOf("g1"));
}
graph0.deleteTable(TABLE_NAME);
Assert.assertTrue(graph0.existsTable(TABLE_NAME));
iterator = graph0.scanIterator(TABLE_NAME);
Assert.assertFalse(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
}
iterator = graph1.scanIterator(TABLE_NAME);
Assert.assertTrue(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
Assert.assertEquals(0, toStr(entry.value()).indexOf("g1"));
}
}
@Test
public void dropTableTest() {
System.out.println("--- test dropTableTest ---");
HgStoreSession graph0 = storeClient.openSession("hugegraph0");
HgStoreSession graph1 = storeClient.openSession("hugegraph1");
graph0.truncate();
graph1.truncate();
for (int i = 0; i < 2; i++) {
HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i);
byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i);
graph0.put(TABLE_NAME, key, value0);
byte[] value1 = toBytes("g1 owner-" + i + ";ownerKey-" + i);
graph1.put(TABLE_NAME, key, value1);
}
HgKvIterator<HgKvEntry> iterator = graph0.scanIterator(TABLE_NAME);
Assert.assertTrue(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
Assert.assertEquals(0, toStr(entry.value()).indexOf("g0"));
}
iterator = graph1.scanIterator(TABLE_NAME);
Assert.assertTrue(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
Assert.assertEquals(0, toStr(entry.value()).indexOf("g1"));
}
graph0.dropTable(TABLE_NAME);
Assert.assertTrue(graph0.existsTable(TABLE_NAME));
iterator = graph0.scanIterator(TABLE_NAME);
Assert.assertFalse(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
}
iterator = graph1.scanIterator(TABLE_NAME);
Assert.assertTrue(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
Assert.assertEquals(0, toStr(entry.value()).indexOf("g1"));
}
}
@Test
public void moveDataPathTest() {
System.out.println("--- test moveDataPathTest ---");
HgStoreSession graph0 = storeClient.openSession("hugegraph0");
graph0.truncate();
for (int i = 0; i < 2; i++) {
HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i);
byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i);
graph0.put(TABLE_NAME, key, value0);
}
HgKvIterator<HgKvEntry> iterator = graph0.scanIterator(TABLE_NAME);
Assert.assertTrue(iterator.hasNext());
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
Assert.assertEquals(0, toStr(entry.value()).indexOf("g0"));
}
}
}

78
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgPairTest.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import org.apache.hugegraph.store.term.Bits;
import org.apache.hugegraph.store.term.HgPair;
import org.apache.hugegraph.store.term.HgTriple;
import org.junit.Before;
import org.junit.Test;
public class HgPairTest {
private HgPair<String, String> pair;
private HgTriple<String, String, String> triple;
@Before
public void setUp() {
pair = new HgPair<>("key", "value");
triple = new HgTriple<>("x", "y", "z");
}
@Test
public void testPair() {
int hashCode = pair.hashCode();
pair.toString();
pair.setKey("key1");
pair.setValue("value1");
pair.getKey();
pair.getValue();
assertEquals(new HgPair<>("key1", "value1"), pair);
var pair2 = new HgPair<>();
pair2.setKey("key1");
pair2.hashCode();
assertNotEquals(pair2, pair);
triple.getZ();
triple.getX();
triple.getY();
triple.toString();
triple.hashCode();
triple.hashCode();
assertEquals(triple, new HgTriple<>("x", "y", "z"));
assertNotEquals(pair2, triple);
}
@Test
public void testBits() {
byte[] buf = new byte[4];
Bits.putInt(buf, 0, 3);
int i = Bits.getInt(buf, 0);
assertEquals(3, i);
buf = new byte[2];
Bits.putShort(buf, 0, 2);
int s = Bits.getShort(buf, 0);
assertEquals(2, s);
buf = new byte[4];
Bits.put(buf, 0, new byte[]{0, 0, 0, 66});
int toInt = Bits.toInt(buf);
assertEquals(66, toInt);
}
}

685
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgSessionManagerRaftPDTest.java Normal file
View File

@ -0,0 +1,685 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client;
import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES;
import static org.apache.hugegraph.store.util.HgStoreTestUtil.batchPut;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgKvStore;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.client.grpc.KvCloseableIterator;
import org.apache.hugegraph.store.client.util.HgStoreClientConfig;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.client.util.MetricX;
import org.apache.hugegraph.store.grpc.common.ScanOrderType;
import org.apache.hugegraph.store.util.HgStoreTestUtil;
import org.junit.Assert;
import org.junit.Test;
public class HgSessionManagerRaftPDTest extends HgStoreClientBase {
public static final String GRAPH_NAME_X = "default/hugegraph/x";
public static final String GRAPH_NAME_Y = "default/hugegraph/y";
public static final String GRAPH_NAME_Z = "default/hugegraph/z";
public static final String TABLE_NAME_1 = "table1";
public static final String TABLE_NAME_2 = "table2";
public static final String TABLE_NAME_3 = "table3";
private HgStoreSession getStoreSession() {
return storeClient.openSession(HgStoreTestUtil.GRAPH_NAME);
}
private HgStoreSession getStoreSession(String graph) {
return storeClient.openSession(graph);
}
@Test
public void putGet() {
System.out.println("--- test put & get ---");
HgStoreSession session = getStoreSession(HgStoreTestUtil.GRAPH_NAME);
HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY");
byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE");
long stime = System.currentTimeMillis();
batchPut(session, TABLE_NAME, "BATCH-PUT-TEST", 30000);
System.out.println("Time is " + (System.currentTimeMillis() - stime));
}
public void putGet2() {
System.out.println("--- test put & get ---");
HgStoreSession session = getStoreSession("testGraph");
HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY");
byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE");
long stime = System.currentTimeMillis();
HgStoreTestUtil.batchPut(session, "testTable", "BATCH-PUT-TEST", 30000);
System.out.println("Time is " + (System.currentTimeMillis() - stime));
}
@Test
public void scanPartition() {
HgStoreTestUtil.println("--- test scanPartition ---");
HgStoreSession session = getStoreSession();
Iterator iterator =
session.scanIterator(TABLE_NAME, 0, 65535, HgKvStore.SCAN_HASHCODE, EMPTY_BYTES);
System.out.println(HgStoreTestUtil.amountOf(iterator));
}
@Test
public void check() {
System.out.println("--- test check ---");
HgStoreSession session = getStoreSession();
HgKvIterator<HgKvEntry> iterator = session.scanIterator(TABLE_NAME);
int count = 0;
while (iterator.hasNext()) {
iterator.next();
count++;
}
System.out.println(count);
}
@Test
public void putGetUnique() {
System.out.println("--- test put & get ---");
HgStoreSession session = getStoreSession();
// add timestamp into key to avoid key duplication
Date date = new Date();
SimpleDateFormat formatter = new SimpleDateFormat("HHmmss");
String timestamp = formatter.format(date);
HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY-" + timestamp);
byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE");
Assert.assertTrue(session.put(TABLE_NAME, key, value));
Assert.assertEquals(
HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get(TABLE_NAME, key)));
}
@Test
public void testBatchPutExt() throws IOException {
System.out.println("--- test batchPut ---");
HgStoreSession session = getStoreSession();
session.truncate();
String keyPrefix = "BATCH-GET-UNIT";
Map<HgOwnerKey, byte[]> map = HgStoreTestUtil.batchPut(session, keyPrefix, 10);
File outputFile = new File("tmp/batch_put_list");
if (outputFile.exists()) {
FileUtils.forceDelete(outputFile);
}
FileUtils.forceMkdir(new File("tmp/"));
ObjectOutputStream oo = new ObjectOutputStream(new FileOutputStream(outputFile));
oo.writeObject(map);
oo.close();
System.out.printf("%d entries have be put into graph %s\n", map.size(),
HgStoreTestUtil.GRAPH_NAME);
int count = 0;
HgKvIterator<HgKvEntry> iterator = null;
iterator = session.scanIterator(TABLE_NAME);
while (iterator.hasNext()) {
iterator.next();
count++;
}
Assert.assertEquals(10, count);
}
// @Test
public void testBatchGetExt() throws IOException, ClassNotFoundException {
File outputFile = new File("tmp/batch_put_list");
ObjectInputStream ois = new ObjectInputStream(new FileInputStream(outputFile));
Map<HgOwnerKey, byte[]> map = (Map<HgOwnerKey, byte[]>) ois.readObject();
ois.close();
System.out.printf("%d entries get from %s\n", map.size(), outputFile.getPath());
HgStoreSession session = getStoreSession();
List<HgOwnerKey> keyList =
map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList());
List<HgKvEntry> resList = session.batchGetOwner(TABLE_NAME, keyList);
Assert.assertTrue(
(resList.stream().map(e -> map.containsKey(HgStoreTestUtil.toOwnerKey(e.key())))
.allMatch(Boolean::booleanValue)));
}
@Test
public void testBatchPutUniqueExt() throws IOException {
System.out.println("--- test batchPut ---");
HgStoreSession session = getStoreSession();
// add timestamp into key to avoid key duplication
Date date = new Date();
SimpleDateFormat formatter = new SimpleDateFormat("HHmmss");
String timestamp = formatter.format(date);
String keyPrefix = "BATCH-GET-UNIT-" + timestamp;
Map<HgOwnerKey, byte[]> map = HgStoreTestUtil.batchPut(session, keyPrefix, 30);
File outputFile = new File("tmp/batch_put_list");
if (outputFile.exists()) {
FileUtils.forceDelete(outputFile);
}
FileUtils.forceMkdir(new File("tmp/"));
ObjectOutputStream oo = new ObjectOutputStream(new FileOutputStream(outputFile));
oo.writeObject(map);
oo.close();
System.out.printf("%d entries have be put into graph %s\n", map.size(),
HgStoreTestUtil.GRAPH_NAME);
}
@Test
public void testBatchPutMultiGraph() throws IOException {
System.out.println("--- test testBatchPutMultiGraph ---");
Date date = new Date();
SimpleDateFormat formatter = new SimpleDateFormat("HHmmss");
String timestamp = formatter.format(date);
HgStoreSession session1 = getStoreSession(GRAPH_NAME_X);
HgStoreSession session2 = getStoreSession(GRAPH_NAME_Y);
HgStoreSession session3 = getStoreSession(GRAPH_NAME_Z);
String keyPrefix1 = "BATCH-PUT-UNIT-X-" + timestamp;
String keyPrefix2 = "BATCH-PUT-UNIT-Y-" + timestamp;
String keyPrefix3 = "BATCH-PUT-UNIT-Z-" + timestamp;
HgStoreTestUtil.batchPut(session1, TABLE_NAME_1, keyPrefix1, 1000);
HgStoreTestUtil.batchPut(session1, TABLE_NAME_2, keyPrefix1, 1000);
HgStoreTestUtil.batchPut(session1, TABLE_NAME_3, keyPrefix1, 1000);
HgStoreTestUtil.batchPut(session2, TABLE_NAME_1, keyPrefix2, 1000);
HgStoreTestUtil.batchPut(session2, TABLE_NAME_2, keyPrefix2, 1000);
HgStoreTestUtil.batchPut(session2, TABLE_NAME_3, keyPrefix2, 1000);
HgStoreTestUtil.batchPut(session3, TABLE_NAME_1, keyPrefix3, 1000);
HgStoreTestUtil.batchPut(session3, TABLE_NAME_2, keyPrefix3, 1000);
HgStoreTestUtil.batchPut(session3, TABLE_NAME_3, keyPrefix3, 1000);
}
// @Test
// CAUTION: ONLY FOR LONG！
// 注意：目前只可以对 long 类型 value 进行 Merge 操作。
public void merge() {
System.out.println("--- test merge (1+1=2)---");
HgStoreSession session = getStoreSession();
String mergeKey = "merge-key";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey);
byte[] value = HgStoreTestUtil.toBytes(1L);
System.out.println("- put " + mergeKey + ":1 -");
session.put(TABLE_NAME, key, value);
System.out.println("- merge " + mergeKey + ":1 -");
session.merge(TABLE_NAME, key, value);
long res = HgStoreTestUtil.toLong(session.get(TABLE_NAME, key));
System.out.printf("after merge " + mergeKey + "=%s%n", res);
Assert.assertEquals(2L, res);
String putStr = "19";
session.put(TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr));
byte[] b1 = session.get(TABLE_NAME, key);
Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1));
}
@Test
public void delete() {
System.out.println("--- test delete ---");
HgStoreSession session = getStoreSession();
String delKey = "del-key";
String delValue = "del-value";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey);
byte[] value = HgStoreTestUtil.toBytes(delValue);
HgStoreTestUtil.println("- put " + delKey + ": " + delValue);
session.put(TABLE_NAME, key, value);
HgStoreTestUtil.println("- delete " + delKey);
session.delete(TABLE_NAME, key);
value = session.get(TABLE_NAME, key);
HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value));
Assert.assertEquals(EMPTY_BYTES, value);
}
@Test
public void deleteSingle() {
System.out.println("--- test deleteSingle ---");
HgStoreSession session = getStoreSession();
String delKey = "del-single-key";
String delValue = "del-single-value";
HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey);
byte[] value = HgStoreTestUtil.toBytes(delValue);
session.put(TABLE_NAME, key, value);
value = session.get(TABLE_NAME, key);
Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value));
HgStoreTestUtil.println("- delete-single : [" + delKey + "]");
session.deleteSingle(TABLE_NAME, key);
value = session.get(TABLE_NAME, key);
HgStoreTestUtil.println(
"- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value));
Assert.assertEquals("", HgStoreTestUtil.toStr(value));
}
@Test
public void deleteRange() {
HgStoreTestUtil.println("--- test deleteRange ---");
HgStoreSession session = getStoreSession();
String rangePrefix = "DEL-RANGE-KEY";
String owner = "batch-delete-owner";
Map<HgOwnerKey, byte[]> map =
HgStoreTestUtil.batchPut(
session,
TABLE_NAME,
rangePrefix,
10,
key -> {
return HgStoreTestUtil.toOwnerKey(owner, key);
});
HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00");
HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05");
Assert.assertTrue(session.deleteRange(TABLE_NAME, startKey, endKey));
HgStoreTestUtil.println(
"- after delete range from [" + HgStoreTestUtil.toStr(startKey.getKey()) +
"] to [" +
HgStoreTestUtil.toStr(endKey.getKey()) + "]");
for (int i = 0; i < 10; i++) {
HgOwnerKey key =
HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" +
HgStoreTestUtil.padLeftZeros(
String.valueOf(i), 2));
String value = HgStoreTestUtil.toStr(session.get(TABLE_NAME, key));
// TODO: [start,end)?
if (i < 5) {
Assert.assertEquals("", value);
} else {
// println(value);
Assert.assertNotEquals("", value);
}
}
}
@Test
public void deletePrefix() {
System.out.println("--- test deletePrefix ---");
HgStoreSession session = getStoreSession();
String prefixStr = "DEL-PREFIX-KEY";
String owner = "batch-delete-owner";
Map<HgOwnerKey, byte[]> map =
HgStoreTestUtil.batchPut(
session,
TABLE_NAME,
prefixStr,
10,
key -> {
return HgStoreTestUtil.toOwnerKey(owner, key);
});
// printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()));
HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr);
Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator(TABLE_NAME, prefix)));
session.deletePrefix(TABLE_NAME, prefix);
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(TABLE_NAME, prefix)));
HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]");
for (int i = 0; i < 10; i++) {
HgOwnerKey key =
HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2));
String value = HgStoreTestUtil.toStr(session.get(TABLE_NAME, key));
Assert.assertEquals("", value);
}
}
// @Test
// TODO: this test's result is unstable
public void scanIterator() {
HgStoreTestUtil.println("--- test scanIterator ---");
String tableName = TABLE_NAME;
String keyName = "SCAN-ITER";
HgStoreSession session = getStoreSession();
HgStoreTestUtil.batchPut(session, tableName, keyName, 10000);
int count = 0;
int limit = 0;
int max = 99999;
HgKvIterator<HgKvEntry> iterator = null;
HgStoreTestUtil.println("-- test 0 element --");
iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"),
HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0);
Assert.assertFalse(iterator.hasNext());
try {
iterator.next();
Assert.fail();
} catch (Throwable t) {
HgStoreTestUtil.println("-- test NoSuchElementException --");
Assert.assertTrue(t instanceof NoSuchElementException);
}
for (int i = 1; i <= 10; i++) {
limit = i;
iterator =
session.scanIterator(
tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
}
Assert.assertEquals(limit, count);
}
for (int i = 1; i <= 10; i++) {
limit = i;
iterator =
session.scanIterator(
tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-00001"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-00005"), limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
}
if (i <= 5) {
Assert.assertEquals(limit, count);
} else {
Assert.assertEquals(5, count);
}
}
limit = 0;
iterator =
session.scanIterator(
tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"),
HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count >= max) {
break;
}
}
Assert.assertEquals(10000, count);
iterator = session.scanIterator(tableName);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry ignore = iterator.next();
if (count >= max) {
break;
}
}
Assert.assertEquals(10000, count);
iterator =
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-01"));
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count >= max) {
break;
}
}
Assert.assertEquals(1000, count);
}
@Test
public void paging() {
HgStoreTestUtil.println("--- test scanIterator_range ---");
String graph = "UNIT/paging";
String tableName = TABLE_NAME;
String keyName = "SCAN-PAGING";
int keyAmt = 100;
HgStoreSession session = getStoreSession(graph);
HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt);
int count = 0;
int limit = 0;
int max = 99999;
HgKvIterator<HgKvEntry> iterator = null;
count = 0;
iterator =
session.scanIterator(
tableName,
HgStoreTestUtil.toAllPartitionKey(keyName + "-000"),
HgStoreClientConst.EMPTY_OWNER_KEY,
0,
HgKvStore.SCAN_ANY,
EMPTY_BYTES);
List<byte[]> positionList = new LinkedList<>();
while (iterator.hasNext()) {
HgKvEntry entry = iterator.next();
positionList.add(iterator.position());
if (count >= max) {
break;
}
}
iterator = session.scanIterator(tableName, 100);
byte[] position = positionList.get(50);
HgStoreTestUtil.println("seek: " + Arrays.toString(position));
iterator.seek(position);
HgStoreTestUtil.println(iterator);
// Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator));
}
@Test
public void truncate() {
HgStoreTestUtil.println("--- test truncate ---");
String graph = "graph_truncate";
String tableName = TABLE_NAME;
String keyName = "KEY_TRUNCATE";
HgStoreSession session = getStoreSession(graph);
HgStoreTestUtil.batchPut(session, tableName, keyName, 100);
Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName)));
session.truncate();
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName)));
}
// @Test
public void scanIteratorHuge() {
/*************** test no limit, with 10 millions **************/
String tableName = TABLE_NAME;
String keyName = "SCAN-HUGE";
int amount = 10_000_000;
HgStoreSession session = getStoreSession();
if (HgStoreTestUtil.amountOf(
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) <
10) {
HgStoreTestUtil.batchPut(session, tableName, keyName, amount);
}
int count = 0;
HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName);
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
}
Assert.assertEquals(amount, count);
}
@Test
public void scanTable() {
HgStoreSession session = getStoreSession("DEFAULT/hg1/g");
HgStoreTestUtil.println(session.scanIterator("g+v", 10));
}
@Test
public void testDelGraph() {
HgStoreSession session = getStoreSession();
session.deleteGraph(HgStoreTestUtil.GRAPH_NAME);
}
// TODO: need figure out
// @Test
public void benchmarkScanBatch() {
HgStoreTestUtil.println("--- Benchmark scanBatch ---");
String tableName = TABLE_NAME;
String keyName = "SCAN-BATCH";
int keyAmt = 30001;
HgStoreSession session = getStoreSession();
if (HgStoreTestUtil.amountOf(
session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) <
1) {
HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt);
}
HgStoreTestUtil.println("-- Starting scan --");
List<HgKvIterator<HgKvEntry>> iterators = session.scanBatch(HgScanQuery.tableOf(tableName));
Assert.assertEquals(keyAmt, HgStoreTestUtil.amountIn(iterators));
}
@Test
public void benchmarkScanBatch2() throws IOException {
HgStoreTestUtil.println("--- Benchmark scanBatch2 ---");
String tableName = TABLE_NAME;
HgStoreSession session = getStoreSession();
String keyName = "SCAN-BATCH";
int keyAmt = 300;
Map<HgOwnerKey, byte[]> data =
HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt);
HgStoreTestUtil.println("-- Starting scan --");
MetricX metrics = MetricX.ofStart();
// HgStoreTestUtil.println(session.scanIterator(tableName));
long t = System.currentTimeMillis();
int count = 0;
List<HgOwnerKey> keys = new ArrayList<>();
data.forEach((k, v) -> keys.add(k));
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterators =
session.scanBatch2(
HgScanQuery.prefixIteratorOf(tableName, keys.iterator(),
ScanOrderType.ORDER_NONE)
.builder()
.setScanType(0x40)
.build());
while (iterators.hasNext()) {
HgKvIterator<HgKvEntry> iterator = iterators.next();
count += HgStoreTestUtil.amountOf(iterator);
}
keys.clear();
iterators.close();
}
@Test
public void benchmarkScanBatchSkipDegree() throws IOException {
HgStoreTestUtil.println("--- Benchmark scanBatch2 1Owner---");
String tableName = TABLE_NAME;
HgStoreSession session = getStoreSession();
String keyName = "SCAN-BATCH";
int keyAmt = 300;
byte[] owner = "Owner".getBytes();
Map<HgOwnerKey, byte[]> data =
HgStoreTestUtil.batchPut(
session,
tableName,
keyName,
keyAmt,
key -> {
return HgStoreTestUtil.toOwnerKey(owner, key);
});
HgStoreTestUtil.println("-- Starting scan --");
MetricX metrics = MetricX.ofStart();
long t = System.currentTimeMillis();
int count = 0;
List<HgOwnerKey> keys = new ArrayList<>();
keys.add(HgStoreTestUtil.toOwnerKey(owner, keyName));
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterators =
session.scanBatch2(
HgScanQuery.prefixIteratorOf(tableName, keys.iterator(),
ScanOrderType.ORDER_NONE)
.builder()
.setScanType(0x40)
.setSkipDegree(1)
.build());
while (iterators.hasNext()) {
HgKvIterator<HgKvEntry> iterator = iterators.next();
count += HgStoreTestUtil.amountOf(iterator);
}
keys.clear();
iterators.close();
iterators = null;
HgStoreTestUtil.println("time is " + (System.currentTimeMillis() - t));
metrics.end();
HgStoreTestUtil.println("*************************************************");
HgStoreTestUtil.println("*********** Batch Scan Completed **************");
HgStoreTestUtil.println("Total : " + (metrics.past() / 1000) + " (sec)");
HgStoreTestUtil.println(" Sum : " + (MetricX.getIteratorWait() / 1000) + " (sec)");
HgStoreTestUtil.println(" Amt : " + MetricX.getIteratorCount() + " (scans).");
HgStoreTestUtil.println(" Avg : " + MetricX.getIteratorWaitAvg() + " (ms)");
HgStoreTestUtil.println(" Max : " + MetricX.getIteratorWaitMax() + " (ms)");
HgStoreTestUtil.println(" Fail : " + metrics.getFailureCount() + " (times)");
HgStoreTestUtil.println(
" Page : " + HgStoreClientConfig.of().getNetKvScannerPageSize() + " (KVs)");
HgStoreTestUtil.println(" size is " + count);
HgStoreTestUtil.println("*************************************************");
}
}

51
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreClientBase.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.client.PDConfig;
import org.apache.hugegraph.store.HgStoreClient;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.UnitTestBase;
import org.junit.After;
import org.junit.Before;
public class HgStoreClientBase {
protected static String GRAPH_NAME = "testGraphName";
protected static String TABLE_NAME = UnitTestBase.DEFAULT_TEST_TABLE;
private static final String PD_ADDRESS = "127.0.0.1:8686";
protected HgStoreClient storeClient;
protected PDClient pdClient;
@Before
public void setup() throws Exception {
storeClient = HgStoreClient.create(PDConfig.of(PD_ADDRESS)
.setEnableCache(true));
pdClient = storeClient.getPdClient();
HgStoreSession session = storeClient.openSession(TABLE_NAME);
session.dropTable(TABLE_NAME);
session.truncate();
}
@After
public void teardown() {
// pass
}
}

207
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreClientTest.java Normal file
View File

@ -0,0 +1,207 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client;
import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER;
import java.util.List;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgKvStore;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.util.HgStoreTestUtil;
import org.junit.Assert;
import org.junit.Test;
public class HgStoreClientTest extends HgStoreClientBase {
public static final byte[] EMPTY_BYTES = new byte[0];
@Test
public void testPutData() {
HgStoreSession session = storeClient.openSession(GRAPH_NAME);
long start = System.currentTimeMillis();
int loop = 100000;
session.truncate();
HgStoreTestUtil.batchPut(session, TABLE_NAME, "testKey", loop);
System.out.println("Time is " + (System.currentTimeMillis() - start));
try (HgKvIterator<HgKvEntry> iterator = session.scanIterator(TABLE_NAME)) {
Assert.assertEquals(loop, HgStoreTestUtil.amountOf(iterator));
}
}
@Test
public void testPutData2() {
String GRAPH_NAME = "testGRAPH_NAME2";
HgStoreSession session = storeClient.openSession(GRAPH_NAME);
long start = System.currentTimeMillis();
int loop = 100000;
session.truncate();
HgStoreTestUtil.batchPut(session, TABLE_NAME, "testKey", loop);
System.out.println("Time is " + (System.currentTimeMillis() - start));
try (HgKvIterator<HgKvEntry> iterator = session.scanIterator(TABLE_NAME)) {
Assert.assertEquals(loop, HgStoreTestUtil.amountOf(iterator));
}
}
@Test
public void testScan() throws PDException {
HgStoreSession session = storeClient.openSession(GRAPH_NAME);
HgStoreTestUtil.batchPut(session, TABLE_NAME, "testKey", 12);
int count = 0;
byte[] position = null;
try (HgKvIterator<HgKvEntry> iterator = session.scanIterator(TABLE_NAME)) {
while (iterator.hasNext()) {
iterator.next();
position = iterator.position();
dumpPosition(position);
if (++count > 5) {
break;
}
}
}
System.out.println("--------------------------------");
try (HgKvIterator<HgKvEntry> iterator = session.scanIterator(TABLE_NAME)) {
iterator.seek(position);
while (iterator.hasNext()) {
iterator.next();
dumpPosition(iterator.position());
}
}
System.out.println("--------------------------------");
byte[] start = new byte[]{0x0};
byte[] end = new byte[]{-1};
try (HgKvIterator<HgKvEntry> iterator = session.scanIterator(TABLE_NAME,
HgOwnerKey.of(
ALL_PARTITION_OWNER,
start),
HgOwnerKey.of(
ALL_PARTITION_OWNER,
end))) {
iterator.seek(position);
while (iterator.hasNext()) {
iterator.next();
dumpPosition(iterator.position());
}
}
}
public void dumpPosition(byte[] b) {
byte[] buf = new byte[Long.BYTES];
System.arraycopy(b, 0, buf, 0, Long.BYTES);
// long storeId = HgStoreTestUtil.toLong(buf);
buf = new byte[Integer.BYTES];
System.arraycopy(b, Long.BYTES, buf, 0, Integer.BYTES);
// int partId = HgStoreTestUtil.toInt(buf);
// String key = new String(b);
// System.out.println(" " + storeId + ", " + partId + ", " + key);
}
// @Test
// TODO: unstable
public void testDeleteData() {
HgStoreSession session = storeClient.openSession(GRAPH_NAME);
int ownerCode = 1;
HgStoreTestUtil.batchPut(session, TABLE_NAME, "T", 10, (key) -> {
return HgStoreTestUtil.toOwnerKey(ownerCode, key);
}
);
try (HgKvIterator<HgKvEntry> iterators = session.scanIterator(TABLE_NAME)) {
// while (iterators.hasNext()){
// System.out.println(new String(iterators.next().key()));
// }
Assert.assertEquals(10, HgStoreTestUtil.amountOf(iterators));
}
session.beginTx();
session.deletePrefix(TABLE_NAME, HgStoreTestUtil.toOwnerKey(ownerCode, "T"));
session.commit();
System.out.println("=================================");
try (HgKvIterator<HgKvEntry> iterators = session.scanIterator(TABLE_NAME)) {
Assert.assertEquals(0, HgStoreTestUtil.amountOf(iterators));
// while (iterators.hasNext()){
// System.out.println(new String(iterators.next().key()));
// }
}
}
// @Test
// TODO: unstable
public void testDropTable() throws PDException {
HgStoreSession session = storeClient.openSession(GRAPH_NAME);
String table1 = TABLE_NAME;
session.createTable(table1);
HgStoreTestUtil.batchPut(session, table1, "testKey", 1000);
try (HgKvIterator<HgKvEntry> iterators = session.scanIterator(table1)) {
Assert.assertEquals(1000, HgStoreTestUtil.amountOf(iterators));
}
session.dropTable(table1);
try (HgKvIterator<HgKvEntry> iterators = session.scanIterator(table1)) {
Assert.assertEquals(0, HgStoreTestUtil.amountOf(iterators));
}
deleteGraph(GRAPH_NAME);
}
public void deleteGraph(String GRAPH_NAME) throws PDException {
HgStoreSession session = storeClient.openSession(GRAPH_NAME);
session.deleteGraph(GRAPH_NAME);
pdClient.delGraph(GRAPH_NAME);
Metapb.Graph graph = null;
try {
graph = pdClient.getGraph(GRAPH_NAME);
} catch (PDException e) {
Assert.assertEquals(103, e.getErrorCode());
}
Assert.assertNull(graph);
}
@Test
public void testScanPartition() throws PDException {
// testPutData();
List<Metapb.Partition> partitions = pdClient.getPartitions(0, "DEFAULT/hugegraph/g");
HgStoreSession session = storeClient.openSession("DEFAULT/hugegraph/g");
for (Metapb.Partition partition : partitions) {
try (HgKvIterator<HgKvEntry> iterators = session.scanIterator("g+v",
(int) (partition.getStartKey()),
(int) (partition.getEndKey()),
HgKvStore.SCAN_HASHCODE,
EMPTY_BYTES)) {
System.out.println(
" " + partition.getId() + " " + HgStoreTestUtil.amountOf(iterators));
}
}
}
}

66
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeSessionTest.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.util.HgStoreTestUtil;
import org.junit.Assert;
/**
* created on 2021/10/12
*/
public class HgStoreNodeSessionTest {
private static final HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance();
private static HgStoreNode node;
// @BeforeClass
public static void init() {
node = nodeManager.addNode(HgStoreTestUtil.GRAPH_NAME,
nodeManager.getNodeBuilder().setAddress("localhost:9080")
.build());
}
private static HgStoreSession getStoreSession() {
return node.openSession(HgStoreTestUtil.GRAPH_NAME);
}
private HgStoreNode getOneNode() {
return node;
}
// @Test
public void truncate() {
HgStoreTestUtil.println("--- test truncate ---");
String tableName = "UNIT_TRUNCATE_1";
String keyName = "KEY_TRUNCATE";
HgStoreSession session = getStoreSession();
HgStoreTestUtil.batchPut(session, tableName, keyName, 100);
Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName)));
String tableName2 = "UNIT_TRUNCATE_2";
HgStoreTestUtil.batchPut(session, tableName2, keyName, 100);
Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2)));
session.truncate();
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName)));
Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2)));
}
}

59
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeStateTest.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hugegraph.store.util.HgStoreTestUtil;
import org.junit.Test;
public class HgStoreNodeStateTest {
private static final HgStoreNodeManager NODE_MANAGER = HgStoreNodeManager.getInstance();
static int nodeNumber = 0;
static {
registerNode(HgStoreTestUtil.GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9180");
registerNode(HgStoreTestUtil.GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9280");
registerNode(HgStoreTestUtil.GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9380");
}
private static void registerNode(String graphName, Long nodeId, String address) {
NODE_MANAGER.addNode(graphName,
NODE_MANAGER.getNodeBuilder().setNodeId(nodeId).setAddress(address)
.build());
}
@Test
public void isNodeHealthy() {
AtomicInteger count = new AtomicInteger(0);
for (int i = 0; i < 100; i++) {
NODE_MANAGER.getStoreNodes(HgStoreTestUtil.GRAPH_NAME)
.stream().map(
node -> {
System.out.println(node.getNodeId() + " " + node.getAddress()
+ "is healthy: " + node.isHealthy());
return node.isHealthy();
}
).count();
Thread.yield();
}
}
}

188
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeStreamTest.java Normal file
View File

@ -0,0 +1,188 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client;
import static org.apache.hugegraph.store.util.HgStoreTestUtil.GRAPH_NAME;
import static org.apache.hugegraph.store.util.HgStoreTestUtil.batchPut;
import static org.apache.hugegraph.store.util.HgStoreTestUtil.println;
import static org.apache.hugegraph.store.util.HgStoreTestUtil.toOwnerKey;
import java.util.NoSuchElementException;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.client.HgStoreNode;
import org.apache.hugegraph.store.client.HgStoreNodeManager;
import org.apache.hugegraph.store.util.HgStoreTestUtil;
import org.junit.Assert;
/**
* created on 2021/10/12
*/
public class HgStoreNodeStreamTest {
private static final HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance();
private static HgStoreNode node;
// @BeforeClass
public static void init() {
node = nodeManager.addNode(HgStoreTestUtil.GRAPH_NAME,
nodeManager.getNodeBuilder().setAddress("localhost:9080")
.build());
}
private static HgStoreSession getStoreSession() {
return node.openSession(HgStoreTestUtil.GRAPH_NAME);
}
private HgStoreNode getOneNode() {
return node;
}
// @Test
public void scanIterator() {
HgStoreTestUtil.println("--- test scanIterator ---");
String tableName = "UNIT_SCAN";
String keyName = "SCAN-ITER";
HgStoreSession session = getStoreSession();
HgStoreTestUtil.batchPut(session, tableName, keyName, 10000);
int count = 0;
int limit = 0;
int max = 99999;
HgKvIterator<HgKvEntry> iterator = null;
HgStoreTestUtil.println("-- test 0 element --");
iterator =
session.scanIterator(tableName, HgStoreTestUtil.toOwnerKey("__SCAN-001"), HgStoreTestUtil.toOwnerKey("__SCAN-100"),
0);
Assert.assertFalse(iterator.hasNext());
try {
iterator.next();
Assert.fail();
} catch (Throwable t) {
HgStoreTestUtil.println("-- test NoSuchElementException --");
Assert.assertTrue(t instanceof NoSuchElementException);
}
HgStoreTestUtil.println("-- test limit 1 to 10 --");
for (int i = 1; i <= 10; i++) {
HgStoreTestUtil.println("- limit " + i + " -");
limit = i;
iterator = session.scanIterator(tableName, HgStoreTestUtil.toOwnerKey(keyName + "-0"),
HgStoreTestUtil.toOwnerKey(keyName + "-1"), limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
HgStoreTestUtil.println(entry);
}
Assert.assertEquals(limit, count);
}
HgStoreTestUtil.println("-- test limit 1 to 10 not enough --");
for (int i = 1; i <= 10; i++) {
HgStoreTestUtil.println("- limit " + i + " -");
limit = i;
iterator = session.scanIterator(tableName,
HgStoreTestUtil.toOwnerKey(keyName + "-00001"),
HgStoreTestUtil.toOwnerKey(keyName + "-00005"), limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
HgStoreTestUtil.println(entry);
}
if (i <= 5) {
Assert.assertEquals(limit, count);
} else {
Assert.assertEquals(5, count);
}
}
HgStoreTestUtil.println("-- test limit 0 (no limit) --");
limit = 0;
iterator = session.scanIterator(tableName, HgStoreTestUtil.toOwnerKey(keyName + "-0"),
HgStoreTestUtil.toOwnerKey(keyName + "-1"), limit);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 1000 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(10000, count);
HgStoreTestUtil.println("-- test scan all --");
iterator = session.scanIterator(tableName);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 100 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(10000, count);
HgStoreTestUtil.println("-- test scan prefix --");
iterator = session.scanIterator(tableName, HgStoreTestUtil.toOwnerKey(keyName + "-01"));
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 100 == 0) {
HgStoreTestUtil.println(entry);
}
if (count >= max) break;
}
Assert.assertEquals(1000, count);
}
//// @Test
public void scanIteratorBenchmark() {
/*************** test no limit, with 10 millions **************/
String tableName = "UNIT_HUGE";
String keyName = "SCAN-HUGE";
HgStoreSession session = getStoreSession();
//batchPut(session,tableName, keyName, 10000000);
int count = 0;
int limit = 0;
HgKvIterator<HgKvEntry> iterator = null;
limit = 0;
iterator = session.scanIterator(tableName);
count = 0;
while (iterator.hasNext()) {
count++;
HgKvEntry entry = iterator.next();
if (count % 100000 == 0) {
HgStoreTestUtil.println(entry);
}
}
Assert.assertEquals(10000000, count);
}
}

182
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/graph/GraphStoreClientTest.java Normal file
View File

@ -0,0 +1,182 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.client.graph;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hugegraph.store.grpc.GraphStoreGrpc;
import org.apache.hugegraph.store.grpc.GraphStoreGrpc.GraphStoreStub;
import org.apache.hugegraph.store.grpc.Graphpb.ScanPartitionRequest;
import org.apache.hugegraph.store.grpc.Graphpb.ScanPartitionRequest.Reply;
import org.apache.hugegraph.store.grpc.Graphpb.ScanResponse;
import org.junit.Test;
import com.google.protobuf.util.JsonFormat;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class GraphStoreClientTest {
private final AtomicInteger sum = new AtomicInteger();
ConcurrentHashMap<Long, StreamObserver<ScanPartitionRequest>> observers =
new ConcurrentHashMap<>();
@Test
public void getData() {
long start = System.currentTimeMillis();
String[] addresses = new String[]{"10.14.139.71:8500",
"10.14.139.70:8500",
"10.14.139.69:8500"};
Arrays.stream(addresses).parallel().forEach(address -> {
int size = 72;
CountDownLatch latch = new CountDownLatch(size);
for (int i = 0; i < size; i++) {
int finalI = i;
new Thread(() -> getData(finalI, latch, address)).start();
}
try {
latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
});
long end = System.currentTimeMillis();
System.out.println(
"all rows are: " + sum.get() + ", end: " + (end - start));
}
public void getData(int pId, CountDownLatch latch, String address) {
try {
ScanPartitionRequest.Builder builder =
ScanPartitionRequest.newBuilder();
ScanPartitionRequest.Request.Builder srb =
ScanPartitionRequest.Request.newBuilder();
ArrayList<Long> properties = new ArrayList<>() {{
add(2L);
add(3L);
}};
ScanPartitionRequest.Request request =
srb.setGraphName("DEFAULT/hugegraph0/g")
.setScanType(
ScanPartitionRequest.ScanType.SCAN_VERTEX).setBoundary(0x10)
// .addAllProperties(properties)
// .setCondition("element.id().asString().equals
// ('1:marko1')")
.setPartitionId(pId).build();
ManagedChannel c =
ManagedChannelBuilder.forTarget(address)
.usePlaintext().build();
int maxSize = 1024 * 1024 * 1024;
GraphStoreStub stub;
stub = GraphStoreGrpc.newStub(c)
.withMaxInboundMessageSize(maxSize)
.withMaxOutboundMessageSize(maxSize);
AtomicInteger count = new AtomicInteger();
long start = System.currentTimeMillis();
long id = Thread.currentThread().getId();
StreamObserver<ScanResponse> ro =
new StreamObserver<ScanResponse>() {
@Override
public void onNext(ScanResponse value) {
try {
int edgeSize = value.getEdgeCount();
int vertexSize = value.getVertexCount();
if (request.getScanType().equals(
ScanPartitionRequest.ScanType.SCAN_VERTEX)) {
count.getAndAdd(vertexSize);
} else {
count.getAndAdd(edgeSize);
}
String print = JsonFormat.printer().print
(value);
System.out.println(print);
ScanPartitionRequest.Builder builder =
ScanPartitionRequest.newBuilder();
builder.setScanRequest(request);
Reply.Builder reply = Reply.newBuilder();
reply.setSeqNo(1);
builder.setReplyRequest(reply);
observers.get(id).onNext(builder.build());
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public void onError(Throwable t) {
System.out.println(t);
latch.countDown();
}
@Override
public void onCompleted() {
long time = System.currentTimeMillis() - start;
System.out.println(
"scan id :" + pId + ", complete: " +
count.get() + ",time: " + time);
sum.addAndGet(count.get());
latch.countDown();
}
};
StreamObserver<ScanPartitionRequest> observer =
stub.scanPartition(ro);
observers.put(id, observer);
builder.setScanRequest(request);
observer.onNext(builder.build());
} catch (Exception e) {
e.printStackTrace();
}
}
@Test
public void getDataSingle() {
CountDownLatch latch = new CountDownLatch(1);
new Thread(() -> getData(58, latch, "10.14.139.71:8500")).start();
try {
latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("all rows are: " + sum.get());
}
@Test
public void getNativeDataSingle() {
CountDownLatch latch = new CountDownLatch(1);
new Thread(() -> getData(0, latch, "127.0.0.1:8500")).start();
try {
latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("all rows are: " + sum.get());
}
}

1
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/BaseCommonTest.java
View File

@ -21,7 +21,6 @@ import org.junit.After;
import org.junit.BeforeClass;
public class BaseCommonTest {
@BeforeClass
public static void beforeClass() throws Exception {
}

1
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/BitsTest.java
View File

@ -22,7 +22,6 @@ import org.junit.Assert;
// import org.junit.Test;
public class BitsTest {
// @Test
public void test() {
for (int i = 0; i < Integer.MAX_VALUE; i = i + 10) {

1
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/ByteBufferAllocatorTest.java
View File

@ -27,7 +27,6 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ByteBufferAllocatorTest extends BaseCommonTest {
@Test
public void getAndReleaseTest() throws InterruptedException {
final CountDownLatch latch = new CountDownLatch(2);

1
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/CommonSuiteTest.java
View File

@ -30,5 +30,4 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class CommonSuiteTest {
}

47
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/BaseCoreTest.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.BeforeClass;
public class BaseCoreTest {
@BeforeClass
public static void setup() throws Exception {
}
public static void deleteDirectory(File dir) {
try {
FileUtils.deleteDirectory(dir);
} catch (IOException e) {
System.out.printf("Failed to start ....,%s%n", e.getMessage());
}
}
@After
public void teardown() throws Exception {
// pass
}
}

49
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core;
import lombok.extern.slf4j.Slf4j;
// TODO: uncomment it until all test can run free.
//@RunWith(Suite.class)
//@Suite.SuiteClasses({
// HgCmdClientTest.class,
// HgSnapshotHandlerTest.class,
// RaftUtilsTest.class,
// RaftOperationTest.class,
// UnsafeUtilTest.class,
// CopyOnWriteCacheTest.class,
// MetricServiceTest.class,
// TaskManagerTest.class,
// CleanTaskTest.class,
// MetadataKeyHelperTest.class,
// HgStoreEngineTest.class,
// PartitionEngineTest.class,
// PartitionManagerTest.class,
// PartitionMetaStoreWrapperTest.class,
// ZipUtilsTest.class,
// MiscUtilClassTest.class,
// PartitionInstructionProcessorTest.class,
// // 尽量放到最后
// HgBusinessImplTest.class
//})
@Slf4j
public class CoreSuiteTest {
}

34
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreTest.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core;
import org.junit.Assert;
import org.junit.Test;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class CoreTest {
@Test
public void testDemo() {
String s = "i am core";
log.info("UT:{}", s);
Assert.assertTrue(s.startsWith("i"));
}
}

178
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/HgBusinessImplTest.java Normal file
View File

@ -0,0 +1,178 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.UnitTestBase;
import org.apache.hugegraph.store.business.BusinessHandler;
import org.apache.hugegraph.util.Bytes;
import org.junit.Assert;
import org.junit.Test;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class HgBusinessImplTest extends StoreEngineTestBase {
public static final String TABLE_NAME = UnitTestBase.DEFAULT_TEST_TABLE;
public BusinessHandler getBusinessHandler() {
return getStoreEngine().getBusinessHandler();
}
@Test
public void testPut() throws InterruptedException {
String graph1 = "testPut-graph1";
String graph2 = "testPut-graph2";
createPartitionEngine(0, graph1);
createPartitionEngine(0, graph2);
BusinessHandler handler = getBusinessHandler();
handler.doPut(graph1, 0, TABLE_NAME, "key1".getBytes(), "value1".getBytes());
handler.doPut(graph1, 0xF, TABLE_NAME, "key2".getBytes(), "value1".getBytes());
handler.doPut(graph1, 0xFF, TABLE_NAME, "key3".getBytes(), "value1".getBytes());
handler.doPut(graph1, 0xFFF, TABLE_NAME, "key4".getBytes(), "value1".getBytes());
handler.doPut(graph2, 0, TABLE_NAME, "key21".getBytes(), "value1".getBytes());
handler.doPut(graph2, 0xF, TABLE_NAME, "key22".getBytes(), "value1".getBytes());
handler.doPut(graph2, 0xFF, TABLE_NAME, "key23".getBytes(), "value1".getBytes());
handler.doPut(graph2, 0xFFF, TABLE_NAME, "key24".getBytes(), "value1".getBytes());
System.out.println("--------------------dump all -------");
dump(handler, graph1, 0);
System.out.println("--------------------dump scan 0 0xff -------");
ScanIterator iterator = handler.scan(graph1, TABLE_NAME, 0, 0xff);
int count = 0;
while (iterator.hasNext()) {
RocksDBSession.BackendColumn entry = iterator.next();
System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name));
count++;
}
Assert.assertEquals(2, count);
System.out.println("--------------------dump scan prefix -------");
iterator = handler.scanPrefix(graph1, 0, TABLE_NAME, "key".getBytes());
count = 0;
while (iterator.hasNext()) {
RocksDBSession.BackendColumn entry = iterator.next();
System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name));
count++;
}
Assert.assertEquals(4, count);
System.out.println("--------------------dump scan range key1 key4 -------");
iterator = handler.scan(graph1, 0, TABLE_NAME, "key1".getBytes(), "key4".getBytes(),
ScanIterator.Trait.SCAN_LT_END);
count = 0;
while (iterator.hasNext()) {
RocksDBSession.BackendColumn entry = iterator.next();
System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name));
count++;
}
Assert.assertEquals(3, count);
boolean cleanResult = handler.cleanPartition(graph1, 0);
Assert.assertTrue(cleanResult);
Thread.sleep(2000);
handler.closeAll();
}
@Test
public void testLoadSnapshot() throws InterruptedException {
String graph1 = "LoadSnapshot-graph1";
createPartitionEngine(0, graph1);
BusinessHandler handler = getBusinessHandler();
for (int i = 0; i < 256; i++) {
handler.doPut(graph1, i, TABLE_NAME, ("key" + i).getBytes(), "value1".getBytes());
}
ScanIterator iterator = handler.scanAll(graph1, TABLE_NAME);
System.out.println(iterator.count());
String snapshotPath;
try (RocksDBSession session = handler.getSession(0)) {
snapshotPath = session.getDbPath();
}
handler.closeAll();
System.out.println("start loadSnapshot");
handler.loadSnapshot(snapshotPath, graph1, 0, 10);
iterator = handler.scanAll(graph1, TABLE_NAME);
Assert.assertEquals(255, iterator.count());
try (RocksDBSession session = handler.getSession(0)) {
System.out.println(session.getDbPath());
}
CountDownLatch latch = new CountDownLatch(1);
RocksDBFactory.getInstance()
.addRocksdbChangedListener(new RocksDBFactory.RocksdbChangedListener() {
@Override
public void onCompacted(String dbName) {
RocksDBFactory.RocksdbChangedListener.super.onCompacted(dbName);
}
@Override
public void onDBDeleteBegin(String dbName, String filePath) {
RocksDBFactory.RocksdbChangedListener.super.onDBDeleteBegin(dbName,
filePath);
}
@Override
public void onDBDeleted(String dbName, String filePath) {
latch.countDown();
}
@Override
public void onDBSessionReleased(RocksDBSession dbSession) {
RocksDBFactory.RocksdbChangedListener.super.onDBSessionReleased(
dbSession);
}
});
latch.await(1, TimeUnit.SECONDS);
}
public void dump(BusinessHandler handler, String graph, int partId) {
ScanIterator cfIterator = handler.scanRaw(graph, partId, 0);
while (cfIterator.hasNext()) {
try (ScanIterator iterator = cfIterator.next()) {
byte[] cfName = cfIterator.position();
System.out.println(graph + "-" + partId + "-" + new String(cfName) + "--------");
while (iterator.hasNext()) {
RocksDBSession.BackendColumn col = iterator.next();
System.out.println(new String(col.name) + " -- " + Bytes.toHex(col.name));
}
}
}
cfIterator.close();
}
}

359
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/HgCmdClientTest.java Normal file
View File

@ -0,0 +1,359 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core;
import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toIntBytes;
import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toOwnerKey;
import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toStr;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.client.PDConfig;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgStoreClient;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.UnitTestBase;
import org.apache.hugegraph.store.cmd.BatchPutRequest;
import org.apache.hugegraph.store.cmd.BatchPutResponse;
import org.apache.hugegraph.store.cmd.CleanDataRequest;
import org.apache.hugegraph.store.cmd.CleanDataResponse;
import org.apache.hugegraph.store.cmd.HgCmdClient;
import org.apache.hugegraph.store.meta.Store;
import org.apache.hugegraph.store.pd.DefaultPdProvider;
import org.apache.hugegraph.store.pd.PdProvider;
import org.junit.Assert;
import org.junit.Test;
import com.alipay.sofa.jraft.JRaftUtils;
import com.alipay.sofa.jraft.option.RpcOptions;
import com.alipay.sofa.jraft.util.Endpoint;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class HgCmdClientTest {
private static AtomicLong id;
private final String pdAddress = "127.0.0.1:8686";
private final String graphName = "hugegraph";
private final String tableName = UnitTestBase.DEFAULT_TEST_TABLE;
private PdProvider pdProvider;
private HgCmdClient hgCmdClient;
private HgStoreClient storeClient;
private PDClient pdClient;
public static String getMd5(String txt) {
String rs = "";
String[] hexDigits =
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"};
try {
MessageDigest messageDigest = MessageDigest.getInstance("MD5");
byte[] b = messageDigest.digest(txt.getBytes());
StringBuffer resultSb = new StringBuffer();
for (int i = 0; i < b.length; i++) {
int n = b[i];
if (n < 0) {
n = 256 + n;
}
int d1 = n / 16;
int d2 = n % 16;
resultSb.append(hexDigits[d1] + hexDigits[d2]);
}
rs = resultSb.toString();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
return rs;
}
public static Long getId() {
// 如果需要更长 或者更大冗余空间，只需要 time * 10^n 即可
// 当前可保证 1 毫秒 生成 10000 条不重复
Long time = Long.valueOf(new SimpleDateFormat("HHmmssSSS").format(new Date())) * 10000 +
(long) (Math.random() * 100);
// Long time = Long.valueOf(new SimpleDateFormat("MMddhhmmssSSS").format(new Date())
// .toString());
// System.out.println(time);
if (id == null) {
id = new AtomicLong(time);
return id.get();
}
if (time <= id.get()) {
id.addAndGet(1);
} else {
id = new AtomicLong(time);
}
return id.get();
}
@Test
public void testGetStoreInfo() {
hgCmdClient = new HgCmdClient();
pdProvider = new DefaultPdProvider(pdAddress);
hgCmdClient.init(new RpcOptions(), null);
Store response = hgCmdClient.getStoreInfo(pdAddress);
}
// @Test
public void testBatchPut() {
hgCmdClient = new HgCmdClient();
pdProvider = new DefaultPdProvider(pdAddress);
hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() {
@Override
public Endpoint getPartitionLeader(String graphName, int partitionId) {
Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, partitionId);
return JRaftUtils.getEndPoint(
pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress());
}
});
storeClient = HgStoreClient.create(PDConfig.of(pdAddress)
.setEnableCache(true));
HgStoreSession session = storeClient.openSession(graphName);
pdClient = storeClient.getPdClient();
session.createTable(tableName);
String createGraph = "create_graph";
HgOwnerKey hgOwnerKey = toOwnerKey(createGraph);
// 需要写数据，才会创建图
session.put(tableName,
hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(createGraph, toStr(session.get(tableName, hgOwnerKey)));
Integer partId = 0;
String key = "key-1";
List<BatchPutRequest.KV> kvs = new LinkedList<>();
int x = 0;
for (int i = 1; i <= 3; i++) {
key = "key-" + i;
BatchPutRequest.KV kv = BatchPutRequest.KV.of(tableName, 1,
key.getBytes(StandardCharsets.UTF_8),
key.getBytes(StandardCharsets.UTF_8));
kvs.add(kv);
BatchPutRequest request = new BatchPutRequest();
request.setGraphName(graphName);
request.setPartitionId(partId);
request.setEntries(kvs);
try {
BatchPutResponse response = hgCmdClient.batchPut(request);
if (response == null) {
log.error("response is null ");
} else if (response.getStatus() == null) {
log.error("response status is null");
}
log.info("response status:{} {}", response.getStatus(), i);
Assert.assertTrue(response.getStatus().isOK());
} catch (Exception e) {
e.printStackTrace();
}
}
HgKvIterator<HgKvEntry> hgKvIterator = session.scanIterator(tableName);
Assert.assertTrue(hgKvIterator.hasNext());
boolean findKey = false;
while (hgKvIterator.hasNext()) {
HgKvEntry entry = hgKvIterator.next();
if (toStr(entry.key()).equals(key) && toStr(entry.value()).equals(key)) {
log.info("key={} value={}", toStr(entry.key()), toStr(entry.value()));
findKey = true;
}
}
Assert.assertTrue(findKey);
}
// @Test
public void testCleanData() {
hgCmdClient = new HgCmdClient();
pdProvider = new DefaultPdProvider(pdAddress);
hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() {
@Override
public Endpoint getPartitionLeader(String graphName, int partitionId) {
Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, partitionId);
return JRaftUtils.getEndPoint(
pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress());
}
});
storeClient = HgStoreClient.create(PDConfig.of(pdAddress)
.setEnableCache(true));
HgStoreSession session = storeClient.openSession(graphName);
pdClient = storeClient.getPdClient();
session.createTable(tableName);
String createGraph = "create_graph";
HgOwnerKey hgOwnerKey = toOwnerKey(createGraph);
// 需要写数据，才会创建图
session.put(tableName,
hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(createGraph, toStr(session.get(tableName, hgOwnerKey)));
Integer partId = 0;
Metapb.Partition pt = Metapb.Partition.newBuilder().build();
CleanDataRequest request = new CleanDataRequest();
request.setGraphName(graphName);
request.setPartitionId(partId);
try {
CleanDataResponse response = hgCmdClient.cleanData(request);
if (response == null) {
log.error("response is null ");
} else if (response.getStatus() == null) {
log.error("response status is null");
}
log.info("response status:{} ", response.getStatus());
Assert.assertTrue(response.getStatus().isOK());
} catch (Exception e) {
e.printStackTrace();
}
}
// @Test
// TODO: partition not local
public void testUpdatePartitionLeader() {
hgCmdClient = new HgCmdClient();
pdProvider = new DefaultPdProvider(pdAddress);
hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() {
@Override
public Endpoint getPartitionLeader(String graphName, int partitionId) {
Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, partitionId);
return JRaftUtils.getEndPoint(
pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress());
}
});
storeClient = HgStoreClient.create(PDConfig.of(pdAddress)
.setEnableCache(true));
HgStoreSession session = storeClient.openSession(graphName);
pdClient = storeClient.getPdClient();
session.createTable(tableName);
String createGraph = "create_graph";
HgOwnerKey hgOwnerKey = toOwnerKey(createGraph);
// 需要写数据，才会创建图
session.put(tableName,
hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(createGraph, toStr(session.get(tableName, hgOwnerKey)));
}
// @Test
// TODO: partition not local
public void testData() {
hgCmdClient = new HgCmdClient();
pdProvider = new DefaultPdProvider(pdAddress);
hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() {
@Override
public Endpoint getPartitionLeader(String graphName, int partitionId) {
Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, partitionId);
return JRaftUtils.getEndPoint(
pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress());
}
});
storeClient = HgStoreClient.create(PDConfig.of(pdAddress)
.setEnableCache(true));
HgStoreSession session = storeClient.openSession("hugegraphtest");
pdClient = storeClient.getPdClient();
session.truncate();
int loop = 3;
for (int i = 0; i < loop; i++) {
HgOwnerKey hgOwnerKey = toOwnerKey(i + "owner:" + i, i + "k:" + i);
session.put(tableName, hgOwnerKey, toIntBytes(i));
}
try {
HgKvIterator<HgKvEntry> iterable = session.scanIterator(tableName);
int x = 0;
while (iterable.hasNext()) {
HgKvEntry entry = iterable.next();
log.info("data:{}-{}", toStr(entry.key()), entry.value());
x++;
}
log.info("x={}", x);
Assert.assertEquals(x, loop);
} catch (Exception e) {
e.printStackTrace();
}
}
// @Test
// TODO: partition not local
public void testCompressionData() {
hgCmdClient = new HgCmdClient();
pdProvider = new DefaultPdProvider(pdAddress);
hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() {
@Override
public Endpoint getPartitionLeader(String graphName, int partitionId) {
Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, partitionId);
return JRaftUtils.getEndPoint(
pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress());
}
});
storeClient = HgStoreClient.create(PDConfig.of(pdAddress)
.setEnableCache(true));
HgStoreSession session = storeClient.openSession("hugegraphtest");
pdClient = storeClient.getPdClient();
session.truncate();
int loop = 10;
for (int i = 0; i < loop; i++) {
String key = "d41d8cd98f00b204e9800998ecf8427e" + getMd5("a" + i) + getId();
String value = "10000" + getId() + getId();
HgOwnerKey hgOwnerKey = toOwnerKey("d41d8cd98f00b204e9800998ecf8427e", key);
session.put(tableName, hgOwnerKey, value.getBytes());
}
try {
HgKvIterator<HgKvEntry> iterable = session.scanIterator(tableName);
int x = 0;
while (iterable.hasNext()) {
HgKvEntry entry = iterable.next();
x++;
}
log.info("x={}", x);
Assert.assertEquals(x, loop);
} catch (Exception e) {
e.printStackTrace();
}
}
}

63
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/MetricServiceTest.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core;
import java.util.Map;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.metric.HgMetricService;
import org.apache.hugegraph.store.metric.SystemMetricService;
import org.junit.Before;
import org.junit.Test;
public class MetricServiceTest {
private SystemMetricService service;
private HgMetricService hgService;
@Before
public void setUp() {
service = new SystemMetricService();
HgStoreEngine instance = HgStoreEngine.getInstance();
service.setStoreEngine(instance);
hgService = HgMetricService.getInstance().setHgStoreEngine(instance);
}
@Test
public void testGetStorageEngine() {
HgStoreEngine result = service.getStorageEngine();
}
@Test
public void testGetSystemMetrics() {
try {
Map<String, Long> systemMetrics = service.getSystemMetrics();
Thread.sleep(1000);
systemMetrics = service.getSystemMetrics();
} catch (Exception e) {
}
}
@Test
public void testGetHgMetrics() {
// Setup
Metapb.StoreStats.Builder systemMetrics = hgService.getMetrics();
}
}

138
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/StoreEngineTestBase.java Normal file
View File

@ -0,0 +1,138 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core;
import java.io.File;
import java.util.HashMap;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.PartitionEngine;
import org.apache.hugegraph.store.UnitTestBase;
import org.apache.hugegraph.store.business.DefaultDataMover;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.meta.ShardGroup;
import org.apache.hugegraph.store.options.HgStoreEngineOptions;
import org.apache.hugegraph.store.options.RaftRocksdbOptions;
import org.apache.hugegraph.store.pd.FakePdServiceProvider;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import com.alipay.sofa.jraft.util.StorageOptionsFactory;
import lombok.extern.slf4j.Slf4j;
/**
* 使用 FakePd 和 FakePdOptions，初始化 HgStoreEngine，该类的各项 get 函数可用
*/
@Slf4j
public class StoreEngineTestBase {
private static final String DB_PATH = "/tmp/junit";
private static int initCount = 0;
@BeforeClass
public static void initEngine() {
UnitTestBase.deleteDir(new File(DB_PATH));
HgStoreEngineOptions options = new HgStoreEngineOptions();
options.setDataPath(DB_PATH);
options.setRaftPath(DB_PATH);
options.setFakePD(true);
options.setRocksdbConfig(new HashMap<>() {{
put("rocksdb.write_buffer_size", "1048576");
}});
options.setGrpcAddress("127.0.0.1:6511");
options.setRaftAddress("127.0.0.1:6510");
options.setDataTransfer(new DefaultDataMover());
options.setFakePdOptions(new HgStoreEngineOptions.FakePdOptions() {{
setStoreList("127.0.0.1");
setPeersList("127.0.0.1");
setPartitionCount(1);
setShardCount(1);
}});
if (initCount == 0) {
StorageOptionsFactory.releaseAllOptions();
RaftRocksdbOptions.initRocksdbGlobalConfig(options.getRocksdbConfig());
initCount++;
}
HgStoreEngine.getInstance().init(options);
}
public static Partition getPartition(int partitionId) {
return getPartition(partitionId, "graph0");
}
public static Partition getPartition(int partitionId, String graphName) {
Partition partition = new Partition();
partition.setId(partitionId);
partition.setGraphName(graphName);
partition.setStartKey(0);
partition.setEndKey(65535);
partition.setWorkState(Metapb.PartitionState.PState_Normal);
partition.setVersion(1);
return partition;
}
/**
* 创建 分区为 0 的 partition engine. 该分区 1 个 shard，为 leader, graph name: graph0
*
* @return
*/
public static PartitionEngine createPartitionEngine(int partitionId) {
return createPartitionEngine(partitionId, "graph0");
}
public static PartitionEngine createPartitionEngine(int partitionId, String graphName) {
Metapb.Shard shard = Metapb.Shard.newBuilder()
.setStoreId(FakePdServiceProvider.makeStoreId(
"127.0.0.1:6511"))
.setRole(Metapb.ShardRole.Leader)
.build();
Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder()
.setId(partitionId)
.setConfVer(1)
.setVersion(1)
.setState(
Metapb.PartitionState.PState_Normal)
.addShards(shard)
.build();
getStoreEngine().getPartitionManager().updateShardGroup(ShardGroup.from(shardGroup));
var engine = getStoreEngine().createPartitionEngine(getPartition(partitionId, graphName));
engine.waitForLeader(2000);
return engine;
}
public static HgStoreEngine getStoreEngine() {
return HgStoreEngine.getInstance();
}
@AfterClass
public static void shutDownEngine() {
try {
HgStoreEngine.getInstance().shutdown();
} catch (Exception e) {
log.error("shut down engine error: {}", e.getMessage());
}
}
}

247
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java Normal file
View File

@ -0,0 +1,247 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.raft;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.apache.hugegraph.store.raft.HgStoreStateMachine;
import org.apache.hugegraph.store.raft.RaftClosure;
import org.apache.hugegraph.store.raft.RaftOperation;
import org.apache.hugegraph.store.raft.RaftStateListener;
import org.apache.hugegraph.store.raft.RaftTaskHandler;
import org.apache.hugegraph.store.snapshot.HgSnapshotHandler;
import org.apache.hugegraph.store.util.HgStoreException;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import com.alipay.sofa.jraft.Closure;
import com.alipay.sofa.jraft.Iterator;
import com.alipay.sofa.jraft.Status;
import com.alipay.sofa.jraft.conf.Configuration;
import com.alipay.sofa.jraft.entity.LeaderChangeContext;
import com.alipay.sofa.jraft.entity.PeerId;
import com.alipay.sofa.jraft.entity.Task;
import com.alipay.sofa.jraft.error.RaftError;
import com.alipay.sofa.jraft.error.RaftException;
@RunWith(MockitoJUnitRunner.class)
public class HgStoreStateMachineTest {
@Mock
private HgSnapshotHandler mockSnapshotHandler;
private HgStoreStateMachine hgStoreStateMachineUnderTest;
@Before
public void setUp() {
hgStoreStateMachineUnderTest = new HgStoreStateMachine(0, mockSnapshotHandler);
}
@Test
public void testAddTaskHandler() {
// Setup
final RaftTaskHandler handler = new RaftTaskHandler() {
@Override
public boolean invoke(int groupId, byte[] request, RaftClosure response) throws
HgStoreException {
return false;
}
@Override
public boolean invoke(int groupId, byte methodId, Object req, RaftClosure response)
throws HgStoreException {
return false;
}
};
// Run the test
hgStoreStateMachineUnderTest.addTaskHandler(handler);
// Verify the results
}
@Test
public void testAddStateListener() {
// Setup
final RaftStateListener mockListener = new RaftStateListener() {
@Override
public void onLeaderStart(long newTerm) {
}
@Override
public void onError(RaftException e) {
}
};
// Run the test
hgStoreStateMachineUnderTest.addStateListener(mockListener);
// Verify the results
}
@Test
public void testIsLeader() {
// Setup
// Run the test
final boolean result = hgStoreStateMachineUnderTest.isLeader();
// Verify the results
assertFalse(result);
}
@Test
public void testOnApply() {
RaftOperation op = RaftOperation.create((byte) 0b0);
final Task task = new Task();
task.setData(ByteBuffer.wrap(op.getValues()));
task.setDone(new HgStoreStateMachine.RaftClosureAdapter(op, closure -> {
}));
List<Task> tasks = new ArrayList<>();
tasks.add(task);
// Setup
final Iterator inter = new Iterator() {
final java.util.Iterator<Task> iterator = tasks.iterator();
Task task;
@Override
public ByteBuffer getData() {
return task.getData();
}
@Override
public long getIndex() {
return 0;
}
@Override
public long getTerm() {
return 0;
}
@Override
public Closure done() {
return null;
}
@Override
public void setErrorAndRollback(long ntail, Status st) {
}
@Override
public boolean hasNext() {
return iterator.hasNext();
}
@Override
public ByteBuffer next() {
task = iterator.next();
return task.getData();
}
};
// Run the test
hgStoreStateMachineUnderTest.onApply(inter);
// Verify the results
}
@Test
public void testGetLeaderTerm() {
// Setup
// Run the test
final long result = hgStoreStateMachineUnderTest.getLeaderTerm();
// Verify the results
assertEquals(-1L, result);
}
@Test
public void testOnLeaderStart() {
// Setup
// Run the test
hgStoreStateMachineUnderTest.onLeaderStart(0L);
// Verify the results
}
@Test
public void testOnLeaderStop() {
// Setup
final Status status = new Status(RaftError.UNKNOWN, "fmt", "args");
// Run the test
hgStoreStateMachineUnderTest.onLeaderStop(status);
// Verify the results
}
@Test
public void testOnStartFollowing() {
// TODO: uncomment later (jraft)
// // Setup
// final LeaderChangeContext ctx =
// new LeaderChangeContext(new PeerId("ip", 0, 0, 0), "groupId", 0L,
// new Status(RaftError.UNKNOWN, "fmt", "args"));
//
// // Run the test
// hgStoreStateMachineUnderTest.onStartFollowing(ctx);
// Verify the results
}
@Test
public void testOnStopFollowing() {
// TODO: uncomment later (jraft)
// // Setup
// final LeaderChangeContext ctx =
// new LeaderChangeContext(new PeerId("ip", 0, 0, 0), "groupId", 0L,
// new Status(RaftError.UNKNOWN, "fmt", "args"));
//
// // Run the test
// hgStoreStateMachineUnderTest.onStopFollowing(ctx);
// Verify the results
}
@Test
public void testOnConfigurationCommitted() {
// Setup
final Configuration conf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)),
List.of(new PeerId("ip", 0, 0, 0)));
// Run the test
hgStoreStateMachineUnderTest.onConfigurationCommitted(conf);
// Verify the results
}
}

30
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/MockStateMachine.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.raft;
import com.alipay.sofa.jraft.Iterator;
import com.alipay.sofa.jraft.core.StateMachineAdapter;
public class MockStateMachine extends StateMachineAdapter {
@Override
public void onApply(Iterator iter) {
while (iter.hasNext()) {
iter.next();
}
}
}

75
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/RaftOperationTest.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.raft;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.raft.RaftOperation;
import org.junit.Before;
import org.junit.Test;
import com.google.protobuf.GeneratedMessageV3;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
public class RaftOperationTest {
private RaftOperation raftOperationUnderTest;
@Before
public void setUp() {
raftOperationUnderTest = new RaftOperation();
}
@Test
public void testCreate1() {
// Run the test
final RaftOperation result = RaftOperation.create((byte) 0b0);
assertEquals(null, result.getReq());
assertEquals((byte) 0b0, result.getOp());
}
@Test
public void testCreate2() {
// Run the test
final RaftOperation result = RaftOperation.create((byte) 0b0, "content".getBytes(), "req");
assertArrayEquals("content".getBytes(), result.getValues());
assertEquals("req", result.getReq());
assertEquals((byte) 0b0, result.getOp());
}
@Test
public void testCreate3() {
// Run the test
final RaftOperation result = RaftOperation.create((byte) 0b0, "req");
assertEquals("req", result.getReq());
assertEquals((byte) 0b0, result.getOp());
}
@Test
public void testCreate4() throws Exception {
// Setup
final GeneratedMessageV3 req = Metapb.Graph.newBuilder().setGraphName("name").build();
// Run the test
final RaftOperation result = RaftOperation.create((byte) 0b0, req);
assertEquals((byte) 0b0, result.getOp());
}
}

179
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/RaftUtilsTest.java Normal file
View File

@ -0,0 +1,179 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.raft;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.Collections;
import java.util.List;
import org.apache.hugegraph.store.UnitTestBase;
import org.apache.hugegraph.store.raft.util.RaftUtils;
import org.junit.Before;
import org.junit.Test;
import com.alipay.sofa.jraft.Node;
import com.alipay.sofa.jraft.NodeManager;
import com.alipay.sofa.jraft.conf.Configuration;
import com.alipay.sofa.jraft.core.NodeImpl;
import com.alipay.sofa.jraft.entity.PeerId;
import com.alipay.sofa.jraft.option.NodeOptions;
import com.alipay.sofa.jraft.util.Endpoint;
public class RaftUtilsTest {
final String ip = "127.0.0.1";
final int port = 12456;
final String dataPath = "tmp/raftUtils";
@Before
public void setUp() {
UnitTestBase.deleteDir(new File(dataPath));
new File(dataPath).mkdirs();
}
@Test
public void testGetAllEndpoints1() throws InterruptedException {
final Endpoint addr = new Endpoint(ip, port);
final PeerId peer = new PeerId(addr, 0);
NodeManager.getInstance().addAddress(addr);
final NodeOptions nodeOptions = new NodeOptions();
nodeOptions.setFsm(new MockStateMachine());
nodeOptions.setLogUri(this.dataPath + File.separator + "log");
nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer)));
final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
assertTrue(node.init(nodeOptions));
node.isLeader(true);
// Run the test
final List<String> result = RaftUtils.getAllEndpoints(node);
// Verify the results
assertEquals(1, result.size());
node.shutdown();
node.join();
}
@Test
public void testGetAllEndpoints2() {
final Endpoint addr = new Endpoint(ip, port);
final PeerId peer = new PeerId(addr, 0);
Configuration conf = new Configuration(Collections.singletonList(peer));
// Run the test
final List<String> result = RaftUtils.getAllEndpoints(conf);
// Verify the results
assertEquals(1, result.size());
}
@Test
public void testGetPeerEndpoints1() throws InterruptedException {
final Endpoint addr = new Endpoint(ip, port);
final PeerId peer = new PeerId(addr, 0);
NodeManager.getInstance().addAddress(addr);
final NodeOptions nodeOptions = new NodeOptions();
nodeOptions.setFsm(new MockStateMachine());
nodeOptions.setLogUri(this.dataPath + File.separator + "log");
nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer)));
final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
assertTrue(node.init(nodeOptions));
node.isLeader(true);
// Run the test
final List<String> result = RaftUtils.getPeerEndpoints(node);
// Verify the results
assertEquals(1, result.size());
node.shutdown();
node.join();
}
@Test
public void testGetPeerEndpoints2() {
// Setup
final Configuration conf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)),
List.of(new PeerId("ip", 0, 0, 0)));
// Run the test
final List<String> result = RaftUtils.getPeerEndpoints(conf);
// Verify the results
assertEquals(1, result.size());
}
@Test
public void testGetLearnerEndpoints1() throws InterruptedException {
// Setup
final Endpoint addr = new Endpoint(ip, port);
final PeerId peer = new PeerId(addr, 0);
final PeerId peer2 = new PeerId(new Endpoint(ip, 13456), 0);
NodeManager.getInstance().addAddress(addr);
final NodeOptions nodeOptions = new NodeOptions();
nodeOptions.setFsm(new MockStateMachine());
nodeOptions.setLogUri(this.dataPath + File.separator + "log");
nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer),
Collections.singletonList(peer2)));
final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
assertTrue(node.init(nodeOptions));
node.isLeader(true);
// Run the test
final List<String> result = RaftUtils.getLearnerEndpoints(node);
// Verify the results
assertEquals(1, result.size());
node.shutdown();
node.join();
}
@Test
public void testGetLearnerEndpoints2() {
// Setup
final Configuration conf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)),
List.of(new PeerId("ip", 0, 0, 0)));
// Run the test
final List<String> result = RaftUtils.getLearnerEndpoints(conf);
// Verify the results
assertEquals(1, result.size());
}
@Test
public void testConfigurationEquals() {
// Setup
final Configuration oldConf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)),
List.of(new PeerId("ip", 0, 0, 0)));
final Configuration newConf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)),
List.of(new PeerId("ip", 0, 0, 0)));
// Run the test
final boolean result = RaftUtils.configurationEquals(oldConf, newConf);
// Verify the results
assertTrue(result);
}
}

187
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java Normal file
View File

@ -0,0 +1,187 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.snapshot;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.hugegraph.store.core.StoreEngineTestBase;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.snapshot.HgSnapshotHandler;
import org.junit.Before;
import org.junit.Test;
import com.alipay.sofa.jraft.entity.RaftOutter;
import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
import com.google.protobuf.Message;
public class HgSnapshotHandlerTest extends StoreEngineTestBase {
private static HgSnapshotHandler hgSnapshotHandlerUnderTest;
@Before
public void setUp() throws IOException {
hgSnapshotHandlerUnderTest = new HgSnapshotHandler(createPartitionEngine(0));
FileUtils.forceMkdir(new File("/tmp/snapshot"));
FileUtils.forceMkdir(new File("/tmp/snapshot/data"));
}
@Test
public void testGetPartitions() {
// Run the test
final Map<String, Partition> result = hgSnapshotHandlerUnderTest.getPartitions();
// Verify the results
assertEquals(1, result.size());
}
@Test
public void testOnSnapshotSaveAndLoad() {
String path = "/tmp/snapshot";
// Setup
final SnapshotWriter writer = new SnapshotWriter() {
@Override
public boolean saveMeta(RaftOutter.SnapshotMeta meta) {
return false;
}
@Override
public boolean addFile(String fileName, Message fileMeta) {
return false;
}
@Override
public boolean removeFile(String fileName) {
return false;
}
@Override
public void close(boolean keepDataOnError) throws IOException {
}
@Override
public boolean init(Void opts) {
return false;
}
@Override
public void shutdown() {
}
@Override
public String getPath() {
return path;
}
@Override
public Set<String> listFiles() {
return null;
}
@Override
public Message getFileMeta(String fileName) {
return null;
}
@Override
public void close() throws IOException {
}
};
// Run the test
hgSnapshotHandlerUnderTest.onSnapshotSave(writer);
// Verify the results
// Setup
final SnapshotReader reader = new SnapshotReader() {
final String path = "/tmp/snapshot";
@Override
public RaftOutter.SnapshotMeta load() {
return null;
}
@Override
public String generateURIForCopy() {
return null;
}
@Override
public boolean init(Void opts) {
return false;
}
@Override
public void shutdown() {
}
@Override
public String getPath() {
return path;
}
@Override
public Set<String> listFiles() {
return null;
}
@Override
public Message getFileMeta(String fileName) {
return null;
}
@Override
public void close() throws IOException {
}
};
// Run the test
hgSnapshotHandlerUnderTest.onSnapshotLoad(reader, 0L);
}
@Test
public void testTrimStartPath() {
assertEquals("str", HgSnapshotHandler.trimStartPath("str", "prefix"));
}
@Test
public void testFindFileList() {
// Setup
final File dir = new File("filename.txt");
final File rootDir = new File("filename.txt");
// Run the test
HgSnapshotHandler.findFileList(dir, rootDir, List.of("value"));
// Verify the results
}
}

175
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/HgStoreEngineTest.java Normal file
View File

@ -0,0 +1,175 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.store;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import java.util.List;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.core.StoreEngineTestBase;
import org.apache.hugegraph.store.pd.FakePdServiceProvider;
import org.apache.hugegraph.testutil.Assert;
import org.junit.Before;
import org.junit.Test;
public class HgStoreEngineTest extends StoreEngineTestBase {
private HgStoreEngine engine;
@Before
public void setup() {
engine = HgStoreEngine.getInstance();
}
@Test
public void testGetInstance() {
assertNotNull(HgStoreEngine.getInstance());
}
// @Test
// TODO: npe and not leader
public void testStateChanged() {
createPartitionEngine(0);
var storeId = FakePdServiceProvider.makeStoreId("127.0.0.1:6511");
var store = engine.getPartitionManager().getStore(storeId);
engine.stateChanged(store, Metapb.StoreState.Offline, Metapb.StoreState.Up);
assertEquals(engine.getPartitionEngines().size(), 1);
}
@Test
public void testCreatePartitionEngine() {
var partition = getPartition(0);
assertNotNull(engine.createPartitionEngine(partition));
}
@Test
public void testCreatePartitionGroups() {
var partition = getPartition(0);
engine.createPartitionGroups(partition);
}
@Test
public void testDestroyPartitionEngine() {
createPartitionEngine(16);
engine.destroyPartitionEngine(16, List.of("graph0"));
// assertEquals(engine.getPartitionEngines().size(), 0);
}
@Test
public void testDeletePartition() {
createPartitionEngine(0);
engine.deletePartition(0, "graph0");
// TODO: check logic
assertEquals(engine.getPartitionEngines().size(), 1);
}
@Test
public void testGetLeaderPartition() throws InterruptedException {
createPartitionEngine(0);
assertEquals(engine.getLeaderPartition().size(), 1);
}
@Test
public void testGetAlivePeers() throws InterruptedException {
createPartitionEngine(0);
assertEquals(engine.getAlivePeers(0).size(), 1);
}
@Test
public void testGetLeaderTerm() {
createPartitionEngine(0);
// no vote
assertEquals(engine.getLeaderTerm(0), -1);
}
@Test
public void testGetCommittedIndex() throws InterruptedException {
createPartitionEngine(0);
// write something background
Assert.assertTrue(engine.getCommittedIndex(0) > 0);
}
@Test
public void testGetRaftRpcServer() {
assertNotNull(engine.getRaftRpcServer());
}
@Test
public void testGetPartitionManager() {
assertNotNull(engine.getPartitionManager());
}
@Test
public void testGetDataMover() {
assertNotNull(engine.getDataMover());
}
@Test
public void testGetPdProvider() {
assertNotNull(engine.getPdProvider());
}
@Test
public void testGetCmdClient() {
assertNotNull(engine.getHgCmdClient());
}
@Test
public void testGetHeartbeatService() {
assertNotNull(engine.getHeartbeatService());
}
@Test
public void testIsClusterReady() throws InterruptedException {
// wait heart beat
Thread.sleep(2000);
assertNotNull(engine.isClusterReady());
}
@Test
public void testGetDataLocations() {
assertEquals(engine.getDataLocations().size(), 1);
}
@Test
public void testGetPartitionEngine() {
createPartitionEngine(0);
assertNotNull(engine.getPartitionEngine(0));
}
@Test
public void testGetPartitionEngines() {
createPartitionEngine(0);
assertEquals(engine.getPartitionEngines().size(), 1);
}
@Test
public void testGetNodeMetrics() {
assertNotNull(engine.getNodeMetrics());
}
@Test
public void testGetRaftGroupCount() {
createPartitionEngine(0);
assertEquals(engine.getRaftGroupCount(), 1);
}
}

93
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/PartitionEngineTest.java Normal file
View File

@ -0,0 +1,93 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.store;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import org.apache.hugegraph.store.PartitionEngine;
import org.apache.hugegraph.store.core.StoreEngineTestBase;
import org.junit.Before;
import org.junit.Test;
public class PartitionEngineTest extends StoreEngineTestBase {
PartitionEngine engine;
@Before
public void setup() {
engine = createPartitionEngine(0);
}
@Test
public void testHasPartition() {
assertTrue(engine.hasPartition("graph0"));
}
@Test
public void testGetGroupId() {
assertEquals(engine.getGroupId().intValue(), 0);
}
@Test
public void testGetShardGroup() {
assertEquals(engine.getShardGroup().getShards().size(), 1);
}
@Test
public void testIsLeader() {
System.out.println(engine.isLeader());
}
@Test
public void testGetLeader() {
assertEquals(engine.getLeader().toString(), engine.getOptions().getRaftAddress());
}
@Test
public void testGetAlivePeers() {
try {
System.out.println(engine.getAlivePeers().size());
} catch (Exception e) {
e.printStackTrace();
}
}
@Test
public void testGetRaftNode() {
assertNotNull(engine.getRaftNode());
}
@Test
public void testGetPartitions() {
assertEquals(engine.getPartitions().get("graph0").getId(), 0);
}
@Test
public void testGetPartition() {
assertEquals(engine.getPartition("graph0").getId(), 0);
}
@Test
public void testGetCommittedIndex() throws InterruptedException {
Thread.sleep(1000);
System.out.println(engine.getCommittedIndex());
}
}

109
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/PartitionInstructionProcessorTest.java Normal file
View File

@ -0,0 +1,109 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.store;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.List;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.pd.grpc.pulse.DbCompaction;
import org.apache.hugegraph.pd.grpc.pulse.SplitPartition;
import org.apache.hugegraph.pd.grpc.pulse.TransferLeader;
import org.apache.hugegraph.store.PartitionInstructionProcessor;
import org.apache.hugegraph.store.core.StoreEngineTestBase;
import org.apache.hugegraph.store.pd.FakePdServiceProvider;
import org.junit.Before;
import org.junit.Test;
public class PartitionInstructionProcessorTest extends StoreEngineTestBase {
PartitionInstructionProcessor processor;
@Before
public void init() {
processor = new PartitionInstructionProcessor(getStoreEngine());
}
@Test
public void testTransferLeader() {
var engine = createPartitionEngine(0);
engine.waitForLeader(1000);
var shard = Metapb.Shard.newBuilder()
.setStoreId(FakePdServiceProvider.makeStoreId("127.0.0.1:6511"))
.setRole(Metapb.ShardRole.Leader)
.build();
TransferLeader trans = TransferLeader.newBuilder()
.setShard(shard)
.build();
processor.onTransferLeader(1, getPartition(0), trans, integer -> {
assertEquals(0, integer.intValue());
});
}
@Test
public void testDbCompaction() throws InterruptedException {
var engine = createPartitionEngine(0);
engine.waitForLeader(1000);
DbCompaction dbCompaction = DbCompaction.newBuilder()
.setTableName("test")
.build();
processor.onDbCompaction(2, getPartition(0), dbCompaction, integer -> {
assertEquals(0, integer.intValue());
});
Thread.sleep(2000);
}
@Test
public void testSplitPartition() throws InterruptedException {
var engine = createPartitionEngine(0);
engine.waitForLeader(1000);
var partition = getPartition(0);
System.out.println(partition);
List<Metapb.Partition> list = new ArrayList<>();
list.add(Metapb.Partition.newBuilder(partition.getProtoObj())
.setStartKey(0)
.setEndKey(20000)
.build());
list.add(Metapb.Partition.newBuilder(partition.getProtoObj())
.setStartKey(20000)
.setEndKey(65535)
.setId(1).build());
SplitPartition splitPartition = SplitPartition.newBuilder()
.addAllNewPartition(list)
.build();
processor.onSplitPartition(3, partition, splitPartition, integer -> {
assertEquals(0, integer.intValue());
});
Thread.sleep(2000);
System.out.println(getStoreEngine().getPartitionEngines());
}
@Test
public void testChangeShard() {
}
}

34
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/MetadataKeyHelperTest.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.store.meta;
import static org.junit.Assert.assertArrayEquals;
import org.apache.hugegraph.store.meta.MetadataKeyHelper;
import org.junit.Test;
public class MetadataKeyHelperTest {
@Test
public void testKey() {
assertArrayEquals("HUGEGRAPH/TASK/".getBytes(), MetadataKeyHelper.getTaskPrefix());
assertArrayEquals("HUGEGRAPH/TASK/0/".getBytes(), MetadataKeyHelper.getTaskPrefix(0));
assertArrayEquals("HUGEGRAPH/TASK_DONE/0000000000000000".getBytes(),
MetadataKeyHelper.getDoneTaskKey(0));
}
}

198
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/PartitionManagerTest.java Normal file
View File

@ -0,0 +1,198 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.store.meta;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.List;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.cmd.UpdatePartitionRequest;
import org.apache.hugegraph.store.core.StoreEngineTestBase;
import org.apache.hugegraph.store.meta.Graph;
import org.apache.hugegraph.store.meta.GraphManager;
import org.apache.hugegraph.store.meta.PartitionManager;
import org.apache.hugegraph.store.pd.FakePdServiceProvider;
import org.junit.Before;
import org.junit.Test;
public class PartitionManagerTest extends StoreEngineTestBase {
private PartitionManager manager;
@Before
public void setup() {
manager = getStoreEngine().getPartitionManager();
}
@Test
public void testGetDeletedFileManager() {
assertNotNull(manager.getDeletedFileManager());
}
@Test
public void testGetPdProvider() {
assertEquals(manager.getPdProvider().getClass(), FakePdServiceProvider.class);
}
@Test
public void testGetStoreMetadata() {
assertNotNull(manager.getStoreMetadata());
}
@Test
public void testSetStore() {
var storeId = FakePdServiceProvider.makeStoreId("127.0.0.1:6511");
var store = manager.getStore(storeId);
manager.setStore(store);
var store2 = manager.getStoreMetadata().getStore();
assertEquals(store.getId(), store2.getId());
}
@Test
public void testUpdatePartition() {
var partition = getPartition(5);
manager.updatePartition(partition.getProtoObj(), true);
var partition2 = manager.findPartition("graph0", 5);
assertEquals(partition.getGraphName(), partition2.getGraphName());
var partition3 = manager.loadPartitionFromSnapshot(partition2);
assertEquals(partition3.getGraphName(), partition2.getGraphName());
}
@Test
public void testChangeState() {
createPartitionEngine(4);
var partition = getPartition(4);
manager.changeState(partition, Metapb.PartitionState.PState_Offline);
var partition2 = manager.findPartition("graph0", 4);
assertEquals(partition2.getWorkState(), Metapb.PartitionState.PState_Offline);
}
@Test
public void testChangeKeyRange() {
createPartitionEngine(4);
var partition = getPartition(4);
manager.changeKeyRange(partition, 1000, 2000);
var partition2 = manager.findPartition("graph0", 4);
assertEquals(partition2.getStartKey(), 1000);
assertEquals(partition2.getEndKey(), 2000);
}
@Test
public void testUpdatePartitionRangeOrState() {
createPartitionEngine(4);
UpdatePartitionRequest request = new UpdatePartitionRequest();
request.setPartitionId(4);
request.setGraphName("graph0");
request.setStartKey(2000);
request.setEndKey(3000);
request.setWorkState(Metapb.PartitionState.PState_Offline);
manager.updatePartitionRangeOrState(request);
var partition = manager.findPartition("graph0", 4);
assertEquals(partition.getStartKey(), 2000);
assertEquals(partition.getEndKey(), 3000);
assertEquals(partition.getWorkState(), Metapb.PartitionState.PState_Offline);
}
@Test
public void testGetLeaderPartitionIds() {
createPartitionEngine(0);
createPartitionEngine(4);
createPartitionEngine(5);
System.out.println(manager.getLeaderPartitionIds("graph0"));
assertEquals(manager.getLeaderPartitionIds("graph0").size(), 3);
}
@Test
public void testisLocal() {
createPartitionEngine(0);
assertTrue(manager.isLocalPartition(0));
assertTrue(manager.isLocalPartition(getPartition(0)));
assertTrue(manager.isLocalStore(manager.getStore()));
}
@Test
public void testUploadToPd() throws PDException {
createPartitionEngine(0);
var partition = manager.findPartition("graph0", 0);
var list = new ArrayList<Metapb.Partition>();
list.add(partition.getProtoObj());
// fake pd, return nothing
assertEquals(1, manager.updatePartitionToPD(list).size());
manager.reportTask(null);
var partitions = manager.changePartitionToOnLine(list);
assertSame(partitions.get(0).getState(), Metapb.PartitionState.PState_Normal);
// fake pd
// TODO: uncomment it until fix it
// assertNotNull(manager.findPartition("graph0", 1000));
}
@Test
public void testShards2Peers() {
var storeId = FakePdServiceProvider.makeStoreId("127.0.0.1:6511");
Metapb.Shard shard = Metapb.Shard.newBuilder()
.setStoreId(storeId)
.setRole(Metapb.ShardRole.Leader)
.build();
List<Metapb.Shard> list = new ArrayList<>();
list.add(shard);
var peers = manager.shards2Peers(list);
assertEquals("127.0.0.1:6510", peers.get(0));
}
@Test
public void testLoad() {
createPartitionEngine(0);
var graphManager = new GraphManager(manager.getOptions(), manager.getPdProvider());
var graph = Metapb.Graph.newBuilder()
.setGraphName("graph0")
.setPartitionCount(12)
.build();
graphManager.updateGraph(new Graph(graph));
manager.load();
assertNotNull(manager.getLeaderPartitionIds("graph0"));
}
@Test
public void testSyncPartitionsFromPD() throws PDException {
createPartitionEngine(0);
// from fake pd
manager.syncPartitionsFromPD(partition -> {
});
assertTrue(manager.getPartitions().isEmpty());
}
}

108
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/TaskManagerTest.java Normal file
View File

@ -0,0 +1,108 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.store.meta;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import org.apache.hugegraph.pd.grpc.MetaTask;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.core.StoreEngineTestBase;
import org.apache.hugegraph.store.meta.TaskManager;
import org.apache.hugegraph.store.meta.asynctask.AsyncTaskState;
import org.apache.hugegraph.store.meta.asynctask.CleanTask;
import org.junit.Before;
import org.junit.Test;
public class TaskManagerTest extends StoreEngineTestBase {
private TaskManager manager;
@Before
public void setup() {
createPartitionEngine(0, "graph0");
manager = getStoreEngine().getPartitionEngine(0).getTaskManager();
}
@Test
public void testAsyncTaskReadAndWrite() {
var task1 = new CleanTask(0, "graph0", AsyncTaskState.START, null);
var task2 = new CleanTask(0, "graph0", AsyncTaskState.FAILED, null);
var task3 = new CleanTask(0, "graph0", AsyncTaskState.SUCCESS, null);
var task4 = new CleanTask(0, "graph0", AsyncTaskState.FAILED, null);
manager.putAsyncTask(task1);
manager.putAsyncTask(task2);
manager.putAsyncTask(task3);
manager.putAsyncTask(task4);
var list = manager.scanAsyncTasks(0, "graph0");
assertEquals(list.size(), 4);
var newTask1 = (CleanTask) manager.getOneAsyncTask(0, "graph0", task1.getId());
assertEquals(task1.getState(), newTask1.getState());
assertEquals(task1.getType(), newTask1.getType());
manager.updateAsyncTaskState(0, "graph0", task4.getId(), AsyncTaskState.SUCCESS);
var newTask4 = (CleanTask) manager.getOneAsyncTask(0, "graph0", task4.getId());
assertEquals(newTask4.getState(), AsyncTaskState.SUCCESS);
assertNull(manager.getOneAsyncTask(1, "graph0", ""));
}
@Test
public void testTaskOp() {
MetaTask.Task task1 = MetaTask.Task.newBuilder()
.setId(1)
.setState(MetaTask.TaskState.Task_Ready)
.setType(MetaTask.TaskType.Split_Partition)
.setPartition(Metapb.Partition.newBuilder()
.setGraphName("graph0")
.setId(0).build())
.build();
manager.updateTask(task1);
assertTrue(manager.taskExists(task1));
assertFalse(manager.taskExists(0, "graph0",
MetaTask.TaskType.Split_Partition.name()));
assertFalse(manager.partitionTaskRepeat(0, "graph0",
MetaTask.TaskType.Split_Partition.name()));
MetaTask.Task task2 = MetaTask.Task.newBuilder(task1).setId(2).build();
manager.updateTask(task2);
assertTrue(manager.taskExists(0, "graph0",
MetaTask.TaskType.Split_Partition.name()));
assertTrue(manager.partitionTaskRepeat(0, "graph0",
MetaTask.TaskType.Split_Partition.name()));
MetaTask.Task task3 = MetaTask.Task.newBuilder(task1)
.setId(3)
.setState(MetaTask.TaskState.Task_Success)
.setPartition(
Metapb.Partition.newBuilder(task1.getPartition())
.setGraphName("graph1")
.setId(1)
.build())
.build();
manager.updateTask(task3);
assertFalse(manager.taskExists(task3));
}
}

49
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/asynctask/CleanTaskTest.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.store.meta.asynctask;
import static org.junit.Assert.assertEquals;
import org.apache.hugegraph.store.meta.asynctask.AbstractAsyncTask;
import org.apache.hugegraph.store.meta.asynctask.AsyncTask;
import org.apache.hugegraph.store.meta.asynctask.AsyncTaskState;
import org.apache.hugegraph.store.meta.asynctask.CleanTask;
import org.junit.Test;
import org.apache.hugegraph.store.core.StoreEngineTestBase;
public class CleanTaskTest extends StoreEngineTestBase {
@Test
public void testSerialize() {
CleanTask task = new CleanTask(0, "graph0", AsyncTaskState.SUCCESS, null);
byte[] bytes = task.toBytes();
AsyncTask task2 = AbstractAsyncTask.fromBytes(bytes);
assertEquals(CleanTask.class, task2.getClass());
System.out.println(task2);
createPartitionEngine(0);
CleanTask task3 = new CleanTask(0, "graph0", AsyncTaskState.START, null);
CleanTask task4 = new CleanTask(0, "graph0", AsyncTaskState.FAILED, null);
task3.handleTask();
task4.handleTask();
}
}

93
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/MiscUtilClassTest.java Normal file
View File

@ -0,0 +1,93 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.store.util;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.concurrent.TimeUnit;
import org.apache.hugegraph.store.util.Asserts;
import org.apache.hugegraph.store.util.HgRaftError;
import org.apache.hugegraph.store.util.HgStoreException;
import org.apache.hugegraph.store.util.ManualResetEvent;
import org.junit.Test;
import com.alipay.sofa.jraft.Status;
public class MiscUtilClassTest {
@Test
public void testHgRaftError() {
HgRaftError error = HgRaftError.forNumber(0);
assertEquals(0, error.getNumber());
assertEquals("OK", error.getMsg());
assertEquals(Status.OK().getCode(), error.toStatus().getCode());
}
@Test(expected = NullPointerException.class)
public void testAsserts() {
assertTrue(Asserts.isInvalid(null));
assertTrue(Asserts.isInvalid());
assertTrue(Asserts.isInvalid(null));
assertTrue(Asserts.isInvalid(""));
assertFalse(Asserts.isInvalid("a"));
Asserts.isNonNull(null);
Asserts.isNonNull(null, "msg");
}
@Test(expected = IllegalArgumentException.class)
public void testAsserts2() {
Asserts.isTrue(false, "");
Asserts.isFalse(true, "");
Asserts.isTrue(true, null);
}
@Test
public void testHgStoreException() {
var exception = new HgStoreException();
assertEquals(0, exception.getCode());
exception = new HgStoreException("invalid");
assertEquals(1000, exception.getCode());
exception = new HgStoreException(1000, "invalid");
assertEquals(1000, exception.getCode());
exception = new HgStoreException(1000, new Throwable());
assertEquals(1000, exception.getCode());
exception = new HgStoreException("invalid", new Throwable());
assertEquals(1000, exception.getCode());
exception = new HgStoreException(1000, "%s", "invalid");
assertEquals(1000, exception.getCode());
}
@Test
public void testManualResetEvent() throws InterruptedException {
ManualResetEvent event = new ManualResetEvent(false);
assertFalse(event.isSignalled());
event.set();
assertTrue(event.isSignalled());
event.reset();
assertFalse(event.waitOne(1, TimeUnit.SECONDS));
event.set();
event.waitOne();
assertTrue(event.isSignalled());
}
}

87
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/PartitionMetaStoreWrapperTest.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.store.util;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.core.StoreEngineTestBase;
import org.apache.hugegraph.store.meta.MetadataKeyHelper;
import org.apache.hugegraph.store.util.PartitionMetaStoreWrapper;
import org.junit.Before;
import org.junit.Test;
public class PartitionMetaStoreWrapperTest extends StoreEngineTestBase {
private PartitionMetaStoreWrapper wrapper;
private Metapb.Partition partition;
public static void putToDb(Metapb.Partition partition, PartitionMetaStoreWrapper wrapper) {
byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId());
wrapper.put(partition.getId(), key, partition.toByteArray());
}
@Before
public void setup() {
wrapper = new PartitionMetaStoreWrapper();
partition = Metapb.Partition.newBuilder()
.setId(1)
.setGraphName("graph0")
.setStartKey(0L)
.setEndKey(65535L)
.build();
}
@Test
public void testGet() {
putToDb(partition, wrapper);
byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId());
assertEquals(partition, wrapper.get(1, key, Metapb.Partition.parser()));
byte[] key2 = MetadataKeyHelper.getPartitionKey("not_exists", partition.getId());
assertNull(wrapper.get(1, key2, Metapb.Partition.parser()));
}
@Test
public void testPut() {
putToDb(partition, wrapper);
byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId());
var list = wrapper.scan(partition.getId(), Metapb.Partition.parser(), key);
assertEquals(list.size(), 1);
assertEquals(list.get(0).getGraphName(), partition.getGraphName());
}
@Test
public void testDelete() {
putToDb(partition, wrapper);
byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId());
wrapper.delete(partition.getId(), key);
var list = wrapper.scan(partition.getId(), Metapb.Partition.parser(), key);
assertEquals(list.size(), 0);
}
@Test
public void testScan() {
putToDb(partition, wrapper);
byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId());
var list = wrapper.scan(partition.getId(), Metapb.Partition.parser(), key);
assertEquals(list.size(), 1);
}
}

53
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/ZipUtilsTest.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.core.store.util;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.zip.CRC32;
import org.apache.hugegraph.rocksdb.access.util.ZipUtils;
import org.apache.hugegraph.store.UnitTestBase;
import org.apache.logging.log4j.core.util.FileUtils;
import org.junit.Before;
import org.junit.Test;
public class ZipUtilsTest {
private static final String ZIP_TEST_PATH = "/tmp/zip_util_test";
@Before
public void init() throws IOException {
UnitTestBase.deleteDir(new File(ZIP_TEST_PATH));
FileUtils.mkdir(new File(ZIP_TEST_PATH), true);
FileUtils.mkdir(new File(ZIP_TEST_PATH + "/input"), true);
FileUtils.mkdir(new File(ZIP_TEST_PATH + "/output"), true);
Files.createFile(Paths.get(ZIP_TEST_PATH + "/input/foo.txt"));
}
@Test
public void testZip() throws IOException {
ZipUtils.compress(ZIP_TEST_PATH, "input", ZIP_TEST_PATH + "/foo.zip", new CRC32());
ZipUtils.decompress(ZIP_TEST_PATH + "/foo.zip", ZIP_TEST_PATH + "/output", new CRC32());
assertTrue(Files.exists(Paths.get(ZIP_TEST_PATH + "/output/input/foo.txt")));
}
}

85
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphIDManagerTest.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import java.io.File;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.store.UnitTestBase;
import org.apache.hugegraph.store.meta.base.DBSessionBuilder;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class GraphIDManagerTest extends UnitTestBase {
@Before
public void init() {
String dbPath = "/tmp/junit";
UnitTestBase.deleteDir(new File(dbPath));
super.initDB(dbPath);
}
@Test
public void test() throws PDException {
GraphIdManager.maxGraphID = 64;
int max = GraphIdManager.maxGraphID;
try (RocksDBSession session = getDBSession("test")) {
GraphIdManager gid = new GraphIdManager(new DBSessionBuilder() {
@Override
public RocksDBSession getSession(int partId) {
return session.clone();
}
}, 0);
for (int i = 0; i < max; i++) {
Assert.assertEquals(i, gid.getCId("Test", max));
}
Assert.assertEquals(-1, gid.getCId("Test", max));
gid.delCId("Test", 3);
Assert.assertEquals(3, gid.getCId("Test", max));
Assert.assertEquals(-1, gid.getCId("Test", max));
long start = System.currentTimeMillis();
for (int i = 0; i < GraphIdManager.maxGraphID; i++) {
long id = gid.getGraphId("g" + i);
Assert.assertEquals(i, id);
}
System.out.println("time is " + (System.currentTimeMillis() - start));
{
gid.releaseGraphId("g" + 10);
long id = gid.getGraphId("g" + 10);
Assert.assertEquals(10, id);
}
start = System.currentTimeMillis();
for (int i = 0; i < GraphIdManager.maxGraphID; i++) {
long id = gid.releaseGraphId("g" + i);
Assert.assertEquals(i, id);
}
System.out.println("time is " + (System.currentTimeMillis() - start));
start = System.currentTimeMillis();
for (int i = 0; i < GraphIdManager.maxGraphID; i++) {
long id = gid.getCId(GraphIdManager.GRAPH_ID_PREFIX, GraphIdManager.maxGraphID);
// long id = gid.getGraphId("g" + i);
Assert.assertTrue(id >= 0);
}
System.out.println("time is " + (System.currentTimeMillis() - start));
}
}
}

45
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphManagerTest.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import org.junit.Assert;
// import org.junit.Test;
public class GraphManagerTest {
// @Test
public void testCloneGraph() {
Graph graph = new Graph();
graph.setGraphName("test1");
Graph graph1 = graph.clone();
Assert.assertNotSame(graph, graph1);
Assert.assertEquals(graph.getGraphName(), graph1.getGraphName());
graph1.setGraphName("test4");
Assert.assertNotEquals(graph.getGraphName(), graph1.getGraphName());
Assert.assertEquals(graph.getGraphName(), "test1");
}
}

62
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/BytesCarrierTest.java Normal file
View File

@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.raftcore;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.junit.Test;
import com.google.protobuf.BytesCarrier;
public class BytesCarrierTest {
@Test
public void testWrite() throws IOException {
byte[] bytes = new byte[]{10, 20, 30};
BytesCarrier carrier = new BytesCarrier();
// not valid
carrier.write((byte) 1);
assertNull(carrier.getValue());
assertFalse(carrier.isValid());
// not valid
ByteBuffer buffer = ByteBuffer.allocate(10);
carrier.write(buffer);
assertNull(carrier.getValue());
// not valid
carrier.writeLazy(buffer);
assertNull(carrier.getValue());
// ok, write done
carrier.write(bytes, 0, bytes.length);
assertNotNull(carrier.getValue());
assertTrue(carrier.isValid());
// has data
carrier.writeLazy(bytes, 0, bytes.length);
assertFalse(carrier.isValid());
}
}

30
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/RaftSuiteTest.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.raftcore;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@RunWith(Suite.class)
@Suite.SuiteClasses({
BytesCarrierTest.class,
ZeroByteStringHelperTest.class
})
public class RaftSuiteTest {
}

69
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/ZeroByteStringHelperTest.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.raftcore;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import org.junit.Test;
import com.google.protobuf.ZeroByteStringHelper;
public class ZeroByteStringHelperTest {
private static final String STR = "hello word!";
@Test
public void testWrap() {
byte[] b1 = new byte[]{10, 20, 30};
byte[] b2 = new byte[]{40, 50};
var h1 = ZeroByteStringHelper.wrap(b1);
var h2 = ZeroByteStringHelper.wrap(b2, 0, b2.length);
ByteBuffer buffer = ByteBuffer.allocate(5);
buffer.put(b1);
buffer.put(b2);
var h3 = ZeroByteStringHelper.wrap(buffer);
assertEquals(h3.isEmpty(), true);
var h4 = ZeroByteStringHelper.concatenate(h1, h2);
assertTrue(Arrays.equals(ZeroByteStringHelper.getByteArray(h4), buffer.array()));
}
@Test
public void testConcatenate() {
byte[] b1 = new byte[]{10, 20, 30};
byte[] b2 = new byte[]{40, 50};
ByteBuffer buffer1 = ByteBuffer.allocate(5);
buffer1.put(b1);
ByteBuffer buffer2 = ByteBuffer.allocate(5);
buffer1.put(b2);
var array = new ArrayList<ByteBuffer>();
array.add(buffer1);
array.add(buffer2);
var bs = ZeroByteStringHelper.concatenate(array);
assertEquals(bs.toByteArray().length, 5);
}
}

51
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/BaseRocksDbTest.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.rocksdb;
import java.util.HashMap;
import java.util.Map;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.config.OptionSpace;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBOptions;
import org.junit.After;
import org.junit.BeforeClass;
public class BaseRocksDbTest {
@BeforeClass
public static void init() {
OptionSpace.register("org/apache/hugegraph/store/rocksdb",
"org.apache.hugegraph.rocksdb.access.RocksDBOptions");
RocksDBOptions.instance();
Map<String, Object> configMap = new HashMap<>();
configMap.put("rocksdb.write_buffer_size", "1048576");
configMap.put("rocksdb.bloom_filter_bits_per_key", "10");
HugeConfig hConfig = new HugeConfig(configMap);
RocksDBFactory rFactory = RocksDBFactory.getInstance();
rFactory.setHugeConfig(hConfig);
}
@After
public void teardown() throws Exception {
// pass
}
}

81
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/RocksDBFactoryTest.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.rocksdb;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.SessionOperator;
import org.junit.Test;
public class RocksDBFactoryTest extends BaseRocksDbTest {
@Test
public void testCreateSession() {
RocksDBFactory factory = RocksDBFactory.getInstance();
try (RocksDBSession dbSession = factory.createGraphDB("./tmp", "test1")) {
SessionOperator op = dbSession.sessionOp();
op.prepare();
try {
op.put("tbl", "k1".getBytes(), "v1".getBytes());
op.commit();
} catch (Exception e) {
op.rollback();
}
}
factory.destroyGraphDB("test1");
}
@Test
public void testTotalKeys() {
RocksDBFactory dbFactory = RocksDBFactory.getInstance();
System.out.println(dbFactory.getTotalSize());
System.out.println(dbFactory.getTotalKey().entrySet()
.stream().map(e -> e.getValue()).reduce(0L, Long::sum));
}
@Test
public void releaseAllGraphDB() {
System.out.println(RocksDBFactory.class);
RocksDBFactory rFactory = RocksDBFactory.getInstance();
if (rFactory.queryGraphDB("bj01") == null) {
rFactory.createGraphDB("./tmp", "bj01");
}
if (rFactory.queryGraphDB("bj02") == null) {
rFactory.createGraphDB("./tmp", "bj02");
}
if (rFactory.queryGraphDB("bj03") == null) {
rFactory.createGraphDB("./tmp", "bj03");
}
RocksDBSession dbSession = rFactory.queryGraphDB("bj01");
dbSession.checkTable("test");
SessionOperator sessionOp = dbSession.sessionOp();
sessionOp.prepare();
sessionOp.put("test", "hi".getBytes(), "byebye".getBytes());
sessionOp.commit();
rFactory.releaseAllGraphDB();
}
}

32
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/RocksDbSuiteTest.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.rocksdb;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import lombok.extern.slf4j.Slf4j;
@RunWith(Suite.class)
@Suite.SuiteClasses({
RocksDBFactoryTest.class
})
@Slf4j
public class RocksDbSuiteTest {
}

33
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/BaseServerTest.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.service;
import org.junit.After;
import org.junit.BeforeClass;
public class BaseServerTest {
@BeforeClass
public static void init() {
}
@After
public void teardown() {
// pass
}
}

32
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/ServerSuiteTest.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.service;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import lombok.extern.slf4j.Slf4j;
@RunWith(Suite.class)
@Suite.SuiteClasses({
})
@Slf4j
public class ServerSuiteTest {
}

30
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/SessionTest.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.service;
import org.junit.Test;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class SessionTest {
@Test
public void testQuota() {
}
}

106
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/CopyOnWriteCacheTest.java Normal file
View File

@ -0,0 +1,106 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import com.caucho.hessian.io.Hessian2Input;
import com.caucho.hessian.io.Hessian2Output;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Map;
import java.util.Set;
import org.apache.hugegraph.store.util.CopyOnWriteCache;
import org.junit.Before;
import org.junit.Test;
public class CopyOnWriteCacheTest {
private CopyOnWriteCache<String, String> writeCache;
@Before
public void setUp() {
writeCache = new CopyOnWriteCache<>(5000L);
}
@Test
public void testContainsKey() {
// Setup
// Run the test
writeCache.put("k", "v");
final boolean result = writeCache.containsKey("k");
Map<? extends String, ? extends String> allKeys =
Map.ofEntries(Map.entry("key1", "value1"));
writeCache.putAll(allKeys);
// Verify the results
assertTrue(result);
final Set<Map.Entry<String, String>> entries = writeCache.entrySet();
Set<String> keySet = writeCache.keySet();
String val = writeCache.get("k");
boolean isEmpty = writeCache.isEmpty();
writeCache.size();
writeCache.values();
// Verify the results
assertFalse(isEmpty);
writeCache.remove("k");
writeCache.putIfAbsent("k", "v");
writeCache.replace("k", "original", "replacement");
writeCache.replace("k", "v");
writeCache.clear();
assertTrue(writeCache.isEmpty());
}
// @Test
public void testCache() throws InterruptedException {
Map<String, String> cache = new CopyOnWriteCache<>(1000);
cache.put("1", "1");
Thread.sleep(2000);
Asserts.isTrue(!cache.containsKey("1"), "cache do not clear");
}
// @Test
public void test() {
byte[] bytes;
try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
long[] l = new long[]{1, 2};
Hessian2Output output = new Hessian2Output(bos);
output.writeObject(l);
output.flush();
bytes = bos.toByteArray();
} catch (IOException e) {
throw new RuntimeException(e);
}
try (ByteArrayInputStream bis = new ByteArrayInputStream(bytes)) {
Hessian2Input input = new Hessian2Input(bis);
long[] obj = (long[]) input.readObject();
input.close();
for (long l : obj) {
System.out.println(l);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

43
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/FutureClosureTest.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import org.junit.Assert;
import org.junit.Test;
import com.alipay.sofa.jraft.Status;
public class FutureClosureTest {
@Test
public void test() {
FutureClosure closure = new FutureClosure();
new Thread(() -> {
try {
Thread.sleep(1000);
closure.run(Status.OK());
} catch (InterruptedException e) {
closure.run(new Status(-1, e.getMessage()));
}
}).start();
Assert.assertEquals(closure.get().getCode(), Status.OK().getCode());
Assert.assertEquals(closure.get().getCode(), Status.OK().getCode());
Assert.assertEquals(closure.get().getCode(), Status.OK().getCode());
}
}

486
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/HgStoreTestUtil.java Normal file
View File

@ -0,0 +1,486 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import java.io.Closeable;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.client.util.MetricX;
public class HgStoreTestUtil {
public static final String GRAPH_NAME = "default/hugegraph/g";
public static final String GRAPH_NAME2 = "default/hugegraph2/g";
public static final String TABLE_NAME = "unit-table";
public static final String TABLE_NAME2 = "unit-table-2";
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String keyPrefix) {
return batchPut(session, keyPrefix, 100);
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String keyPrefix,
int loop) {
return batchPut(session, TABLE_NAME, keyPrefix, loop);
}
public static Map<HgOwnerKey, byte[]> batchPut(
HgStoreSession session, String tableName, String keyPrefix, int loop) {
return batchPut(session, tableName, keyPrefix, loop, 1, key -> toOwnerKey(key));
}
public static Map<HgOwnerKey, byte[]> batchPut(
HgStoreSession session, String tableName, byte[] keyPrefix, int loop) {
return batchPut(session, tableName, keyPrefix, loop, 1,
(prefix, key) -> toOwnerKey(prefix, key));
}
public static Map<HgOwnerKey, byte[]> batchPut(
HgStoreSession session, String tableName, String keyPrefix, int loop, int start) {
return batchPut(session, tableName, keyPrefix, loop, start, key -> toOwnerKey(key));
}
public static Map<HgOwnerKey, byte[]> batchPut(
HgStoreSession session, String tableName, String keyPrefix, int loop,
Function<String, HgOwnerKey> f) {
return batchPut(session, tableName, keyPrefix, loop, 1, f);
}
public static Map<HgOwnerKey, byte[]> batchPut(
HgStoreSession session,
String tableName,
String keyPrefix,
int loop,
int start,
Function<String, HgOwnerKey> f) {
Map<HgOwnerKey, byte[]> res = new LinkedHashMap<>();
int length = String.valueOf(loop).length();
session.beginTx();
for (int i = start; i <= loop; i++) {
HgOwnerKey key = f.apply(keyPrefix + "-" + padLeftZeros(String.valueOf(i), length));
byte[] value = toBytes(keyPrefix + "-V-" + i);
res.put(key, value);
session.put(tableName, key, value);
if ((i + 1) % 10000 == 0) {
println("commit: " + (i + 1));
session.commit();
session.beginTx();
}
}
if (session.isTx()) {
session.commit();
}
return res;
}
public static Map<HgOwnerKey, byte[]> batchPut(
HgStoreSession session,
String tableName,
byte[] keyPrefix,
int loop,
int start,
BiFunction<byte[], String, HgOwnerKey> f) {
Map<HgOwnerKey, byte[]> res = new LinkedHashMap<>();
int length = String.valueOf(loop).length();
session.beginTx();
for (int i = start; i <= loop; i++) {
HgOwnerKey key = f.apply(keyPrefix, padLeftZeros(String.valueOf(i), length));
byte[] value = toBytes(keyPrefix + "-V-" + i);
res.put(key, value);
session.put(tableName, key, value);
if ((i + 1) % 10000 == 0) {
println("commit: " + (i + 1));
session.commit();
session.beginTx();
}
}
if (session.isTx()) {
session.commit();
}
return res;
}
/*---- common -----*/
public static void printOwner(List<HgOwnerKey> list) {
if (list == null) {
return;
}
for (HgOwnerKey entry : list) {
println(entry);
}
}
public static void printNum(List<HgKvEntry> list, String title) {
if (list == null) {
return;
}
println(title + " size: " + list.size());
}
public static int println(Iterable<HgKvIterator<HgKvEntry>> iterators) {
AtomicInteger counter = new AtomicInteger();
iterators.forEach(e -> counter.addAndGet(HgStoreTestUtil.println(e)));
return counter.get();
}
public static int println(HgKvIterator<HgKvEntry> iterator) {
if (iterator == null) {
return 0;
}
AtomicInteger counter = new AtomicInteger();
while (iterator.hasNext()) {
counter.incrementAndGet();
print(iterator.next());
println(" " + Arrays.toString(iterator.position()));
}
iterator.close();
return counter.get();
}
public static void println(HgKvIterator<HgKvEntry> iterator, Function<Integer, Boolean> mod) {
if (iterator == null) {
return;
}
int count = 0;
while (iterator.hasNext()) {
count++;
if (mod.apply(count)) {
print(iterator.next());
println(" " + Arrays.toString(iterator.position()));
}
}
iterator.close();
}
public static void println(List<HgKvEntry> list) {
if (list == null) {
return;
}
for (HgKvEntry entry : list) {
println(entry);
}
}
public static void println(List<HgKvEntry> list, int mod) {
if (list == null) {
return;
}
for (int i = 0; i < list.size(); i++) {
if (i % mod == 0) {
println(list.get(i));
}
}
}
public static void println(HgKvEntry kv) {
if (kv == null) {
System.out.println("null");
return;
}
println("[ " + kv.code() + " " + toStr(kv.key()) + " : " + toStr(kv.value()) + " ]");
}
public static void println(HgOwnerKey key) {
if (key == null) {
System.out.println("null");
return;
}
println("[ " + toInt(key.getOwner()) + " : " + toStr(key.getKey()) + " ]");
}
public static void println(String str) {
System.out.println(str);
}
public static void println(Number num) {
System.out.println(num);
}
public static void print(String str) {
System.out.print(str);
}
public static void print(HgKvEntry kv) {
if (kv == null) {
System.out.println("null");
return;
}
print("[ " + kv.code() + " " + toStr(kv.key()) + " : " + toStr(kv.value()) + " ]");
}
private static byte[] getOwner(String key) {
return getOwner(toBytes(key));
}
private static byte[] getOwner(byte[] key) {
return toBytes(Arrays.hashCode(key));
}
public static HgOwnerKey toAllPartitionKey(String key) {
return HgOwnerKey.of(HgStoreClientConst.ALL_PARTITION_OWNER, toBytes(key));
}
public static HgOwnerKey toAllPartitionKey(byte[] prefix, String key) {
return HgOwnerKey.of(HgStoreClientConst.ALL_PARTITION_OWNER, toBytes(prefix, key));
}
public static HgOwnerKey toAllPartitionKey(byte[] prefix) {
return HgOwnerKey.of(HgStoreClientConst.ALL_PARTITION_OWNER, prefix);
}
public static HgOwnerKey toOwnerKey(byte[] prefix, String key) {
byte[] newKey = toBytes(prefix, key);
return new HgOwnerKey(getOwner(newKey), newKey);
}
public static HgOwnerKey toOwnerKey(String key) {
return new HgOwnerKey(getOwner(key), toBytes(key));
}
public static HgOwnerKey toOwnerKey(byte[] key) {
return new HgOwnerKey(getOwner(key), key);
}
public static HgOwnerKey toOwnerKey(String owner, String key) {
return HgOwnerKey.of(toBytes(owner), toBytes(key));
}
public static HgOwnerKey toOwnerKey(int code, String key) {
return HgOwnerKey.of(code, toBytes(key));
}
public static String toStr(byte[] b) {
if (b == null) {
return "";
}
if (b.length == 0) {
return "";
}
return new String(b, StandardCharsets.UTF_8);
}
public static byte[] toBytes(byte[] prefix, String str) {
if (str == null) {
return null;
}
byte[] buf = str.getBytes(StandardCharsets.UTF_8);
byte[] res = new byte[buf.length + prefix.length];
System.arraycopy(prefix, 0, res, 0, prefix.length);
System.arraycopy(buf, 0, res, prefix.length, buf.length);
return res;
}
public static byte[] toBytes(String str) {
if (str == null) {
return null;
}
return str.getBytes(StandardCharsets.UTF_8);
}
public static byte[] toBytes(long l) {
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
buffer.putLong(l);
return buffer.array();
}
private static byte[] toBytes(final int i) {
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
buffer.putInt(i);
return buffer.array();
}
public static long toLong(byte[] bytes) {
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
buffer.put(bytes);
buffer.flip(); // need flip
return buffer.getLong();
}
public static int toInt(byte[] bytes) {
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
buffer.put(bytes);
buffer.flip(); // need flip
return buffer.getInt();
}
public static String padLeftZeros(String str, int n) {
return String.format("%1$" + n + "s", str).replace(' ', '0');
}
public static String toSuffix(int num, int length) {
return "-" + padLeftZeros(String.valueOf(num), length);
}
public static int amountOf(List list) {
if (list == null) {
return 0;
}
return list.size();
}
public static int amountOf(Iterator iterator) {
if (iterator == null) {
return 0;
}
int count = 0;
while (iterator.hasNext()) {
Object ignore = iterator.next();
++count;
}
return count;
}
public static int oOMAmountOf(Iterator iterator) {
if (iterator == null) {
return 0;
}
int count = 0;
while (iterator.hasNext()) {
Object obj = iterator.next();
++count;
if (count % 10000 == 0) {
println(count);
sleeping(10);
}
if (count % 100000 == 0) {
break;
}
}
if (iterator instanceof Closeable) {
try {
((Closeable) iterator).close();
} catch (Exception e) {
e.printStackTrace();
}
}
return count;
}
public static int amountIn(List<? extends Iterator> iterators) {
return iterators.stream().map(e -> HgStoreTestUtil.amountOf(e)).reduce(0, Integer::sum);
}
public static void sleeping(long time) {
try {
Thread.sleep(time);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
public static void parallelTest(int threads, Runnable runner,
Consumer<Throwable> throwableConsumer) {
int threadsAmount = threads;
CountDownLatch countDownLatch = new CountDownLatch(threadsAmount);
ExecutorService pool = new ThreadPoolExecutor(threadsAmount, threadsAmount + 20,
200, TimeUnit.SECONDS,
new ArrayBlockingQueue<>(1000));
for (int i = 0; i < threadsAmount; i++) {
pool.submit(
() -> {
try {
runner.run();
} catch (Throwable t) {
throwableConsumer.accept(t);
}
countDownLatch.countDown();
});
}
try {
countDownLatch.await();
pool.shutdown();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static void repeatedlyTest(int times, Runnable runner) {
MetricX metrics = MetricX.ofStart();
for (int j = 0; j < times; j++) {
try {
runner.run();
Thread.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
break;
} catch (Throwable t) {
metrics.countFail();
t.printStackTrace();
}
}
metrics.end();
System.out.println("*************************************************");
System.out.println("************* Repeatedly Test Completed **************");
System.out.println("Total: " + metrics.past() / 1000 + " sec.");
System.out.println("Repeated: " + times + " times.");
System.out.println("Fail: " + metrics.getFailureCount() + " times.");
System.out.println("*************************************************");
}
public static void runWaiting() {
while (true) {
sleeping(1000);
}
}
}

175
hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/UnsafeUtilTest.java Normal file
View File

@ -0,0 +1,175 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.lang.reflect.Field;
import org.apache.hugegraph.store.util.UnsafeUtf8Util;
import org.apache.hugegraph.store.util.UnsafeUtil;
import org.junit.Test;
import lombok.Data;
import sun.misc.Unsafe;
public class UnsafeUtilTest {
@Test
public void testHasUnsafe() {
assertTrue(UnsafeUtil.hasUnsafe());
}
@Test
public void testPutByte() {
UnsafeUtil.putByte("content".getBytes(), 0L, (byte) 99);
assertEquals((byte) 99, UnsafeUtil.getByte("content".getBytes(), 0L));
}
@Test
public void testMoveToString() {
assertEquals("a", UnsafeUtil.moveToString(new char[]{'a'}));
}
@Test
public void testEncodedLength() {
assertEquals(10, UnsafeUtf8Util.encodedLength("aa中文aa"));
}
@Test
public void testEncodeUtf8() {
assertEquals(10, UnsafeUtf8Util.encodeUtf8("aa中文aa", new byte[16], 0, 16));
}
@Test
public void testDecodeUtf8() {
assertEquals("co", UnsafeUtf8Util.decodeUtf8("content".getBytes(), 0, 2));
}
@Test
public void testUnsafeUtf8Util() {
String content = "content";
UnsafeUtf8Util.decodeUtf8(content.getBytes(), 0, content.length());
byte[] out = new byte[content.length()];
UnsafeUtf8Util.encodeUtf8(content, out, 0, content.length());
UnsafeUtf8Util.encodedLength(content);
}
@Test
public void testUnsafeAccessor() {
Unsafe unsafe = null;
long offset = 0;
try {
Field unsafeField = Unsafe.class.getDeclaredField("theUnsafe");
unsafeField.setAccessible(true);
unsafe = (Unsafe) unsafeField.get(null);
UnsafeUtil.UnsafeAccessor acc = new UnsafeUtil.UnsafeAccessor(unsafe);
acc.getUnsafe();
TestObject to = new TestObject();
byte byteValue = 126;
offset = acc.objectFieldOffset(TestObject.class.getDeclaredField("b"));
acc.putByte(to, offset, byteValue);
byte b = acc.getByte(to, offset);
assertEquals(byteValue, b);
short shortValue = 1;
acc.putShort(to, offset, shortValue);
short shortResult = acc.getShort(to, offset);
assertEquals(shortValue, shortResult);
int intValue = 99;
acc.putInt(to, offset, intValue);
int i = acc.getInt(to, offset);
assertEquals(intValue, i);
long longValue = 11L;
acc.putLong(to, offset, longValue);
long l = acc.getLong(to, offset);
assertEquals(longValue, l);
acc.putBoolean(to, offset, false);
assertFalse(acc.getBoolean(to, offset));
float f = 1;
acc.putFloat(to, offset, f);
float v = acc.getFloat(to, offset);
assertEquals(f, v, 0.0);
double d = 2;
acc.putDouble(to, offset, d);
double v1 = acc.getDouble(to, offset);
assertEquals(d, v1, 0.0);
TestObject o1 = new TestObject();
acc.putObject(to, offset, o1);
Object o2 = acc.getObject(to, offset);
assertEquals(o1, o2);
offset = unsafe.allocateMemory(1024);
acc.putByte(offset, byteValue);
byte bResult = acc.getByte(offset);
assertEquals(byteValue, bResult);
acc.putShort(offset, shortValue);
short aShort1 = acc.getShort(offset);
assertEquals(aShort1, shortValue);
acc.putInt(offset, intValue);
assertEquals(acc.getInt(offset), intValue);
acc.putLong(offset, longValue);
assertEquals(acc.getLong(offset), longValue);
unsafe.freeMemory(offset);
offset = acc.objectFieldOffset(TestObject.class.getDeclaredField("b"));
acc.putByteVolatile(to, offset, byteValue);
b = acc.getByteVolatile(to, offset);
assertEquals(byteValue, b);
acc.putShortVolatile(to, offset, shortValue);
shortResult = acc.getShortVolatile(to, offset);
assertEquals(shortValue, shortResult);
acc.putIntVolatile(to, offset, intValue);
i = acc.getIntVolatile(to, offset);
assertEquals(intValue, i);
acc.putLongVolatile(to, offset, longValue);
l = acc.getLongVolatile(to, offset);
assertEquals(longValue, l);
acc.putBooleanVolatile(to, offset, false);
assertFalse(acc.getBooleanVolatile(to, offset));
acc.putFloatVolatile(to, offset, f);
v = acc.getFloatVolatile(to, offset);
assertEquals(f, v, 0.0);
acc.putDoubleVolatile(to, offset, d);
v1 = acc.getDoubleVolatile(to, offset);
assertEquals(d, v1, 0.0);
acc.putObjectVolatile(to, offset, o1);
o2 = acc.getObjectVolatile(to, offset);
assertEquals(o1, o2);
} catch (Exception e) {
e.printStackTrace();
}
}
@Data
private class TestObject {
private Object o;
private byte b;
public TestObject() {
}
public TestObject(Object o, byte b) {
this.o = o;
this.b = b;
}
}
}

40
hugegraph-store/hg-store-test/src/main/resources/log4j2.xml
View File

@ -26,8 +26,8 @@
<appenders>
<Console name="console" target="SYSTEM_OUT">
<ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY" />
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n" />
<ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
<!--JsonLayout compact="false" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout-->
@ -37,24 +37,24 @@
<RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"
bufferedIO="false" immediateFlush="true">
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY" />
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n" />
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
<!--JsonLayout compact="true" eventEol="true" complete="true" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout-->
<!-- Trigger after exceeding 1day or 50MB -->
<Policies>
<SizeBasedTriggeringPolicy size="50MB" />
<TimeBasedTriggeringPolicy interval="1" modulate="true" />
<SizeBasedTriggeringPolicy size="50MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 5 files per day & auto Delete after over 2GB or 100 files -->
<DefaultRolloverStrategy max="5">
<Delete basePath="${LOG_PATH}" maxDepth="2">
<IfFileName glob="*/*.log" />
<IfFileName glob="*/*.log"/>
<!-- Limit log amount & size -->
<IfAny>
<IfAccumulatedFileSize exceeds="2GB" />
<IfAccumulatedFileCount exceeds="100" />
<IfAccumulatedFileSize exceeds="2GB"/>
<IfAccumulatedFileCount exceeds="100"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
@ -64,25 +64,25 @@
<RollingRandomAccessFile name="audit" fileName="${LOG_PATH}/audit-${FILE_NAME}.log"
filePattern="${LOG_PATH}/$${date:yyyy-MM}/audit-${FILE_NAME}-%d{yyyy-MM-dd-HH}-%i.gz"
bufferedIO="true" bufferSize="524288" immediateFlush="false">
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY" />
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
<!-- Use simple format for audit log to speed up -->
<!-- PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} - %m%n"/ -->
<JsonLayout compact="true" eventEol="true" complete="false" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}" />
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout>
<!-- Trigger after exceeding 1hour or 500MB -->
<Policies>
<SizeBasedTriggeringPolicy size="500MB" />
<TimeBasedTriggeringPolicy interval="1" modulate="true" />
<SizeBasedTriggeringPolicy size="500MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 2 files per hour & auto Delete [after 60 days] or [over 5GB or 500 files] -->
<DefaultRolloverStrategy max="2">
<Delete basePath="${LOG_PATH}" maxDepth="2">
<IfFileName glob="*/*.gz" />
<IfLastModified age="60d" />
<IfFileName glob="*/*.gz"/>
<IfLastModified age="60d"/>
<IfAny>
<IfAccumulatedFileSize exceeds="5GB" />
<IfAccumulatedFileCount exceeds="500" />
<IfAccumulatedFileSize exceeds="5GB"/>
<IfAccumulatedFileCount exceeds="500"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
@ -91,11 +91,11 @@
<loggers>
<logger name="io.grpc.netty" level="ERROR" additivity="false">
<appender-ref ref="file" />
<appender-ref ref="file"/>
</logger>
<root level="INFO">
<appender-ref ref="console" />
<appender-ref ref="file" />
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</root>
</loggers>

254
hugegraph-store/pom.xml
View File

@ -19,9 +19,7 @@
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>hugegraph-store</artifactId>
<version>${revision}</version>
<packaging>pom</packaging>
@ -77,12 +75,11 @@
<artifactId>hg-store-core</artifactId>
<version>${project.version}</version>
</dependency>
<!-- TODO: uncomment later-->
<!-- <dependency>-->
<!-- <groupId>org.apache.hugegraph</groupId>-->
<!-- <artifactId>hg-store-transfer</artifactId>-->
<!-- <version>${project.version}</version>-->
<!-- </dependency>-->
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-transfer</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
@ -155,56 +152,55 @@
</build>
<profiles>
<!-- TODO: uncomment later-->
<!-- hugegraph store test profiles -->
<!-- <profile>-->
<!-- <id>store-client-test</id>-->
<!-- <activation>-->
<!-- <activeByDefault>true</activeByDefault>-->
<!-- </activation>-->
<!-- <build>-->
<!-- <plugins>-->
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-surefire-plugin</artifactId>-->
<!-- <version>2.20</version>-->
<!-- <executions>-->
<!-- <execution>-->
<!-- <id>store-client-test</id>-->
<!-- <goals>-->
<!-- <goal>test</goal>-->
<!-- </goals>-->
<!-- <phase>test</phase>-->
<!-- </execution>-->
<!-- </executions>-->
<!-- </plugin>-->
<!-- </plugins>-->
<!-- </build>-->
<!-- </profile>-->
<!-- <profile>-->
<!-- <id>store-core-test</id>-->
<!-- <activation>-->
<!-- <activeByDefault>true</activeByDefault>-->
<!-- </activation>-->
<!-- <build>-->
<!-- <plugins>-->
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-surefire-plugin</artifactId>-->
<!-- <version>2.20</version>-->
<!-- <executions>-->
<!-- <execution>-->
<!-- <id>store-core-test</id>-->
<!-- <goals>-->
<!-- <goal>test</goal>-->
<!-- </goals>-->
<!-- <phase>test</phase>-->
<!-- </execution>-->
<!-- </executions>-->
<!-- </plugin>-->
<!-- </plugins>-->
<!-- </build>-->
<!-- </profile>-->
<profile>
<id>store-client-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>store-client-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>store-core-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>store-core-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>store-common-test</id>
<activation>
@ -229,77 +225,77 @@
</plugins>
</build>
</profile>
<!-- <profile>-->
<!-- <id>store-rocksdb-test</id>-->
<!-- <activation>-->
<!-- <activeByDefault>true</activeByDefault>-->
<!-- </activation>-->
<!-- <build>-->
<!-- <plugins>-->
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-surefire-plugin</artifactId>-->
<!-- <version>2.20</version>-->
<!-- <executions>-->
<!-- <execution>-->
<!-- <id>sit -test</id>-->
<!-- <goals>-->
<!-- <goal>test</goal>-->
<!-- </goals>-->
<!-- <phase>test</phase>-->
<!-- </execution>-->
<!-- </executions>-->
<!-- </plugin>-->
<!-- </plugins>-->
<!-- </build>-->
<!-- </profile>-->
<!-- <profile>-->
<!-- <id>store-server-test</id>-->
<!-- <activation>-->
<!-- <activeByDefault>true</activeByDefault>-->
<!-- </activation>-->
<!-- <build>-->
<!-- <plugins>-->
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-surefire-plugin</artifactId>-->
<!-- <version>2.20</version>-->
<!-- <executions>-->
<!-- <execution>-->
<!-- <id>store-server-test</id>-->
<!-- <goals>-->
<!-- <goal>test</goal>-->
<!-- </goals>-->
<!-- <phase>test</phase>-->
<!-- </execution>-->
<!-- </executions>-->
<!-- </plugin>-->
<!-- </plugins>-->
<!-- </build>-->
<!-- </profile>-->
<!-- <profile>-->
<!-- <id>store-raftcore-test</id>-->
<!-- <activation>-->
<!-- <activeByDefault>true</activeByDefault>-->
<!-- </activation>-->
<!-- <build>-->
<!-- <plugins>-->
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-surefire-plugin</artifactId>-->
<!-- <version>2.20</version>-->
<!-- <executions>-->
<!-- <execution>-->
<!-- <id>store-raftcore-test</id>-->
<!-- <goals>-->
<!-- <goal>test</goal>-->
<!-- </goals>-->
<!-- <phase>test</phase>-->
<!-- </execution>-->
<!-- </executions>-->
<!-- </plugin>-->
<!-- </plugins>-->
<!-- </build>-->
<!-- </profile>-->
<profile>
<id>store-rocksdb-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>sit -test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>store-server-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>store-server-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>store-raftcore-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>store-raftcore-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>