anph
/
hugegraph-sync
forked from hugegraph/hugegraph-sync
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(store): integrate `store-core` submodule (#2548) 

* feat(store): integrate store-core submodule
This commit is contained in:
Peng Junzhi 2024-06-07 04:25:42 -05:00 committed by GitHub
parent b056c5facd
commit ac93deebc6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
95 changed files with 14178 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/AbstractSerializer.java
View File

@ -17,6 +17,7 @@
package org.apache.hugegraph.backend.serializer;
import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.backend.BackendException;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.query.ConditionQuery;
@ -24,7 +25,9 @@ import org.apache.hugegraph.backend.query.IdQuery;
import org.apache.hugegraph.backend.query.Query;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.iterator.CIter;
import org.apache.hugegraph.type.HugeType;
import org.apache.tinkerpop.gremlin.structure.Edge;
public abstract class AbstractSerializer
implements GraphSerializer, SchemaSerializer {
@ -89,4 +92,8 @@ public abstract class AbstractSerializer
return query;
}
public CIter<Edge> readEdges(HugeGraph graph, BackendEntry bytesEntry) {
throw new RuntimeException("Method not implemented error.");
}
}

37
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java
View File

@ -40,6 +40,8 @@ import org.apache.hugegraph.backend.serializer.BinaryBackendEntry.BinaryId;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.backend.store.BackendEntry.BackendColumn;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.iterator.CIter;
import org.apache.hugegraph.iterator.MapperIterator;
import org.apache.hugegraph.schema.EdgeLabel;
import org.apache.hugegraph.schema.IndexLabel;
import org.apache.hugegraph.schema.PropertyKey;
@ -69,6 +71,7 @@ import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.JsonUtil;
import org.apache.hugegraph.util.NumericUtil;
import org.apache.hugegraph.util.StringEncoding;
import org.apache.tinkerpop.gremlin.structure.Edge;
public class BinarySerializer extends AbstractSerializer {
@ -524,6 +527,40 @@ public class BinarySerializer extends AbstractSerializer {
return edges.iterator().next();
}
@Override
public CIter<Edge> readEdges(HugeGraph graph, BackendEntry bytesEntry) {
BinaryBackendEntry entry = this.convertEntry(bytesEntry);
// Parse id
Id id = entry.id().origin();
Id vid = id.edge() ? ((EdgeId) id).ownerVertexId() : id;
HugeVertex vertex = new HugeVertex(graph, vid, VertexLabel.NONE);
// Parse all properties and edges of a Vertex
Iterator<BackendColumn> iterator = entry.columns().iterator();
for (int index = 0; iterator.hasNext(); index++) {
BackendColumn col = iterator.next();
if (entry.type().isEdge()) {
// NOTE: the entry id type is vertex even if entry type is edge
// Parse vertex edges
this.parseColumn(col, vertex);
} else {
assert entry.type().isVertex();
// Parse vertex properties
assert entry.columnsSize() >= 1 : entry.columnsSize();
if (index == 0) {
this.parseVertex(col.value, vertex);
} else {
this.parseVertexOlap(col.value, vertex);
}
}
}
// convert to CIter
return new MapperIterator<>(vertex.getEdges().iterator(),
(edge) -> edge);
}
@Override
public BackendEntry writeIndex(HugeIndex index) {
BinaryBackendEntry entry;

5
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java
View File

@ -115,6 +115,10 @@ public final class BytesBuffer extends OutputStream {
return new BytesBuffer(ByteBuffer.wrap(array, offset, length));
}
public static byte getType(int value) {
return (byte) (value & 0x3f);
}
public ByteBuffer asByteBuffer() {
return this.buffer;
}
@ -792,6 +796,7 @@ public final class BytesBuffer extends OutputStream {
/**
* 解析 olap id
*
* @param type
* @param isOlap
* @return

4
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/GraphSerializer.java
View File

@ -22,12 +22,14 @@ import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.query.ConditionQuery;
import org.apache.hugegraph.backend.query.Query;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.iterator.CIter;
import org.apache.hugegraph.structure.HugeEdge;
import org.apache.hugegraph.structure.HugeEdgeProperty;
import org.apache.hugegraph.structure.HugeIndex;
import org.apache.hugegraph.structure.HugeVertex;
import org.apache.hugegraph.structure.HugeVertexProperty;
import org.apache.hugegraph.type.HugeType;
import org.apache.tinkerpop.gremlin.structure.Edge;
public interface GraphSerializer {
@ -45,6 +47,8 @@ public interface GraphSerializer {
HugeEdge readEdge(HugeGraph graph, BackendEntry entry);
CIter<Edge> readEdges(HugeGraph graph, BackendEntry bytesEntry);
BackendEntry writeIndex(HugeIndex index);
HugeIndex readIndex(HugeGraph graph, ConditionQuery query, BackendEntry entry);

8
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java
View File

@ -39,6 +39,7 @@ import org.apache.hugegraph.backend.query.IdRangeQuery;
import org.apache.hugegraph.backend.query.Query;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.iterator.CIter;
import org.apache.hugegraph.schema.EdgeLabel;
import org.apache.hugegraph.schema.IndexLabel;
import org.apache.hugegraph.schema.PropertyKey;
@ -65,6 +66,7 @@ import org.apache.hugegraph.type.define.SchemaStatus;
import org.apache.hugegraph.type.define.WriteType;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.JsonUtil;
import org.apache.tinkerpop.gremlin.structure.Edge;
import com.google.common.collect.ImmutableMap;
@ -352,6 +354,12 @@ public class TextSerializer extends AbstractSerializer {
throw new NotImplementedException("Unsupported readEdge()");
}
public CIter<Edge> readEdges(HugeGraph graph, BackendEntry bytesEntry) {
E.checkNotNull(graph, "serializer graph");
// TODO: implement
throw new NotImplementedException("Unsupported readEdges()");
}
@Override
public BackendEntry writeIndex(HugeIndex index) {
TextBackendEntry entry = newBackendEntry(index.type(), index.id());

173
hugegraph-store/hg-store-core/pom.xml Normal file
View File

@ -0,0 +1,173 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-store</artifactId>
<version>${revision}</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>hg-store-core</artifactId>
<dependencies>
<!-- generic -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.24</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20210307</version>
</dependency>
<dependency>
<groupId>com.alipay.sofa</groupId>
<artifactId>jraft-core</artifactId>
<version>1.3.13</version>
<exclusions>
<exclusion>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
</exclusion>
</exclusions>
</dependency>
<!--
For Mac M1: UnsatisfiedLinkError for M1 Macs while running hstore
default version 5.5.0 from jraft-core:1.3.9-SNAPSHOT dependencies
refer: https://stackoverflow.com/questions/70368863/unsatisfiedlinkerror-for-m1-macs-while-running-play-server-locally
-->
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>5.7.0</version>
</dependency>
<dependency>
<groupId>io.protostuff</groupId>
<artifactId>protostuff-core</artifactId>
<version>1.6.0</version>
</dependency>
<dependency>
<groupId>io.protostuff</groupId>
<artifactId>protostuff-runtime</artifactId>
<version>1.6.0</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.83</version>
</dependency>
<!-- generic end-->
<!-- module -->
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-rocksdb</artifactId>
<version>${revision}</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-pd-client</artifactId>
<version>${revision}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-core</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-common</artifactId>
<version>${revision}</version>
</dependency>
<!-- tinkerpop -->
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-core</artifactId>
<version>3.5.1</version>
<exclusions>
<exclusion>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</exclusion>
<exclusion>
<artifactId>commons-lang3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-configuration2</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-text</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-groovy</artifactId>
<version>3.5.1</version>
<exclusions>
<exclusion>
<groupId>com.github.jeremyh</groupId>
<artifactId>jBCrypt</artifactId>
</exclusion>
<exclusion>
<artifactId>commons-lang3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.9</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-grpc</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-client</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

384
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/HeartbeatService.java Normal file
View File

@ -0,0 +1,384 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.pd.grpc.Pdpb;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.meta.PartitionRole;
import org.apache.hugegraph.store.meta.Store;
import org.apache.hugegraph.store.meta.StoreMetadata;
import org.apache.hugegraph.store.options.HgStoreEngineOptions;
import org.apache.hugegraph.store.options.RaftRocksdbOptions;
import org.apache.hugegraph.store.pd.PdProvider;
import org.apache.hugegraph.store.util.IpUtil;
import org.apache.hugegraph.store.util.Lifecycle;
import org.rocksdb.MemoryUsageType;
import com.alipay.sofa.jraft.entity.PeerId;
import com.alipay.sofa.jraft.util.Utils;
import lombok.extern.slf4j.Slf4j;
/**
* Register and heartbeat, Keep the system online
*/
@Slf4j
public class HeartbeatService implements Lifecycle<HgStoreEngineOptions>, PartitionStateListener {
private static final int MAX_HEARTBEAT_RETRY_COUNT = 5; // 心跳重试次数
private static final int REGISTER_RETRY_INTERVAL = 1; //注册重试时间间隔，单位秒
private final HgStoreEngine storeEngine;
private final List<HgStoreStateListener> stateListeners;
private final Object partitionThreadLock = new Object();
private final Object storeThreadLock = new Object();
private HgStoreEngineOptions options;
private PdProvider pdProvider;
private Store storeInfo;
private Metapb.ClusterStats clusterStats;
private StoreMetadata storeMetadata;
// 心跳失败次数
private int heartbeatFailCount = 0;
private int reportErrCount = 0;
// 线程休眠时间
private volatile int timerNextDelay = 1000;
private boolean terminated = false;
public HeartbeatService(HgStoreEngine storeEngine) {
this.storeEngine = storeEngine;
stateListeners = Collections.synchronizedList(new ArrayList());
}
@Override
public boolean init(HgStoreEngineOptions opts) {
this.options = opts;
storeInfo = storeMetadata.getStore();
if (storeInfo == null) {
storeInfo = new Store();
}
storeInfo.setStoreAddress(options.getGrpcAddress());
storeInfo.setPdAddress(options.getPdAddress());
storeInfo.setRaftAddress(options.getRaftAddress());
storeInfo.setState(Metapb.StoreState.Unknown);
storeInfo.setLabels(options.getLabels());
storeInfo.setCores(Runtime.getRuntime().availableProcessors());
storeInfo.setDeployPath(HeartbeatService.class.getResource("/").getPath());
storeInfo.setDataPath(options.getDataPath());
this.pdProvider = options.getPdProvider();
new Thread(new Runnable() {
@Override
public void run() {
doStoreHeartbeat();
}
}, "heartbeat").start();
new Thread(new Runnable() {
@Override
public void run() {
doPartitionHeartbeat();
}
}, " partition-hb").start();
return true;
}
public HeartbeatService addStateListener(HgStoreStateListener stateListener) {
stateListeners.add(stateListener);
return this;
}
public Store getStoreInfo() {
return storeInfo;
}
public void setStoreMetadata(StoreMetadata storeMetadata) {
this.storeMetadata = storeMetadata;
}
// 集群是否准备就绪
public boolean isClusterReady() {
return clusterStats.getState() == Metapb.ClusterState.Cluster_OK;
}
/**
* 服务状态有四种
* 就绪，在线、离线、死亡（从集群排除）
*/
protected void doStoreHeartbeat() {
while (!terminated) {
try {
switch (storeInfo.getState()) {
case Unknown:
case Offline:
registerStore();
break;
case Up:
storeHeartbeat();
monitorMemory();
break;
case Tombstone:
break;
}
synchronized (storeThreadLock) {
storeThreadLock.wait(timerNextDelay);
}
} catch (Throwable e) {
log.error("heartbeat error: ", e);
}
}
}
protected void doPartitionHeartbeat() {
while (!terminated) {
try {
partitionHeartbeat();
} catch (Exception e) {
log.error("doPartitionHeartbeat error: ", e);
}
try {
synchronized (partitionThreadLock) {
partitionThreadLock.wait(options.getPartitionHBInterval() * 1000L);
}
} catch (InterruptedException e) {
log.error("doPartitionHeartbeat error: ", e);
}
}
}
protected void registerStore() {
try {
// 注册 store，初次注册 PD 产生 id，自动给 storeinfo 赋值
this.storeInfo.setStoreAddress(IpUtil.getNearestAddress(options.getGrpcAddress()));
this.storeInfo.setRaftAddress(IpUtil.getNearestAddress(options.getRaftAddress()));
long storeId = pdProvider.registerStore(this.storeInfo);
if (storeId != 0) {
storeInfo.setId(storeId);
storeMetadata.save(storeInfo);
this.clusterStats = pdProvider.getClusterStats();
if (clusterStats.getState() == Metapb.ClusterState.Cluster_OK) {
timerNextDelay = options.getStoreHBInterval() * 1000;
} else {
timerNextDelay = REGISTER_RETRY_INTERVAL * 1000;
}
log.info("Register Store id= {} successfully. store = {}, clusterStats {}",
storeInfo.getId(), storeInfo, this.clusterStats);
// 监听 partition 消息
pdProvider.startHeartbeatStream(error -> {
onStateChanged(Metapb.StoreState.Offline);
timerNextDelay = REGISTER_RETRY_INTERVAL * 1000;
wakeupHeartbeatThread();
log.error("Connection closed. The store state changes to {}",
Metapb.StoreState.Offline);
});
onStateChanged(Metapb.StoreState.Up);
} else {
timerNextDelay = REGISTER_RETRY_INTERVAL * 1000 / 2;
}
} catch (PDException e) {
int exceptCode = e.getErrorCode();
if (exceptCode == Pdpb.ErrorType.STORE_ID_NOT_EXIST_VALUE) {
log.error(
"The store ID {} does not match the PD. Check that the correct PD is " +
"connected, " +
"and then delete the store ID!!!",
storeInfo.getId());
System.exit(-1);
} else if (exceptCode == Pdpb.ErrorType.STORE_HAS_BEEN_REMOVED_VALUE) {
log.error("The store ID {} has been removed, please delete all data and restart!",
storeInfo.getId());
System.exit(-1);
} else if (exceptCode == Pdpb.ErrorType.STORE_PROHIBIT_DUPLICATE_VALUE) {
log.error(
"The store ID {} maybe duplicated, please check out store raft address " +
"and restart later!",
storeInfo.getId());
System.exit(-1);
}
}
}
protected void storeHeartbeat() {
if (log.isDebugEnabled()) {
log.debug("storeHeartbeat ... ");
}
Metapb.ClusterStats clusterStats = null;
try {
clusterStats = pdProvider.storeHeartbeat(this.storeInfo);
} catch (PDException e) {
int exceptCode = e.getErrorCode();
if (exceptCode == Pdpb.ErrorType.STORE_ID_NOT_EXIST_VALUE) {
log.error("The store ID {} does not match the PD. Check that the correct PD is " +
"connected, and then delete the store ID!!!", storeInfo.getId());
System.exit(-1);
} else if (exceptCode == Pdpb.ErrorType.STORE_HAS_BEEN_REMOVED_VALUE) {
log.error("The store ID {} has been removed, please delete all data and restart!",
storeInfo.getId());
System.exit(-1);
}
}
if (clusterStats.getState().getNumber() >= Metapb.ClusterState.Cluster_Fault.getNumber()) {
if (reportErrCount == 0) {
log.info("The cluster is abnormal, {}", clusterStats);
}
reportErrCount = (++reportErrCount) % 30;
}
if (clusterStats.getState() == Metapb.ClusterState.Cluster_OK) {
timerNextDelay = options.getStoreHBInterval() * 1000;
} else {
timerNextDelay = REGISTER_RETRY_INTERVAL * 1000;
}
if (clusterStats.getState() == Metapb.ClusterState.Cluster_Fault) {
heartbeatFailCount++;
} else {
heartbeatFailCount = 0;
this.clusterStats = clusterStats;
}
if (heartbeatFailCount > MAX_HEARTBEAT_RETRY_COUNT) {
onStateChanged(Metapb.StoreState.Offline);
timerNextDelay = REGISTER_RETRY_INTERVAL * 1000;
this.clusterStats = clusterStats;
log.error("Store heart beat failure. The store state changes to {}",
Metapb.StoreState.Offline);
}
}
protected synchronized void onStateChanged(Metapb.StoreState newState) {
Utils.runInThread(() -> {
Metapb.StoreState oldState = this.storeInfo.getState();
this.storeInfo.setState(newState);
stateListeners.forEach((e) ->
e.stateChanged(this.storeInfo, oldState, newState));
});
}
protected void partitionHeartbeat() {
if (storeEngine == null) {
return;
}
List<PartitionEngine> partitions = storeEngine.getLeaderPartition();
final List<Metapb.PartitionStats> statsList = new ArrayList<>(partitions.size());
Metapb.Shard localLeader = Metapb.Shard.newBuilder()
.setStoreId(
storeEngine.getPartitionManager().getStore()
.getId())
.setRole(Metapb.ShardRole.Leader)
.build();
// 获取各个 shard 信息。
for (PartitionEngine partition : partitions) {
Metapb.PartitionStats.Builder stats = Metapb.PartitionStats.newBuilder();
stats.setId(partition.getGroupId());
stats.addAllGraphName(partition.getPartitions().keySet());
stats.setLeaderTerm(partition.getLeaderTerm());
stats.setConfVer(partition.getShardGroup().getConfVersion());
stats.setLeader(localLeader);
stats.addAllShard(partition.getShardGroup().getMetaPbShard());
// shard 状态
List<Metapb.ShardStats> shardStats = new ArrayList<>();
Map<Long, PeerId> aliveShards = partition.getAlivePeers();
// 统计 shard 状态
partition.getShardGroup().getShards().forEach(shard -> {
Metapb.ShardState state = Metapb.ShardState.SState_Normal;
if (!aliveShards.containsKey(shard.getStoreId())) {
state = Metapb.ShardState.SState_Offline;
}
shardStats.add(Metapb.ShardStats.newBuilder()
.setStoreId(shard.getStoreId())
.setRole(shard.getRole())
.setState(state).build());
});
stats.addAllShardStats(shardStats);
stats.setTimestamp(System.currentTimeMillis());
statsList.add(stats.build());
}
// 发送心跳
if (statsList.size() > 0) {
pdProvider.partitionHeartbeat(statsList);
}
}
public void monitorMemory() {
try {
Map<MemoryUsageType, Long> mems =
storeEngine.getBusinessHandler().getApproximateMemoryUsageByType(null);
if (mems.get(MemoryUsageType.kCacheTotal) >
RaftRocksdbOptions.getWriteCacheCapacity() * 0.9 &&
mems.get(MemoryUsageType.kMemTableUnFlushed) >
RaftRocksdbOptions.getWriteCacheCapacity() * 0.1) {
// storeEngine.getBusinessHandler().flushAll();
log.warn("Less memory, start flush dbs, {}", mems);
}
} catch (Exception e) {
log.error("MonitorMemory exception {}", e);
}
}
@Override
public void shutdown() {
log.info("HeartbeatService shutdown");
terminated = true;
synchronized (partitionThreadLock) {
partitionThreadLock.notify();
}
}
@Override
public void partitionRoleChanged(Partition partition, PartitionRole newRole) {
if (newRole == PartitionRole.LEADER) {
// leader 发生改变，激活心跳
synchronized (partitionThreadLock) {
partitionThreadLock.notifyAll();
}
}
}
@Override
public void partitionShardChanged(Partition partition, List<Metapb.Shard> oldShards,
List<Metapb.Shard> newShards) {
if (partition.isLeader()) {
synchronized (partitionThreadLock) {
partitionThreadLock.notifyAll();
}
}
}
private void wakeupHeartbeatThread() {
synchronized (storeThreadLock) {
storeThreadLock.notifyAll();
}
}
}

726
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/HgStoreEngine.java Normal file
View File

@ -0,0 +1,726 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.store.business.BusinessHandler;
import org.apache.hugegraph.store.business.BusinessHandlerImpl;
import org.apache.hugegraph.store.business.DataMover;
import org.apache.hugegraph.store.cmd.HgCmdClient;
import org.apache.hugegraph.store.cmd.HgCmdProcessor;
import org.apache.hugegraph.store.cmd.UpdatePartitionRequest;
import org.apache.hugegraph.store.cmd.UpdatePartitionResponse;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.meta.PartitionManager;
import org.apache.hugegraph.store.meta.ShardGroup;
import org.apache.hugegraph.store.meta.Store;
import org.apache.hugegraph.store.metric.HgMetricService;
import org.apache.hugegraph.store.options.HgStoreEngineOptions;
import org.apache.hugegraph.store.options.PartitionEngineOptions;
import org.apache.hugegraph.store.pd.DefaultPdProvider;
import org.apache.hugegraph.store.pd.FakePdServiceProvider;
import org.apache.hugegraph.store.pd.PdProvider;
import org.apache.hugegraph.store.raft.RaftClosure;
import org.apache.hugegraph.store.raft.RaftOperation;
import org.apache.hugegraph.store.util.HgRaftError;
import org.apache.hugegraph.store.util.Lifecycle;
import com.alipay.sofa.jraft.JRaftUtils;
import com.alipay.sofa.jraft.Status;
import com.alipay.sofa.jraft.conf.Configuration;
import com.alipay.sofa.jraft.core.NodeMetrics;
import com.alipay.sofa.jraft.entity.PeerId;
import com.alipay.sofa.jraft.option.RpcOptions;
import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
import com.alipay.sofa.jraft.rpc.RpcServer;
import com.alipay.sofa.jraft.util.Endpoint;
import lombok.extern.slf4j.Slf4j;
/**
* The core class of the storage engine, initializing PD client and raft client
*/
@Slf4j
public class HgStoreEngine implements Lifecycle<HgStoreEngineOptions>, HgStoreStateListener {
private final static HgStoreEngine instance = new HgStoreEngine();
private static ConcurrentHashMap<Integer, Object> engineLocks = new ConcurrentHashMap<>();
// 分区raft引擎，key为GraphName_PartitionID
private final Map<Integer, PartitionEngine> partitionEngines = new ConcurrentHashMap<>();
private RpcServer rpcServer;
private HgStoreEngineOptions options;
private PdProvider pdProvider;
private HgCmdClient hgCmdClient;
private PartitionManager partitionManager;
private HeartbeatService heartbeatService;
private BusinessHandler businessHandler;
private HgMetricService metricService;
private DataMover dataMover;
public static HgStoreEngine getInstance() {
return instance;
}
/**
* 1、读取StoreId，向pd注册，初次注册由PD生成StoreId，存储到本地
* 2、注册成功，启动raft服务
* 3、定时发送Store心跳和Partition心跳，与PD保持联系
*
* @param opts
* @return
*/
@Override
public synchronized boolean init(final HgStoreEngineOptions opts) {
if (rpcServer != null) {
log.info("HgStoreEngine already started.");
return true;
}
this.options = opts;
BusinessHandlerImpl.initRocksdb(opts.getRocksdbConfig(), getRocksdbListener());
if (opts.isFakePD()) {
pdProvider = new FakePdServiceProvider(opts.getFakePdOptions());
} else {
pdProvider = new DefaultPdProvider(opts.getPdAddress());
pdProvider.addPartitionInstructionListener(new PartitionInstructionProcessor(this));
}
options.setPdProvider(pdProvider);
partitionManager = new PartitionManager(pdProvider, opts);
partitionManager.addPartitionChangedListener(new PartitionChangedListener());
businessHandler = new BusinessHandlerImpl(partitionManager);
// 需要businessHandler 初始化后
partitionManager.load();
rpcServer = createRaftRpcServer(opts.getRaftAddress());
hgCmdClient = new HgCmdClient();
hgCmdClient.init(new RpcOptions(), (graphName, ptId) -> {
// 分裂的时候，还未及时的上报pd
if (getPartitionEngine(ptId) != null) {
return getPartitionEngine(ptId).waitForLeader(
options.getWaitLeaderTimeout() * 1000);
} else {
// 可能出现跨分区的迁移
Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, ptId);
return JRaftUtils.getEndPoint(
pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress());
}
});
heartbeatService = new HeartbeatService(this);
heartbeatService.setStoreMetadata(partitionManager.getStoreMetadata());
heartbeatService.addStateListener(this).init(options);
metricService = HgMetricService.getInstance();
metricService.setHgStoreEngine(this).init(null);
dataMover = opts.getDataTransfer();
if (dataMover != null) {
this.dataMover.setBusinessHandler(this.businessHandler);
this.dataMover.setCmdClient(hgCmdClient);
}
return true;
}
/**
* 创建raft rpc server，用于store之间通讯
*/
private RpcServer createRaftRpcServer(String raftAddr) {
Endpoint endpoint = JRaftUtils.getEndPoint(raftAddr);
RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(endpoint,
JRaftUtils.createExecutor(
"RAFT-RPC-",
options.getRaftRpcThreadPoolSize()),
null);
HgCmdProcessor.registerProcessor(rpcServer, this);
rpcServer.init(null);
return rpcServer;
}
@Override
public void shutdown() {
if (rpcServer == null) {
return;
}
partitionEngines.forEach((k, v) -> {
v.shutdown();
});
partitionEngines.clear();
rpcServer.shutdown();
// HgStoreEngine.init function check rpcServer whether is null, skipped if the instance
// exists even shut down.
rpcServer = null;
heartbeatService.shutdown();
metricService.shutdown();
// close all db session
RocksDBFactory.getInstance().releaseAllGraphDB();
}
public void snapshotForTest() {
partitionEngines.forEach((k, v) -> {
v.snapshot();
});
}
/**
* Store注册状态发生改变
*/
@Override
public void stateChanged(Store store, Metapb.StoreState oldState, Metapb.StoreState newState) {
log.info("stateChanged, oldState {}, newState {}", oldState, newState);
if (newState == Metapb.StoreState.Up) {
// 状态变为上线，记录store信息
partitionManager.setStore(store);
partitionManager.loadPartition();
restoreLocalPartitionEngine();
}
}
/**
* 恢复本地的PartitionEngine，恢复PD返回的分区信息
* 1、需要检查本次保存的分区，删除作废的分区
*/
public void restoreLocalPartitionEngine() {
try {
if (!options.isFakePD()) { // FakePD模式不需要同步
partitionManager.syncPartitionsFromPD(partition -> {
log.warn(
"The local partition information is inconsistent with the PD server. " +
"Please delete the redundant data manually, {}", partition);
});
}
partitionManager.getPartitions().forEach((k, g) -> {
g.forEach((id, p) -> {
try {
createPartitionEngine(p);
} catch (Exception e) {
log.error("Partition {}-{} restore exception {}", p.getGraphName(),
p.getId(), e);
}
});
});
} catch (PDException e) {
log.error("HgStoreEngine restoreLocalPartitionEngine error {}", e);
}
}
/**
* 收到store raft addr 变更，需要重新创建raft group
*
* @param storeId 变更的store id
*/
public void rebuildRaftGroup(long storeId) {
partitionEngines.forEach((partId, engine) -> {
try {
var partitions = pdProvider.getPartitionsByStore(storeId);
if (partitions.size() > 0) {
var shards = pdProvider.getShardGroup(partId).getShardsList();
if (shards.stream().anyMatch(s -> s.getStoreId() == storeId)) {
var peers = partitionManager.shards2Peers(shards);
Configuration initConf = engine.getOptions().getConf();
if (initConf == null) {
engine.getOptions().setPeerList(peers);
} else {
peers.stream()
.forEach(peer -> initConf.addPeer(JRaftUtils.getPeerId(peer)));
}
// engine.getOptions().getConf().setPeers();
engine.restartRaftNode();
}
}
} catch (PDException e) {
log.error("rebuild raft group error: {}", e.getMessage());
}
});
}
/**
* 创建 raft Node
*
* @param partition
* @return
*/
public PartitionEngine createPartitionEngine(Partition partition) {
return createPartitionEngine(partition, null);
}
public PartitionEngine createPartitionEngine(Partition partition, Configuration conf) {
partitionManager.updatePartition(partition, false);
var shardGroup = partitionManager.getShardGroup(partition.getId());
return createPartitionEngine(partition.getId(), shardGroup, conf);
}
private PartitionEngine createPartitionEngine(int groupId, ShardGroup shardGroup,
Configuration conf) {
PartitionEngine engine;
if ((engine = partitionEngines.get(groupId)) == null) {
engineLocks.computeIfAbsent(groupId, k -> new Object());
synchronized (engineLocks.get(groupId)) {
// 分区分裂时特殊情况(集群中图分区数量不一样)，会导致分裂的分区，可能不在本机器上.
if (conf != null) {
var list = conf.listPeers();
list.addAll(conf.listLearners());
if (!list.stream().anyMatch(
p -> p.getEndpoint().toString().equals(options.getRaftAddress()))) {
log.info(
"raft {}, conf {} does not contains raft address:{}, skipped " +
"create partition engine",
groupId, conf, options.getRaftAddress());
return null;
}
} else {
var storeId = partitionManager.getStore().getId();
if (!shardGroup.getShards().stream().anyMatch(s -> s.getStoreId() == storeId)) {
log.info("raft {}, shard group {} does not contains current storeId {}, " +
"skipped create partition engine", groupId, shardGroup, storeId);
return null;
}
}
if ((engine = partitionEngines.get(groupId)) == null) {
log.info("createPartitionEngine {}, with shards: {}", groupId, shardGroup);
engine = new PartitionEngine(this, shardGroup);
PartitionEngineOptions ptOpts = new PartitionEngineOptions();
if (conf != null) {
ptOpts.setConf(conf);
} else {
ptOpts.setPeerList(partitionManager.getPartitionPeers(shardGroup));
}
ptOpts.setGroupId(groupId);
ptOpts.setRaftAddress(options.getRaftAddress());
ptOpts.setRaftDataPath(partitionManager.getRaftDataPath(groupId));
ptOpts.setRaftSnapShotPath(partitionManager.getRaftSnapShotPath(groupId));
ptOpts.setRaftOptions(options.getRaftOptions());
// raft任务处理器
ptOpts.setTaskHandler(options.getTaskHandler());
// 分区状态监听
engine.addStateListener(this.heartbeatService);
engine.init(ptOpts);
partitionEngines.put(ptOpts.getGroupId(), engine);
}
}
}
// 检查是否活跃，如果不活跃，则重新创建
engine.checkActivity();
return engine;
}
/**
* 创建 raft分组，除了创建本地raft node，还要通知其他peer创建raft node
* 1、遍历partition.shards
* 2、根据storeId获取Store信息
* 3、建立向其他store的raft rpc，发送StartRaft消息
*
* @param partition
* @return
*/
public PartitionEngine createPartitionGroups(Partition partition) {
PartitionEngine engine = partitionEngines.get(partition.getId());
if (engine == null) {
engine = createPartitionEngine(partition);
if (engine == null) {
return null;
}
var shardGroup = partitionManager.getShardGroup(partition.getId());
if (shardGroup != null) {
// raft不存在，通知follower创建raft
shardGroup.getShards().forEach((shard) -> {
Store store = partitionManager.getStore(shard.getStoreId());
if (store == null || partitionManager.isLocalStore(store)) {
return;
}
// 向其他peer发消息，创建raft 分组。此处是异步发送
hgCmdClient.createRaftNode(store.getRaftAddress(), List.of(partition),
status -> {
log.info(
"send to {} createRaftNode rpc call " +
"result {} partitionId {}",
store.getRaftAddress(), status,
partition.getId());
});
});
}
} else {
// raft存在，修改分区列表，通过raft同步给follower
engine = createPartitionEngine(partition);
}
return engine;
}
public void destroyPartitionGroups(Partition partition) {
var shardGroup = partitionManager.getShardGroup(partition.getId());
if (shardGroup != null) {
shardGroup.getShards().forEach((shard) -> {
Store store = partitionManager.getStore(shard.getStoreId());
if (store == null) {
return;
}
// 向其他peer发消息，创建raft 分组。此处是异步发送
hgCmdClient.destroyRaftNode(store.getRaftAddress(),
Arrays.asList(new Partition[]{partition}),
status -> {
log.info(
"send to {} - {} DestroyRaftNode rpc call" +
" result {}",
store.getRaftAddress(), partition.getId(),
status);
});
});
}
}
/**
* 停止分区，并销毁数据
*/
public synchronized void destroyPartitionEngine(Integer groupId, List<String> graphNames) {
log.info("Partition {} start to be destroyed", groupId);
if (!partitionEngines.containsKey(groupId)) {
return;
}
PartitionEngine ptEngine = partitionEngines.get(groupId);
graphNames.forEach(graphName -> {
ptEngine.removePartition(graphName);
// 删除数据
businessHandler.deletePartition(graphName, groupId);
});
if (ptEngine.getPartitions().size() == 0) {
ptEngine.destroy();
partitionEngines.remove(groupId);
// 删除对应的db文件夹
businessHandler.destroyGraphDB(graphNames.get(0), groupId);
} else {
graphNames.forEach(graphName -> {
businessHandler.dbCompaction(graphName, groupId);
});
}
log.info("Partition {} has been destroyed", groupId);
}
/**
* 删除图数据，删除本地数据，并删除PD上的分区信息
*/
public void deletePartition(Integer groupId, String graphName) {
log.info("Partition {}-{} deletePartition", graphName, groupId);
if (!partitionEngines.containsKey(groupId)) {
return;
}
PartitionEngine ptEngine = partitionEngines.get(groupId);
ptEngine.removePartition(graphName);
// 删除数据
businessHandler.deletePartition(graphName, groupId);
//通知PD删除分区数据
if (ptEngine.isLeader()) {
synchronized (this) {
partitionManager.deletePartition(graphName, groupId);
}
}
}
/**
* 获取所有的leader分区
*
* @return
*/
public List<PartitionEngine> getLeaderPartition() {
List<PartitionEngine> partitions = new ArrayList<>();
this.partitionEngines.forEach((k, v) -> {
if (v.isLeader()) {
partitions.add(v);
}
});
return partitions;
}
/**
* 获取分区所有活跃的peer
*
* @return
*/
public Map<Long, PeerId> getAlivePeers(int groupId) {
PartitionEngine engine = this.partitionEngines.get(groupId);
try {
if (engine != null) {
return engine.getAlivePeers();
}
} catch (Exception e) {
log.error("getAlivePeers {}", e);
}
return new HashMap<>();
}
/**
* 获取分区的最后提交的日志id
*
* @param groupId
* @return
*/
public long getLeaderTerm(int groupId) {
PartitionEngine engine = this.partitionEngines.get(groupId);
return engine.getLeaderTerm();
}
public long getCommittedIndex(int groupId) {
PartitionEngine engine = this.partitionEngines.get(groupId);
if (engine != null) {
return engine.getCommittedIndex();
}
return 0;
}
public RpcServer getRaftRpcServer() {
return rpcServer;
}
public PartitionManager getPartitionManager() {
return partitionManager;
}
// For test
public void setPartitionManager(PartitionManager ptm) {
this.partitionManager = ptm;
}
public DataMover getDataMover() {
return dataMover;
}
public PdProvider getPdProvider() {
return pdProvider;
}
public BusinessHandler getBusinessHandler() {
return businessHandler;
}
public HgCmdClient getHgCmdClient() {
return hgCmdClient;
}
public HeartbeatService getHeartbeatService() {
return heartbeatService;
}
public boolean isClusterReady() {
return heartbeatService.isClusterReady();
}
public List<String> getDataLocations() {
return partitionManager.getStoreMetadata().getDataLocations();
}
/**
* 添加raft任务
* 1、检查partition是否存在
* 1.1、如果不存在，则向PD查询分区是否属于本地
* 1.1.1 如果分区属于本地，则创建raft分组,并通知其他Store
* 1.1.2 如果分区不属于本地，则抛出异常
* 1.2 检查Partition是否是leader
* 1.2.1 如果是leader，则提交任务
* 1.2.2 否则，返回错误
*
* @param partId
* @param operation
*/
public void addRaftTask(String graphName, Integer partId, RaftOperation operation,
RaftClosure closure) {
PartitionEngine engine = getPartitionEngine(graphName, partId);
if (engine == null) {
engineLocks.computeIfAbsent(partId, k -> new Object());
synchronized (engineLocks.get(partId)) {
engine = getPartitionEngine(graphName, partId);
if (engine == null) {
Partition partition = partitionManager.findPartition(graphName, partId);
if (partition != null) {
engine = this.createPartitionGroups(partition);
// 可能迁移，不应该创建, 放到 synchronize体中，避免后面的
if (engine != null) {
engine.waitForLeader(options.getWaitLeaderTimeout() * 1000);
}
}
}
}
}
if (engine != null) {
// 等待Leader
Endpoint leader = engine.waitForLeader(options.getWaitLeaderTimeout() * 1000);
if (engine.isLeader()) {
engine.addRaftTask(operation, closure);
} else if (leader != null) {
// 当前不是leader，返回leader所在的storeId
Store store = partitionManager.getStoreByRaftEndpoint(engine.getShardGroup(),
leader.toString());
if (store.getId() == 0) {
// 本地未找到Leader的Store信息，可能Partition还未同步过来，重新向Leader获取。
Store leaderStore = hgCmdClient.getStoreInfo(leader.toString());
store = leaderStore != null ? leaderStore : store;
log.error("getStoreByRaftEndpoint error store:{}, shard: {}, leader is {}",
store, engine.getShardGroup().toString(), leader);
}
// Leader 不是本机，通知客户端
closure.onLeaderChanged(partId, store.getId());
closure.run(new Status(HgRaftError.NOT_LEADER.getNumber(),
String.format("Partition %s-%d leader changed to %x",
graphName, partId, store.getId())));
log.error("Raft Partition {}-{} not leader, redirectTo leader {}.", graphName,
partId, leader);
} else {
closure.run(new Status(HgRaftError.WAIT_LEADER_TIMEOUT.getNumber(),
HgRaftError.WAIT_LEADER_TIMEOUT.getMsg()));
log.error("Partition {}-{} waiting for leader timeout.", graphName, partId);
}
} else {
closure.run(
new Status(HgRaftError.NOT_LOCAL.getNumber(), HgRaftError.NOT_LOCAL.getMsg()));
log.error("Partition {}-{} does not belong to local store.", graphName, partId);
}
}
public PartitionEngine getPartitionEngine(Integer partitionId) {
PartitionEngine engine = partitionEngines.get(partitionId);
return engine;
}
public PartitionEngine getPartitionEngine(String graphName, Integer partitionId) {
PartitionEngine engine = partitionEngines.get(partitionId);
if (engine != null && engine.hasPartition(graphName)) {
return engine;
}
return null;
}
public Map<Integer, PartitionEngine> getPartitionEngines() {
return partitionEngines;
}
public Map<String, NodeMetrics> getNodeMetrics() {
Map<String, NodeMetrics> metrics = new HashMap();
partitionEngines.forEach((k, v) -> {
metrics.put(Integer.toString(k), v.getNodeMetrics());
});
return metrics;
}
/**
* Number of raft-group.
*
* @return
*/
public int getRaftGroupCount() {
return partitionEngines.size();
}
/**
* 监听rocksdb事件
*
* @return
*/
private RocksDBFactory.RocksdbChangedListener getRocksdbListener() {
return new RocksDBFactory.RocksdbChangedListener() {
@Override
public void onCompacted(String dbName) {
String sid = dbName.substring(dbName.lastIndexOf("/") + 1);
try {
Integer groupId = Integer.parseInt(sid);
PartitionEngine engine = getPartitionEngine(groupId);
if (engine != null) {
engine.addBlankRaftTask();
}
} catch (Exception e) {
}
}
};
}
class PartitionChangedListener implements PartitionManager.PartitionChangedListener {
/**
* Partition对象发生改变，leader通知到其他的follower
*/
@Override
public void onChanged(Partition partition) {
PartitionEngine engine = getPartitionEngine(partition.getId());
if (engine != null && engine.isLeader()) {
try {
engine.addRaftTask(RaftOperation.create(RaftOperation.SYNC_PARTITION,
partition.getProtoObj()),
new RaftClosure() {
@Override
public void run(Status status) {
log.info(
"Partition {}-{}-{} sync partition status " +
"is {}",
partition.getGraphName(), partition.getId(),
partition.getWorkState(),
status);
}
});
} catch (IOException e) {
log.error("Partition {}-{} sync partition exception {}",
partition.getGraphName(), partition.getId(), e);
}
}
}
/**
* Partition对象key范围、状态发生改变，通过主动寻找leader再通知到其他的follower
*/
@Override
public UpdatePartitionResponse rangeOrStateChanged(UpdatePartitionRequest request) {
UpdatePartitionResponse response = null;
try {
response = hgCmdClient.raftUpdatePartition(request);
log.info("not leader request threadId:{} pId:{} range:{}-{} state:{} response:{}",
Thread.currentThread().getId(), request.getPartitionId(),
request.getStartKey(),
request.getEndKey(), request.getWorkState(), response.getStatus());
} catch (Exception e) {
e.printStackTrace();
}
return response;
}
}
}

26
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/HgStoreStateListener.java Normal file
View File

@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.meta.Store;
public interface HgStoreStateListener {
void stateChanged(Store store, Metapb.StoreState oldState, Metapb.StoreState newState);
}

1262
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionEngine.java Normal file
View File

File diff suppressed because it is too large Load Diff

345
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionInstructionProcessor.java Normal file
View File

@ -0,0 +1,345 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.MetaTask;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.pd.grpc.pulse.ChangeShard;
import org.apache.hugegraph.pd.grpc.pulse.CleanPartition;
import org.apache.hugegraph.pd.grpc.pulse.DbCompaction;
import org.apache.hugegraph.pd.grpc.pulse.MovePartition;
import org.apache.hugegraph.pd.grpc.pulse.PartitionKeyRange;
import org.apache.hugegraph.pd.grpc.pulse.SplitPartition;
import org.apache.hugegraph.pd.grpc.pulse.TransferLeader;
import org.apache.hugegraph.store.cmd.CleanDataRequest;
import org.apache.hugegraph.store.cmd.DbCompactionRequest;
import org.apache.hugegraph.store.meta.MetadataKeyHelper;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.pd.PartitionInstructionListener;
import org.apache.hugegraph.store.raft.RaftClosure;
import org.apache.hugegraph.store.raft.RaftOperation;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
import com.alipay.sofa.jraft.Status;
import com.alipay.sofa.jraft.util.Utils;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* PD发给Store的分区指令处理器
*/
public class PartitionInstructionProcessor implements PartitionInstructionListener {
private static final Logger LOG = Log.logger(PartitionInstructionProcessor.class);
private final HgStoreEngine storeEngine;
private final ExecutorService threadPool;
public PartitionInstructionProcessor(HgStoreEngine storeEngine) {
this.storeEngine = storeEngine;
ThreadFactory namedThreadFactory =
new ThreadFactoryBuilder().setNameFormat("instruct-process-pool-%d").build();
threadPool = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(),
1000000,
180L,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(1000000),
namedThreadFactory,
new ThreadPoolExecutor.AbortPolicy());
}
@Override
public void onChangeShard(long taskId, Partition partition, ChangeShard changeShard,
Consumer<Integer> consumer) {
PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId());
if (engine != null) {
// 清理所有的任务，有失败的情况
engine.getTaskManager()
.deleteTask(partition.getId(), MetaTask.TaskType.Change_Shard.name());
}
if (engine != null && engine.isLeader()) {
LOG.info("Partition {}-{} Receive change shard message, {}", partition.getGraphName(),
partition.getId(), changeShard);
String graphName = partition.getGraphName();
int partitionId = partition.getId();
MetaTask.Task task = MetaTask.Task.newBuilder()
.setId(taskId)
.setPartition(partition.getProtoObj())
.setType(MetaTask.TaskType.Change_Shard)
.setState(MetaTask.TaskState.Task_Ready)
.setChangeShard(changeShard)
.build();
try {
storeEngine.addRaftTask(graphName, partitionId,
RaftOperation.create(RaftOperation.SYNC_PARTITION_TASK,
task),
new RaftClosure() {
@Override
public void run(Status status) {
LOG.info(
"Partition {}-{} onChangeShard complete, " +
"status is {}",
graphName, partitionId, status);
consumer.accept(0);
}
});
} catch (Exception e) {
LOG.error("Partition {}-{} onSplitPartition exception {}",
graphName, partitionId, e);
}
}
}
@Override
public void onTransferLeader(long taskId, Partition partition, TransferLeader transferLeader,
Consumer<Integer> consumer) {
PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId());
if (engine != null && engine.isLeader()) {
consumer.accept(0);
Utils.runInThread(() -> {
LOG.info("Partition {}-{} receive TransferLeader instruction, new leader is {}"
, partition.getGraphName(), partition.getId(), transferLeader.getShard());
engine.transferLeader(partition.getGraphName(), transferLeader.getShard());
});
}
}
/**
* Leader接收到PD发送的分区分裂任务
* 添加到raft任务队列，由raft进行任务分发。
*/
@Override
public void onSplitPartition(long taskId, Partition partition, SplitPartition splitPartition,
Consumer<Integer> consumer) {
PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId());
if (preCheckTaskId(taskId, partition.getId())) {
return;
}
if (engine != null && engine.isLeader()) {
// 先应答，避免超时造成pd重复发送
consumer.accept(0);
String graphName = partition.getGraphName();
int partitionId = partition.getId();
MetaTask.Task task = MetaTask.Task.newBuilder()
.setId(taskId)
.setPartition(partition.getProtoObj())
.setType(MetaTask.TaskType.Split_Partition)
.setState(MetaTask.TaskState.Task_Ready)
.setSplitPartition(splitPartition)
.build();
try {
threadPool.submit(() -> {
engine.moveData(task);
});
} catch (Exception e) {
LOG.error("Partition {}-{} onSplitPartition exception {}",
graphName, partitionId, e);
}
}
}
/**
* Leader接收到PD发送的rocksdb compaction任务
* 添加到raft任务队列，由raft进行任务分发。
*/
@Override
public void onDbCompaction(long taskId, Partition partition, DbCompaction dbCompaction,
Consumer<Integer> consumer) {
PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId());
if (engine != null && engine.isLeader()) {
try {
DbCompactionRequest dbCompactionRequest = new DbCompactionRequest();
dbCompactionRequest.setPartitionId(partition.getId());
dbCompactionRequest.setTableName(dbCompaction.getTableName());
dbCompactionRequest.setGraphName(partition.getGraphName());
engine.addRaftTask(RaftOperation.create(RaftOperation.DB_COMPACTION,
dbCompactionRequest),
new RaftClosure() {
@Override
public void run(Status status) {
LOG.info(
"onRocksdbCompaction {}-{} sync partition " +
"status is {}",
partition.getGraphName(), partition.getId(),
status);
}
}
);
} finally {
consumer.accept(0);
}
}
}
@Override
public void onMovePartition(long taskId, Partition partition, MovePartition movePartition,
Consumer<Integer> consumer) {
PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId());
if (preCheckTaskId(taskId, partition.getId())) {
return;
}
if (engine != null && engine.isLeader()) {
// 先应答，避免超时造成pd重复发送
consumer.accept(0);
String graphName = partition.getGraphName();
int partitionId = partition.getId();
MetaTask.Task task = MetaTask.Task.newBuilder()
.setId(taskId)
.setPartition(partition.getProtoObj())
.setType(MetaTask.TaskType.Move_Partition)
.setState(MetaTask.TaskState.Task_Ready)
.setMovePartition(movePartition)
.build();
try {
threadPool.submit(() -> {
engine.moveData(task);
});
} catch (Exception e) {
LOG.error("Partition {}-{} onMovePartition exception {}",
graphName, partitionId, e);
}
}
}
@Override
public void onCleanPartition(long taskId, Partition partition, CleanPartition cleanPartition,
Consumer<Integer> consumer) {
if (preCheckTaskId(taskId, partition.getId())) {
return;
}
PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId());
if (engine != null && engine.isLeader()) {
consumer.accept(0);
CleanDataRequest
request =
CleanDataRequest.fromCleanPartitionTask(cleanPartition, partition, taskId);
storeEngine.addRaftTask(partition.getGraphName(), partition.getId(),
RaftOperation.create(RaftOperation.IN_CLEAN_OP, request),
status -> {
LOG.info(
"onCleanPartition {}-{}, cleanType: {}, " +
"range:{}-{}, status:{}",
partition.getGraphName(),
partition.getId(),
cleanPartition.getCleanType(),
cleanPartition.getKeyStart(),
cleanPartition.getKeyEnd(),
status);
});
}
}
@Override
public void onPartitionKeyRangeChanged(long taskId, Partition partition,
PartitionKeyRange partitionKeyRange,
Consumer<Integer> consumer) {
PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId());
if (engine != null && engine.isLeader()) {
consumer.accept(0);
var partitionManager = storeEngine.getPartitionManager();
var localPartition =
partitionManager.getPartition(partition.getGraphName(), partition.getId());
if (localPartition == null) {
// 如果分区数据为空，本地不会存储
localPartition = partitionManager.getPartitionFromPD(partition.getGraphName(),
partition.getId());
LOG.info("onPartitionKeyRangeChanged, get from pd:{}-{} -> {}",
partition.getGraphName(), partition.getId(), localPartition);
if (localPartition == null) {
return;
}
}
var newPartition = localPartition.getProtoObj().toBuilder()
.setStartKey(partitionKeyRange.getKeyStart())
.setEndKey(partitionKeyRange.getKeyEnd())
.setState(Metapb.PartitionState.PState_Normal)
.build();
partitionManager.updatePartition(newPartition, true);
try {
engine.addRaftTask(RaftOperation.create(RaftOperation.SYNC_PARTITION, newPartition),
status -> {
LOG.info(
"onPartitionKeyRangeChanged, {}-{},key range: " +
"{}-{} status{}",
newPartition.getGraphName(),
newPartition.getId(),
partitionKeyRange.getKeyStart(),
partitionKeyRange.getKeyEnd(),
status);
});
LOG.info("onPartitionKeyRangeChanged: {}, update to pd", newPartition);
partitionManager.updatePartitionToPD(List.of(newPartition));
} catch (IOException e) {
LOG.error("Partition {}-{} onPartitionKeyRangeChanged exception {}",
newPartition.getGraphName(), newPartition.getId(), e);
} catch (PDException e) {
throw new RuntimeException(e);
}
}
}
/**
* is the task exists
*
* @param taskId task id
* @param partId partition id
* @return true if exists, false otherwise
*/
private boolean preCheckTaskId(long taskId, int partId) {
if (storeEngine.getPartitionEngine(partId) == null) {
return false;
}
byte[] key = MetadataKeyHelper.getInstructionIdKey(taskId);
var wrapper = storeEngine.getPartitionManager().getWrapper();
byte[] value = wrapper.get(partId, key);
if (value != null) {
return true;
}
wrapper.put(partId, key, new byte[0]);
return false;
}
}

34
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionStateListener.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store;
import java.util.List;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.meta.PartitionRole;
public interface PartitionStateListener {
// 分区角色发生改变
void partitionRoleChanged(Partition partition, PartitionRole newRole);
// 分区发生改变
void partitionShardChanged(Partition partition, List<Metapb.Shard> oldShards,
List<Metapb.Shard> newShards);
}

67
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/AbstractSelectIterator.java Normal file
View File

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.business;
import org.apache.hugegraph.backend.serializer.AbstractSerializer;
import org.apache.hugegraph.backend.serializer.BinarySerializer;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.iterator.CIter;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.structure.HugeElement;
import org.apache.hugegraph.util.Bytes;
import org.apache.tinkerpop.gremlin.structure.Edge;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractSelectIterator implements ScanIterator {
protected ScanIterator iterator;
protected AbstractSerializer serializer;
public AbstractSelectIterator() {
this.serializer = new BinarySerializer();
}
public boolean belongToMe(BackendEntry entry,
BackendEntry.BackendColumn column) {
return Bytes.prefixWith(column.name, entry.id().asBytes());
}
public HugeElement parseEntry(BackendEntry entry, boolean isVertex) {
try {
if (isVertex) {
return this.serializer.readVertex(null, entry);
} else {
CIter<Edge> itr =
this.serializer.readEdges(null, entry);
// Iterator<HugeEdge> itr = this.serializer.readEdges(
// null, entry, true, false).iterator();
HugeElement el = null;
if (itr.hasNext()) {
el = (HugeElement) itr.next();
}
return el;
}
} catch (Exception e) {
log.error("Failed to parse entry: {}", entry, e);
throw e;
}
}
}

218
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandler.java Normal file
View File

@ -0,0 +1,218 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.business;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Supplier;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.pd.grpc.pulse.CleanType;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.grpc.Graphpb;
import org.apache.hugegraph.store.grpc.common.Key;
import org.apache.hugegraph.store.grpc.common.OpType;
import org.apache.hugegraph.store.grpc.session.BatchEntry;
import org.apache.hugegraph.store.meta.base.DBSessionBuilder;
import org.apache.hugegraph.store.metric.HgStoreMetric;
import org.apache.hugegraph.store.raft.HgStoreStateMachine;
import org.apache.hugegraph.store.term.HgPair;
import org.apache.hugegraph.store.util.HgStoreException;
import org.rocksdb.Cache;
import org.rocksdb.MemoryUsageType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public interface BusinessHandler extends DBSessionBuilder {
Logger log = LoggerFactory.getLogger(HgStoreStateMachine.class);
String tableUnknown = "unknown";
String tableVertex = "g+v";
String tableOutEdge = "g+oe";
String tableInEdge = "g+ie";
String tableIndex = "g+index";
String tableTask = "g+task";
String tableOlap = "g+olap";
String tableServer = "g+server";
String[] tables = new String[]{tableUnknown, tableVertex, tableOutEdge, tableInEdge, tableIndex,
tableTask, tableOlap, tableServer};
void doPut(String graph, int code, String table, byte[] key, byte[] value) throws
HgStoreException;
byte[] doGet(String graph, int code, String table, byte[] key) throws HgStoreException;
ScanIterator scanAll(String graph, String table) throws HgStoreException;
ScanIterator scanAll(String graph, String table, byte[] query) throws HgStoreException;
ScanIterator scan(String graph, String table, int codeFrom, int codeTo) throws HgStoreException;
ScanIterator scan(String graph, int code, String table, byte[] start, byte[] end,
int scanType) throws HgStoreException;
ScanIterator scan(String graph, int code, String table, byte[] start, byte[] end, int scanType,
byte[] conditionQuery) throws HgStoreException;
<T> GraphStoreIterator<T> scan(Graphpb.ScanPartitionRequest request);
ScanIterator scanOriginal(Graphpb.ScanPartitionRequest request);
ScanIterator scanPrefix(String graph, int code, String table, byte[] prefix,
int scanType) throws HgStoreException;
ScanIterator scanPrefix(String graph, int code, String table, byte[] prefix) throws
HgStoreException;
HgStoreMetric.Partition getPartitionMetric(String graph, int partId,
boolean accurateCount) throws HgStoreException;
void batchGet(String graph, String table, Supplier<HgPair<Integer, byte[]>> s,
Consumer<HgPair<byte[], byte[]>> c) throws HgStoreException;
void truncate(String graph, int partId) throws HgStoreException;
void flushAll();
void closeAll();
//
Map<MemoryUsageType, Long> getApproximateMemoryUsageByType(List<Cache> caches);
List<Integer> getLeaderPartitionIds(String graph);
HgStoreMetric.Graph getGraphMetric(String graph, int partId);
void saveSnapshot(String snapshotPath, String graph, int partId) throws HgStoreException;
void loadSnapshot(String snapshotPath, String graph, int partId, long version) throws
HgStoreException;
long getLatestSequenceNumber(String graph, int partId);
// 扫描分区从 seqnum 开始的 kv
ScanIterator scanRaw(String graph, int partId, long seqNum) throws HgStoreException;
void ingestSstFile(String graph, int partId, Map<byte[], List<String>> sstFiles) throws
HgStoreException;
//提交分区分裂，删除旧数据
// 删除分区数据
boolean deletePartition(String graph, int partId);
//清理分区，删除多余的数据
boolean cleanPartition(String graph, int partId);
boolean cleanPartition(String graph, int partId, long startKey, long endKey,
CleanType cleanType);
//所有指定分区图的所有 table 名
List<String> getTableNames(String graph, int partId);
TxBuilder txBuilder(String graph, int partId);
default void doBatch(String graph, int partId, List<BatchEntry> entryList) {
BusinessHandler.TxBuilder builder = txBuilder(graph, partId);
try {
for (BatchEntry b : entryList) {
Key start = b.getStartKey();
String table = tables[b.getTable()];
byte[] startKey = start.getKey().toByteArray();
int number = b.getOpType().getNumber();
if (number == OpType.OP_TYPE_PUT_VALUE) {
builder.put(start.getCode(), table, startKey, b.getValue().toByteArray());
} else {
switch (number) {
case OpType.OP_TYPE_DEL_VALUE:
builder.del(start.getCode(), table, startKey);
continue;
case OpType.OP_TYPE_DEL_PREFIX_VALUE:
builder.delPrefix(start.getCode(), table, startKey);
continue;
case OpType.OP_TYPE_DEL_RANGE_VALUE:
builder.delRange(start.getCode(), table, startKey,
b.getEndKey().getKey().toByteArray());
continue;
case OpType.OP_TYPE_DEL_SINGLE_VALUE:
builder.delSingle(start.getCode(), table, startKey);
continue;
case OpType.OP_TYPE_MERGE_VALUE:
builder.merge(start.getCode(), table, startKey,
b.getValue().toByteArray());
continue;
default:
throw new IllegalArgumentException(
"unsupported batch-op-type: " + b.getOpType().name());
}
}
}
builder.build().commit();
} catch (Throwable e) {
String msg =
String.format("graph data %s-%s do batch insert with error:", graph, partId);
log.error(msg, e);
builder.build().rollback();
throw e;
}
}
boolean existsTable(String graph, int partId, String table);
void createTable(String graph, int partId, String table);
void deleteTable(String graph, int partId, String table);
void dropTable(String graph, int partId, String table);
boolean dbCompaction(String graphName, int partitionId);
boolean dbCompaction(String graphName, int partitionId, String tableName);
void destroyGraphDB(String graphName, int partId) throws HgStoreException;
long count(String graphName, String table);
@NotThreadSafe
interface TxBuilder {
TxBuilder put(int code, String table, byte[] key, byte[] value) throws HgStoreException;
TxBuilder del(int code, String table, byte[] key) throws HgStoreException;
TxBuilder delSingle(int code, String table, byte[] key) throws HgStoreException;
TxBuilder delPrefix(int code, String table, byte[] prefix) throws HgStoreException;
TxBuilder delRange(int code, String table, byte[] start, byte[] end) throws
HgStoreException;
TxBuilder merge(int code, String table, byte[] key, byte[] value) throws HgStoreException;
Tx build();
}
interface Tx {
void commit() throws HgStoreException;
void rollback() throws HgStoreException;
}
}

922
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java Normal file
View File

@ -0,0 +1,922 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.business;
import static org.apache.hugegraph.store.util.HgStoreConst.EMPTY_BYTES;
import static org.apache.hugegraph.store.util.HgStoreConst.SCAN_ALL_PARTITIONS_ID;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.commons.configuration2.MapConfiguration;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.config.OptionSpace;
import org.apache.hugegraph.pd.grpc.pulse.CleanType;
import org.apache.hugegraph.rocksdb.access.DBStoreException;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener;
import org.apache.hugegraph.rocksdb.access.RocksDBOptions;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.SessionOperator;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.cmd.CleanDataRequest;
import org.apache.hugegraph.store.grpc.Graphpb.ScanPartitionRequest;
import org.apache.hugegraph.store.grpc.Graphpb.ScanPartitionRequest.Request;
import org.apache.hugegraph.store.grpc.Graphpb.ScanPartitionRequest.ScanType;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.meta.PartitionManager;
import org.apache.hugegraph.store.meta.asynctask.AsyncTaskState;
import org.apache.hugegraph.store.meta.asynctask.CleanTask;
import org.apache.hugegraph.store.metric.HgStoreMetric;
import org.apache.hugegraph.store.pd.PdProvider;
import org.apache.hugegraph.store.term.Bits;
import org.apache.hugegraph.store.term.HgPair;
import org.apache.hugegraph.store.util.HgStoreException;
import org.rocksdb.Cache;
import org.rocksdb.MemoryUsageType;
import com.alipay.sofa.jraft.util.Utils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class BusinessHandlerImpl implements BusinessHandler {
private static final int batchSize = 10000;
private static final RocksDBFactory factory = RocksDBFactory.getInstance();
private static final HashMap<ScanType, String> tableMapping = new HashMap<>() {{
put(ScanType.SCAN_VERTEX, tableVertex);
put(ScanType.SCAN_EDGE, tableOutEdge);
}};
private static final Map<Integer, String> dbNames = new ConcurrentHashMap<>();
static {
int code = tableUnknown.hashCode();
code = tableVertex.hashCode();
code = tableOutEdge.hashCode();
code = tableInEdge.hashCode();
code = tableIndex.hashCode();
code = tableTask.hashCode();
code = tableTask.hashCode();
log.debug("init table code:{}", code);
}
private final PartitionManager partitionManager;
private final PdProvider provider;
private final InnerKeyCreator keyCreator;
public BusinessHandlerImpl(PartitionManager partitionManager) {
this.partitionManager = partitionManager;
this.provider = partitionManager.getPdProvider();
this.keyCreator = new InnerKeyCreator(this);
factory.addRocksdbChangedListener(new RocksdbChangedListener() {
@Override
public void onDBDeleteBegin(String dbName, String filePath) {
partitionManager.getDeletedFileManager().addDeletedFile(filePath);
}
@Override
public void onDBDeleted(String dbName, String filePath) {
partitionManager.getDeletedFileManager().removeDeletedFile(filePath);
}
@Override
public void onDBSessionReleased(RocksDBSession dbSession) {
}
});
}
public static HugeConfig initRocksdb(Map<String, Object> rocksdbConfig,
RocksdbChangedListener listener) {
// 注册 rocksdb 配置
OptionSpace.register("rocksdb", "org.apache.hugegraph.rocksdb.access.RocksDBOptions");
RocksDBOptions.instance();
HugeConfig hConfig = new HugeConfig(new MapConfiguration(rocksdbConfig));
factory.setHugeConfig(hConfig);
if (listener != null) {
factory.addRocksdbChangedListener(listener);
}
return hConfig;
}
public static String getDbName(int partId) {
String dbName = dbNames.get(partId);
if (dbName == null) {
dbName = String.format("%05d", partId);
dbNames.put(partId, dbName);
}
// 每个分区对应一个 rocksdb 实例，因此 rocksdb 实例名为 partId
return dbName;
}
@Override
public void doPut(String graph, int code, String table, byte[] key, byte[] value) throws
HgStoreException {
int partId = provider.getPartitionByCode(graph, code).getId();
try (RocksDBSession dbSession = getSession(graph, table, partId)) {
SessionOperator op = dbSession.sessionOp();
try {
op.prepare();
byte[] targetKey = keyCreator.getKey(partId, graph, code, key);
op.put(table, targetKey, value);
op.commit();
} catch (Exception e) {
log.error("Graph " + graph + " doPut exception", e);
op.rollback();
throw new HgStoreException(HgStoreException.EC_RKDB_DOPUT_FAIL, e.toString());
}
}
}
@Override
public byte[] doGet(String graph, int code, String table, byte[] key) throws HgStoreException {
int partId = provider.getPartitionByCode(graph, code).getId();
try (RocksDBSession dbSession = getSession(graph, table, partId)) {
byte[] targetKey = keyCreator.getKey(partId, graph, code, key);
return dbSession.sessionOp().get(table, targetKey);
} catch (Exception e) {
log.error("Graph " + graph + " doGet exception", e);
throw new HgStoreException(HgStoreException.EC_RKDB_DOGET_FAIL, e.toString());
}
}
@Override
public ScanIterator scanAll(String graph, String table) throws HgStoreException {
List<Integer> ids = this.getLeaderPartitionIds(graph);
BiFunction<Integer, byte[], ScanIterator> function = (id, position) -> {
try (RocksDBSession dbSession = getSession(graph, table, id)) {
return new InnerKeyFilter(dbSession.sessionOp().scan(table, position == null ?
keyCreator.getStartKey(
id, graph) :
keyCreator.getStartKey(
id, graph,
position),
keyCreator.getEndKey(id,
graph),
ScanIterator.Trait.SCAN_LT_END));
}
};
return MultiPartitionIterator.of(ids, function);
}
@Override
public ScanIterator scanAll(String graph, String table, byte[] query) throws HgStoreException {
return scanAll(graph, table);
}
@Override
public ScanIterator scan(String graph, int code, String table, byte[] start, byte[] end,
int scanType) throws HgStoreException {
List<Integer> ids;
if (code == SCAN_ALL_PARTITIONS_ID) {
ids = this.getLeaderPartitionIds(graph);
} else {
ids = new ArrayList<>();
ids.add(partitionManager.getPartitionIdByCode(graph, code));
}
BiFunction<Integer, byte[], ScanIterator> function = (id, position) -> {
byte[] endKey;
int type;
if (ArrayUtils.isEmpty(end)) {
endKey = keyCreator.getEndKey(id, graph);
type = ScanIterator.Trait.SCAN_LT_END;
} else {
endKey = keyCreator.getEndKey(id, graph, end);
type = scanType;
}
try (RocksDBSession dbSession = getSession(graph, table, id)) {
return new InnerKeyFilter(dbSession.sessionOp().scan(table,
keyCreator.getStartKey(id,
graph,
toPosition(
start,
position)),
endKey, type));
}
};
return MultiPartitionIterator.of(ids, function);
}
/**
* 根据 keyCode 范围返回数据，左闭右开
*
* @param graph
* @param table
* @param codeFrom 起始 code，包含该值
* @param codeTo 结束 code，不包含该值
* @return
* @throws HgStoreException
*/
@Override
public ScanIterator scan(String graph, String table, int codeFrom, int codeTo) throws
HgStoreException {
List<Integer> ids = new ArrayList<>();
ids.add(partitionManager.getPartitionIdByCode(graph, codeFrom));
BiFunction<Integer, byte[], ScanIterator> function = (id, position) -> {
try (RocksDBSession dbSession = getSession(graph, table, id)) {
byte[] startKey;
if (position != null) {
startKey = keyCreator.getStartKey(id, graph, position);
} else {
startKey = keyCreator.getStartKey(id, graph);
}
byte[] endKey = keyCreator.getEndKey(id, graph);
ScanIterator iterator = dbSession.sessionOp().scan(table, startKey, endKey,
ScanIterator.Trait.SCAN_LT_END);
return new InnerKeyFilter(iterator, codeFrom, codeTo);
}
};
return MultiPartitionIterator.of(ids, function);
}
@Override
public ScanIterator scan(String graph, int code, String table, byte[] start, byte[] end,
int scanType, byte[] conditionQuery) throws HgStoreException {
ScanIterator it = null;
if ((scanType & ScanIterator.Trait.SCAN_HASHCODE) == ScanIterator.Trait.SCAN_HASHCODE) {
int codeFrom = Bits.toInt(start);
int codeTo = Bits.toInt(end);
it = scan(graph, table, codeFrom, codeTo);
} else {
it = scan(graph, code, table, start, end, scanType);
}
return it;
}
@Override
public GraphStoreIterator scan(ScanPartitionRequest spr) throws HgStoreException {
return new GraphStoreIterator(scanOriginal(spr), spr);
}
@Override
public ScanIterator scanOriginal(ScanPartitionRequest spr) throws HgStoreException {
Request request = spr.getScanRequest();
String graph = request.getGraphName();
List<Integer> ids;
int partitionId = request.getPartitionId();
int startCode = request.getStartCode();
int endCode = request.getEndCode();
if (partitionId == SCAN_ALL_PARTITIONS_ID) {
ids = this.getLeaderPartitionIds(graph);
} else {
ids = new ArrayList<>();
if (startCode != 0 || endCode != 0) {
ids.add(partitionManager.getPartitionIdByCode(graph, startCode));
} else {
ids.add(partitionId);
}
}
String table = request.getTable();
if (StringUtils.isEmpty(table)) {
table = tableMapping.get(request.getScanType());
}
int scanType = request.getBoundary();
if (scanType == 0) {
scanType = ScanIterator.Trait.SCAN_LT_END;
}
String tab = table;
int st = scanType;
BiFunction<Integer, byte[], ScanIterator> func = (id, position) -> {
try (RocksDBSession dbSession = getSession(graph, tab, id)) {
byte[] startPos = toPosition(EMPTY_BYTES, position);
byte[] startKey = keyCreator.getStartKey(id, graph, startPos);
byte[] endKey = keyCreator.getEndKey(id, graph);
ScanIterator iter = dbSession.sessionOp().scan(tab, startKey, endKey, st);
return new InnerKeyFilter(iter);
}
};
return MultiPartitionIterator.of(ids, func);
}
@Override
public ScanIterator scanPrefix(String graph, int code, String table, byte[] prefix,
int scanType) throws HgStoreException {
List<Integer> ids;
if (code == SCAN_ALL_PARTITIONS_ID) {
ids = this.getLeaderPartitionIds(graph);
} else {
ids = new ArrayList<>();
ids.add(partitionManager.getPartitionIdByCode(graph, code));
}
BiFunction<Integer, byte[], ScanIterator> function = (id, position) -> {
try (RocksDBSession dbSession = getSession(graph, table, id)) {
return new InnerKeyFilter(dbSession.sessionOp().scan(table,
keyCreator.getPrefixKey(id,
graph,
toPosition(
prefix,
position)),
scanType));
}
};
return MultiPartitionIterator.of(ids, function);
}
@Override
public ScanIterator scanPrefix(String graph, int code, String table, byte[] prefix) throws
HgStoreException {
return scanPrefix(graph, code, table, prefix, 0);
}
private byte[] toPosition(byte[] start, byte[] position) {
if (position == null || position.length == 0) {
return start;
}
return position;
}
@Override
public HgStoreMetric.Partition getPartitionMetric(String graph, int partId,
boolean accurateCount) throws
HgStoreException {
// get key count
Map<String, Long> countMap = null;
Map<String, String> sizeMap = null;
try (RocksDBSession dbSession = getSession(graph, partId)) {
countMap = dbSession.getKeyCountPerCF(keyCreator.getStartKey(partId, graph),
keyCreator.getEndKey(partId, graph),
accurateCount);
sizeMap = dbSession.getApproximateCFDataSize(keyCreator.getStartKey(partId, graph),
keyCreator.getEndKey(partId, graph));
HgStoreMetric.Partition partMetric = new HgStoreMetric.Partition();
partMetric.setPartitionId(partId);
List<HgStoreMetric.Table> tables = new ArrayList<>(sizeMap.size());
for (String tableName : sizeMap.keySet()) {
HgStoreMetric.Table table = new HgStoreMetric.Table();
table.setTableName(tableName);
table.setKeyCount(countMap.get(tableName));
table.setDataSize(sizeMap.get(tableName));
tables.add(table);
}
partMetric.setTables(tables);
return partMetric;
}
}
@Override
public HgStoreMetric.Graph getGraphMetric(String graph, int partId) {
HgStoreMetric.Graph graphMetric = new HgStoreMetric.Graph();
try (RocksDBSession dbSession = getSession(graph, partId)) {
graphMetric.setApproxDataSize(
dbSession.getApproximateDataSize(keyCreator.getStartKey(partId, graph),
keyCreator.getEndKey(partId, graph)));
graphMetric.setApproxKeyCount(dbSession.getEstimateNumKeys());
return graphMetric;
}
}
@Override
public void batchGet(String graph, String table, Supplier<HgPair<Integer, byte[]>> s,
Consumer<HgPair<byte[], byte[]>> c) throws HgStoreException {
int count = 0;
while (true) {
// Prevent dead loops
if (count++ == Integer.MAX_VALUE) {
break;
}
HgPair<Integer, byte[]> duality = s.get();
if (duality == null) {
break;
}
int code = duality.getKey();
byte[] key = duality.getValue();
int partId = provider.getPartitionByCode(graph, code).getId();
try (RocksDBSession dbSession = getSession(graph, table, partId)) {
byte[] targetKey = keyCreator.getKey(partId, graph, code, key);
byte[] value = dbSession.sessionOp().get(table, targetKey);
c.accept(new HgPair<>(key, value));
}
}
}
/**
* 清空图数据
*/
@Override
public void truncate(String graphName, int partId) throws HgStoreException {
// 每个分区对应一个 rocksdb 实例，因此 rocksdb 实例名为 rocksdb + partId
try (RocksDBSession dbSession = getSession(graphName, partId)) {
dbSession.sessionOp().deleteRange(keyCreator.getStartKey(partId, graphName),
keyCreator.getEndKey(partId, graphName));
// 释放图 ID
keyCreator.delGraphId(partId, graphName);
}
}
@Override
public void flushAll() {
log.warn("Flush all!!! ");
factory.getGraphNames().forEach(dbName -> {
try (RocksDBSession dbSession = factory.queryGraphDB(dbName)) {
if (dbSession != null) {
dbSession.flush(false);
}
}
});
}
@Override
public void closeAll() {
log.warn("close all db!!! ");
factory.getGraphNames().forEach(dbName -> {
factory.releaseGraphDB(dbName);
});
}
@Override
public Map<MemoryUsageType, Long> getApproximateMemoryUsageByType(List<Cache> caches) {
try {
return factory.getApproximateMemoryUsageByType(null, caches);
} catch (Exception e) {
return new HashMap<>();
}
}
@Override
public List<Integer> getLeaderPartitionIds(String graph) {
return partitionManager.getLeaderPartitionIds(graph);
}
@Override
public void saveSnapshot(String snapshotPath, String graph, int partId) throws
HgStoreException {
try (RocksDBSession dbSession = getSession(graph, partId)) {
dbSession.saveSnapshot(snapshotPath);
} catch (DBStoreException e) {
throw new HgStoreException(HgStoreException.EC_RKDB_EXPORT_SNAPSHOT_FAIL, e.toString());
}
}
@Override
public void loadSnapshot(String snapshotPath, String graph, int partId, long v1) throws
HgStoreException {
try (RocksDBSession dbSession = getSession(graph, partId)) {
dbSession.loadSnapshot(snapshotPath, v1);
keyCreator.clearCache(partId);
factory.destroyGraphDB(dbSession.getGraphName());
} catch (DBStoreException e) {
throw new HgStoreException(HgStoreException.EC_RKDB_IMPORT_SNAPSHOT_FAIL, e.toString());
}
}
@Override
public long getLatestSequenceNumber(String graph, int partId) {
try (RocksDBSession dbSession = getSession(graph, partId)) {
return dbSession.getLatestSequenceNumber();
}
}
@Override
public ScanIterator scanRaw(String graph, int partId, long seqNum) throws HgStoreException {
try (RocksDBSession dbSession = getSession(graph, partId)) {
return dbSession.sessionOp().scanRaw(null, null, seqNum);
} catch (DBStoreException e) {
throw new HgStoreException(HgStoreException.EC_RKDB_EXPORT_SNAPSHOT_FAIL, e.toString());
}
}
@Override
public void ingestSstFile(String graph, int partId, Map<byte[], List<String>> sstFiles) throws
HgStoreException {
try (RocksDBSession dbSession = getSession(graph, partId)) {
dbSession.ingestSstFile(sstFiles);
}
}
@Override
public boolean cleanPartition(String graph, int partId) {
Partition partition = partitionManager.getPartitionFromPD(graph, partId);
cleanPartition(graph, partId, partition.getStartKey(), partition.getEndKey(),
CleanType.CLEAN_TYPE_KEEP_RANGE);
return true;
}
@Override
public boolean cleanPartition(String graph, int partId, long startKey, long endKey,
CleanType cleanType) {
Partition partition = partitionManager.getPartition(graph, partId);
if (partition == null) {
return true;
}
log.info("cleanPartition: graph {}, part id: {}, {} -> {}, cleanType:{}", graph, partId,
startKey, endKey, cleanType);
var taskManager = HgStoreEngine.getInstance().getPartitionEngine(partId).getTaskManager();
CleanDataRequest request = new CleanDataRequest();
request.setPartitionId(partId);
request.setGraphName(graph);
request.setKeyStart(startKey);
request.setKeyEnd(endKey);
request.setCleanType(cleanType);
var cleanTask = new CleanTask(partId, graph, AsyncTaskState.START, request);
taskManager.putAsyncTask(cleanTask);
Utils.runInThread(() -> {
cleanPartition(partition, code -> {
// in range
boolean flag = code >= startKey && code < endKey;
return (cleanType == CleanType.CLEAN_TYPE_KEEP_RANGE) == flag;
});
// 可能被 destroy 了
if (HgStoreEngine.getInstance().getPartitionEngine(partId) != null) {
taskManager.updateAsyncTaskState(partId, graph, cleanTask.getId(),
AsyncTaskState.SUCCESS);
}
});
return true;
}
/**
* 清理分区数据，删除非本分区的数据
* 遍历 partId 的所有 key，读取 code，if code >= splitKey 生成新的 key，写入 newPartId
*/
private boolean cleanPartition(Partition partition,
Function<Integer, Boolean> belongsFunction) {
log.info("Partition {}-{} cleanPartition begin... {}", partition.getGraphName(),
partition.getId(), partition);
int counter = 0;
SessionOperator op = getSession(partition.getGraphName(), partition.getId()).sessionOp();
try {
ScanIterator cfIterator =
op.scanRaw(keyCreator.getStartKey(partition.getId(), partition.getGraphName()),
keyCreator.getEndKey(partition.getId(), partition.getGraphName()),
0);
while (cfIterator.hasNext()) {
ScanIterator iterator = cfIterator.next();
String table = new String(cfIterator.position());
long deleted = 0;
long total = 0;
while (iterator.hasNext()) {
total += 1;
RocksDBSession.BackendColumn col = iterator.next();
int keyCode = keyCreator.parseKeyCode(col.name);
// if (keyCode < partition.getStartKey() || keyCode >= partition.getEndKey()) {
if (!belongsFunction.apply(keyCode)) {
if (counter == 0) {
op.prepare();
}
op.delete(table, col.name); // 删除旧数据
if (++counter > batchSize) {
op.commit();
counter = 0;
}
deleted += 1;
}
}
iterator.close();
log.info("partition {}-{}, table:{}, delete keys {}, total:{}",
partition.getGraphName(), partition.getId(), table, deleted, total);
}
cfIterator.close();
} catch (Exception e) {
log.error("Partition {}-{} cleanPartition exception {}", partition.getGraphName(),
partition.getId(), e);
op.rollback();
throw e;
} finally {
if (counter > 0) {
try {
op.commit();
} catch (Exception e) {
op.rollback();
throw e;
}
}
op.getDBSession().close();
}
op.compactRange();
log.info("Partition {}-{} cleanPartition end", partition.getGraphName(), partition.getId());
return true;
}
@Override
public boolean deletePartition(String graph, int partId) {
try {
deleteGraphDatabase(graph, partId);
} catch (Exception e) {
log.error("Partition {}-{} deletePartition exception {}", graph, partId, e);
}
return true;
}
@Override
public List<String> getTableNames(String graph, int partId) {
try (RocksDBSession dbSession = getSession(graph, partId)) {
List<String> tables = null;
tables = dbSession.getTables().keySet().stream().collect(Collectors.toList());
return tables;
}
}
private RocksDBSession getSession(String graph, String table, int partId) throws
HgStoreException {
RocksDBSession dbSession = getSession(partId);
dbSession.checkTable(table);
return dbSession;
}
private RocksDBSession getSession(String graphName, int partId) throws HgStoreException {
return getSession(partId);
}
/**
* 获取 dbsession，不更新 dbsession 活跃时间
*/
@Override
public RocksDBSession getSession(int partId) throws HgStoreException {
// 每个分区对应一个 rocksdb 实例，因此 rocksdb 实例名为 rocksdb + partId
String dbName = getDbName(partId);
RocksDBSession dbSession = factory.queryGraphDB(dbName);
if (dbSession == null) {
long version = HgStoreEngine.getInstance().getCommittedIndex(partId);
dbSession =
factory.createGraphDB(partitionManager.getDbDataPath(partId, dbName), dbName,
version);
if (dbSession == null) {
log.info("failed to create a new graph db: {}", dbName);
throw new HgStoreException(HgStoreException.EC_RKDB_CREATE_FAIL,
"failed to create a new graph db: {}", dbName);
}
}
dbSession.setDisableWAL(true); //raft 模式，关闭 rocksdb 日志
return dbSession;
}
private void deleteGraphDatabase(String graph, int partId) throws IOException {
truncate(graph, partId);
}
private PartitionManager getPartManager() {
return this.partitionManager;
}
@Override
public TxBuilder txBuilder(String graph, int partId) throws HgStoreException {
return new TxBuilderImpl(graph, partId, getSession(graph, partId));
}
@Override
public boolean existsTable(String graph, int partId, String table) {
try (RocksDBSession session = getSession(graph, partId)) {
return session.tableIsExist(table);
}
}
@Override
public void createTable(String graph, int partId, String table) {
try (RocksDBSession session = getSession(graph, partId)) {
session.checkTable(table);
}
}
@Override
public void deleteTable(String graph, int partId, String table) {
dropTable(graph, partId, table);
// todo 检查表是否为空，为空则真实删除表
// try (RocksDBSession session = getOrCreateGraphDB(graph, partId)) {
// session.deleteTables(table);
// }
}
@Override
public void dropTable(String graph, int partId, String table) {
try (RocksDBSession session = getSession(graph, partId)) {
// session.dropTables(table);
session.sessionOp().deleteRange(table, keyCreator.getStartKey(partId, graph),
keyCreator.getEndKey(partId, graph));
}
}
/**
* 对 rocksdb 进行 compaction
*/
@Override
public boolean dbCompaction(String graphName, int partitionId) {
return this.dbCompaction(graphName, partitionId, "");
}
/**
* 对 rocksdb 进行 compaction
*/
@Override
public boolean dbCompaction(String graphName, int partitionId, String tableName) {
try (RocksDBSession session = getSession(graphName, partitionId)) {
SessionOperator op = session.sessionOp();
if (tableName.isEmpty()) {
op.compactRange();
} else {
op.compactRange(tableName);
}
}
log.info("Partition {}-{} dbCompaction end", graphName, partitionId);
return true;
}
/**
* 销毁图，并删除数据文件
*
* @param graphName
* @param partId
*/
@Override
public void destroyGraphDB(String graphName, int partId) throws HgStoreException {
// 每个图每个分区对应一个 rocksdb 实例，因此 rocksdb 实例名为 rocksdb + partId
String dbName = getDbName(partId);
factory.destroyGraphDB(dbName);
keyCreator.clearCache(partId);
}
@Override
public long count(String graph, String table) {
List<Integer> ids = this.getLeaderPartitionIds(graph);
Long all = ids.parallelStream().map((id) -> {
InnerKeyFilter it = null;
try (RocksDBSession dbSession = getSession(graph, table, id)) {
long count = 0;
SessionOperator op = dbSession.sessionOp();
it = new InnerKeyFilter(op.scan(table, keyCreator.getStartKey(id, graph),
keyCreator.getEndKey(id, graph),
ScanIterator.Trait.SCAN_LT_END));
while (it.hasNext()) {
it.next();
count++;
}
return count;
} catch (Exception e) {
throw e;
} finally {
if (it != null) {
try {
it.close();
} catch (Exception e) {
}
}
}
}).collect(Collectors.summingLong(l -> l));
return all;
}
@NotThreadSafe
private class TxBuilderImpl implements TxBuilder {
private final String graph;
private final int partId;
private final RocksDBSession dbSession;
private final SessionOperator op;
private TxBuilderImpl(String graph, int partId, RocksDBSession dbSession) {
this.graph = graph;
this.partId = partId;
this.dbSession = dbSession;
this.op = this.dbSession.sessionOp();
this.op.prepare();
}
@Override
public TxBuilder put(int code, String table, byte[] key, byte[] value) throws
HgStoreException {
try {
byte[] targetKey = keyCreator.getKey(this.partId, graph, code, key);
this.op.put(table, targetKey, value);
} catch (DBStoreException e) {
throw new HgStoreException(HgStoreException.EC_RKDB_DOPUT_FAIL, e.toString());
}
return this;
}
@Override
public TxBuilder del(int code, String table, byte[] key) throws HgStoreException {
try {
byte[] targetKey = keyCreator.getKey(this.partId, graph, code, key);
this.op.delete(table, targetKey);
} catch (DBStoreException e) {
throw new HgStoreException(HgStoreException.EC_RKDB_DODEL_FAIL, e.toString());
}
return this;
}
@Override
public TxBuilder delSingle(int code, String table, byte[] key) throws HgStoreException {
try {
byte[] targetKey = keyCreator.getKey(this.partId, graph, code, key);
op.deleteSingle(table, targetKey);
} catch (DBStoreException e) {
throw new HgStoreException(HgStoreException.EC_RDKDB_DOSINGLEDEL_FAIL,
e.toString());
}
return this;
}
@Override
public TxBuilder delPrefix(int code, String table, byte[] prefix) throws HgStoreException {
try {
this.op.deletePrefix(table, keyCreator.getPrefixKey(this.partId, graph, prefix));
} catch (DBStoreException e) {
throw new HgStoreException(HgStoreException.EC_RKDB_DODELPREFIX_FAIL, e.toString());
}
return this;
}
@Override
public TxBuilder delRange(int code, String table, byte[] start, byte[] end) throws
HgStoreException {
try {
this.op.deleteRange(table, keyCreator.getStartKey(this.partId, graph, start),
keyCreator.getEndKey(this.partId, graph, end));
} catch (DBStoreException e) {
throw new HgStoreException(HgStoreException.EC_RKDB_DODELRANGE_FAIL, e.toString());
}
return this;
}
@Override
public TxBuilder merge(int code, String table, byte[] key, byte[] value) throws
HgStoreException {
try {
byte[] targetKey = keyCreator.getKey(this.partId, graph, code, key);
op.merge(table, targetKey, value);
} catch (DBStoreException e) {
throw new HgStoreException(HgStoreException.EC_RKDB_DOMERGE_FAIL, e.toString());
}
return this;
}
@Override
public Tx build() {
return new Tx() {
@Override
public void commit() throws HgStoreException {
op.commit(); // commit发生异常后，必须调用rollback，否则造成锁未释放
dbSession.close();
}
@Override
public void rollback() throws HgStoreException {
try {
op.rollback();
} finally {
dbSession.close();
}
}
};
}
}
}

76
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/DataMover.java Normal file
View File

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.business;
import java.util.List;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.cmd.BatchPutRequest;
import org.apache.hugegraph.store.cmd.CleanDataRequest;
import org.apache.hugegraph.store.cmd.HgCmdClient;
import org.apache.hugegraph.store.cmd.UpdatePartitionResponse;
import com.alipay.sofa.jraft.Status;
/**
* 数据转移接口，实现分区分裂和合并，支持跨机器转移数据
*/
public interface DataMover {
void setBusinessHandler(BusinessHandler handler);
void setCmdClient(HgCmdClient client);
/**
* 拷贝分区source内的数据到其他分区targets
* 一个分区，迁移到多个分区
*
* @param source source partition
* @param targets target partitions
* @return execution status
* @throws Exception execution exception
*/
Status moveData(Metapb.Partition source, List<Metapb.Partition> targets) throws Exception;
/**
* 将source target的数据全部拷贝到target上
* 从一个分区迁移到另外一个分区
*
* @param source source partition
* @param target target partition
* @return execution result
* @throws Exception execution exception
*/
Status moveData(Metapb.Partition source, Metapb.Partition target) throws Exception;
// 同步副本之间的分区状态
UpdatePartitionResponse updatePartitionState(Metapb.Partition partition,
Metapb.PartitionState state);
// 同步副本之间分区的范围
UpdatePartitionResponse updatePartitionRange(Metapb.Partition partition, int startKey,
int endKey);
// 清理分区partition内的无效数据
void cleanData(Metapb.Partition partition);
// 写入数据
void doWriteData(BatchPutRequest request);
void doCleanData(CleanDataRequest request);
}

280
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/DefaultDataMover.java Normal file
View File

@ -0,0 +1,280 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.business;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.pd.grpc.pulse.CleanType;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.cmd.BatchPutRequest;
import org.apache.hugegraph.store.cmd.BatchPutResponse;
import org.apache.hugegraph.store.cmd.CleanDataRequest;
import org.apache.hugegraph.store.cmd.HgCmdClient;
import org.apache.hugegraph.store.cmd.UpdatePartitionRequest;
import org.apache.hugegraph.store.cmd.UpdatePartitionResponse;
import org.apache.hugegraph.store.term.Bits;
import com.alipay.sofa.jraft.Status;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DefaultDataMover implements DataMover {
public static int Batch_Put_Size = 2000;
private BusinessHandler businessHandler;
private HgCmdClient client;
private static Metapb.Partition findPartition(List<Metapb.Partition> partitions, int code) {
for (Metapb.Partition partition : partitions) {
if (code >= partition.getStartKey() && code < partition.getEndKey()) {
return partition;
}
}
return null;
}
@Override
public void setBusinessHandler(BusinessHandler handler) {
this.businessHandler = handler;
}
@Override
public void setCmdClient(HgCmdClient client) {
this.client = client;
}
@Override
public Status moveData(Metapb.Partition source, List<Metapb.Partition> targets) throws
Exception {
Status status = Status.OK();
// 开始移动数据之前，先把分区下线
UpdatePartitionResponse response =
updatePartitionState(source, Metapb.PartitionState.PState_Offline);
if (response.getStatus().isOK()) {
status = moveData(source, targets, DefaultDataMover::findPartition);
// 数据迁移成功后，设置新分区范围和上线新分区
for (var target : targets) {
if (status.isOk()) {
if (!(updatePartitionRange(target, (int) target.getStartKey(),
(int) target.getEndKey())
.getStatus().isOK()
&& updatePartitionState(target,
Metapb.PartitionState.PState_Normal).getStatus()
.isOK())) {
status.setError(-3, "new partition online fail");
}
}
}
} else {
status.setError(-1, "source partition offline fail");
}
updatePartitionState(source, Metapb.PartitionState.PState_Normal);
return status;
}
@Override
public Status moveData(Metapb.Partition source, Metapb.Partition target) throws Exception {
// 只写入 target
return moveData(source, Collections.singletonList(target), (partitions, integer) -> target);
}
/**
* move data from partition to targets
*
* @param source source partition
* @param targets target partitions
* @param partitionSelector the key of source partition belongs which target
* @return execution result
* @throws Exception exception when put data
*/
private Status moveData(Metapb.Partition source, List<Metapb.Partition> targets,
BiFunction<List<Metapb.Partition>, Integer, Metapb.Partition> partitionSelector)
throws Exception {
Status status = Status.OK();
String graphName = source.getGraphName();
List<String> tables = businessHandler.getTableNames(graphName, source.getId());
log.info("moveData, graph:{}, partition id:{} tables:{}, {}-{}", source.getGraphName(),
source.getId(), tables,
source.getStartKey(), source.getEndKey());
WriteBatch batch = new WriteBatch(graphName);
// target partition : count
Map<Integer, Long> moveCount = new HashMap<>();
for (String table : tables) {
int total = 0;
moveCount.clear();
try (ScanIterator iterator =
businessHandler.scan(graphName, table, (int) source.getStartKey(),
(int) source.getEndKey())) {
int count = 0;
while (iterator.hasNext() && status.isOk()) {
total += 1;
RocksDBSession.BackendColumn entry = iterator.next();
byte[] innerKey = entry.name;
byte[] key = Arrays.copyOfRange(innerKey, 0, innerKey.length - Short.BYTES);
int code = Bits.getShort(innerKey, innerKey.length - Short.BYTES);
Metapb.Partition partition = partitionSelector.apply(targets, code);
if (partition != null) {
moveCount.put(partition.getId(),
moveCount.getOrDefault(partition.getId(), 0L) + 1);
batch.add(partition.getId(),
BatchPutRequest.KV.of(table, code, key, entry.value));
if (++count >= Batch_Put_Size) {
if (!batch.sync()) {
status.setError(-2, "move data fail");
}
count = 0;
}
}
}
if (count > 0) {
if (!batch.sync()) {
status.setError(-2, "move data fail");
}
}
for (var pair : moveCount.entrySet()) {
log.info("{}-{}, table: {}, move to partition id {}, count:{}, total:{}",
source.getGraphName(), source.getId(), table, pair.getKey(),
pair.getValue(), total);
}
}
}
return status;
}
@Override
public UpdatePartitionResponse updatePartitionState(Metapb.Partition partition,
Metapb.PartitionState state) {
// 分区分裂时，主动需要查找 leader 进行同步信息
UpdatePartitionRequest request = new UpdatePartitionRequest();
request.setWorkState(state);
request.setPartitionId(partition.getId());
request.setGraphName(partition.getGraphName());
return client.raftUpdatePartition(request);
}
@Override
public UpdatePartitionResponse updatePartitionRange(Metapb.Partition partition, int startKey,
int endKey) {
// 分区分裂时，主动需要查找 leader 进行同步信息
UpdatePartitionRequest request = new UpdatePartitionRequest();
request.setStartKey(startKey);
request.setEndKey(endKey);
request.setPartitionId(partition.getId());
request.setGraphName(partition.getGraphName());
return client.raftUpdatePartition(request);
}
@Override
public void cleanData(Metapb.Partition partition) {
String graphName = partition.getGraphName();
CleanDataRequest request = new CleanDataRequest();
request.setGraphName(graphName);
request.setPartitionId(partition.getId());
request.setCleanType(CleanType.CLEAN_TYPE_KEEP_RANGE);
request.setKeyStart(partition.getStartKey());
request.setKeyEnd(partition.getEndKey());
request.setDeletePartition(false);
try {
client.cleanData(request);
} catch (Exception e) {
log.error("exception ", e);
}
}
@Override
public void doWriteData(BatchPutRequest request) {
BusinessHandler.TxBuilder tx =
businessHandler.txBuilder(request.getGraphName(), request.getPartitionId());
for (BatchPutRequest.KV kv : request.getEntries()) {
tx.put(kv.getCode(), kv.getTable(), kv.getKey(), kv.getValue());
}
tx.build().commit();
}
@Override
public void doCleanData(CleanDataRequest request) {
// raft 执行真实数据的清理
businessHandler.cleanPartition(request.getGraphName(), request.getPartitionId(),
request.getKeyStart(), request.getKeyEnd(),
request.getCleanType());
}
class WriteBatch {
private final Map<Integer, List<BatchPutRequest.KV>> data = new HashMap<>();
private final String graphName;
public WriteBatch(String graphName) {
this.graphName = graphName;
}
public WriteBatch add(int partition, BatchPutRequest.KV kv) {
if (!data.containsKey(partition)) {
data.put(partition, new LinkedList<>());
}
data.get(partition).add(kv);
return this;
}
public Boolean sync() throws Exception {
Boolean ret = true;
for (Map.Entry<Integer, List<BatchPutRequest.KV>> entry : data.entrySet()) {
ret = ret && sendData(entry.getKey(), entry.getValue());
}
for (List<BatchPutRequest.KV> list : data.values()) {
list.clear();
}
return ret;
}
public Boolean sendData(Integer partId, List<BatchPutRequest.KV> kvs) throws Exception {
BatchPutRequest request = new BatchPutRequest();
request.setGraphName(graphName);
request.setPartitionId(partId);
request.setEntries(kvs);
BatchPutResponse response = client.batchPut(request);
if (response == null || !response.getStatus().isOK()) {
log.error("sendData moveData error, pId:{} status:{}", partId,
response != null ? response.getStatus() : "EMPTY_RESPONSE");
return false;
}
return true;
}
}
}

126
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/FilterIterator.java Normal file
View File

@ -0,0 +1,126 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.business;
import java.util.Arrays;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hugegraph.backend.query.ConditionQuery;
import org.apache.hugegraph.backend.serializer.BinaryBackendEntry;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.structure.HugeElement;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class FilterIterator<T extends BackendColumn> extends
AbstractSelectIterator
implements ScanIterator {
private final ConditionQuery query;
T current = null;
public FilterIterator(ScanIterator iterator, ConditionQuery query) {
super();
this.iterator = iterator;
this.query = query;
// log.info("operator sinking is used to filter data:{}",
// query.toString());
}
public static ScanIterator of(ScanIterator it, byte[] conditionQuery) {
if (ArrayUtils.isEmpty(conditionQuery)) {
return it;
}
ConditionQuery query = ConditionQuery.fromBytes(conditionQuery);
return new FilterIterator(it, query);
}
@Override
public boolean hasNext() {
boolean match = false;
if (this.query.resultType().isVertex() ||
this.query.resultType().isEdge()) {
BackendEntry entry = null;
while (iterator.hasNext()) {
current = iterator.next();
BackendEntry.BackendColumn column =
BackendEntry.BackendColumn.of(
current.name, current.value);
BackendEntry.BackendColumn[] columns =
new BackendEntry.BackendColumn[]{column};
if (entry == null || !belongToMe(entry, column) ||
this.query.resultType().isEdge()) {
entry = new BinaryBackendEntry(query.resultType(),
current.name);
entry.columns(Arrays.asList(columns));
} else {
// 有可能存在包含多个 column 的情况
entry.columns(Arrays.asList(columns));
continue;
}
HugeElement element = this.parseEntry(entry,
this.query.resultType()
.isVertex());
match = query.test(element);
if (match) {
break;
}
}
} else {
boolean has = iterator.hasNext();
if (has) {
current = iterator.next();
}
return has;
}
return match;
}
@Override
public boolean isValid() {
return false;
}
@Override
public <T> T next() {
return (T) current;
}
@Override
public long count() {
return iterator.count();
}
@Override
public byte[] position() {
return iterator.position();
}
@Override
public void seek(byte[] position) {
this.iterator.seek(position);
}
@Override
public void close() {
iterator.close();
}
}

351
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/GraphStoreIterator.java Normal file
View File

@ -0,0 +1,351 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.business;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import javax.script.Bindings;
import javax.script.CompiledScript;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import org.apache.commons.lang.StringUtils;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.serializer.BinaryBackendEntry;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.schema.EdgeLabel;
import org.apache.hugegraph.schema.PropertyKey;
import org.apache.hugegraph.schema.VertexLabel;
import org.apache.hugegraph.store.grpc.Graphpb;
import org.apache.hugegraph.store.grpc.Graphpb.Edge;
import org.apache.hugegraph.store.grpc.Graphpb.ScanPartitionRequest;
import org.apache.hugegraph.store.grpc.Graphpb.ScanPartitionRequest.Request;
import org.apache.hugegraph.store.grpc.Graphpb.ScanPartitionRequest.ScanType;
import org.apache.hugegraph.store.grpc.Graphpb.Variant.Builder;
import org.apache.hugegraph.store.grpc.Graphpb.VariantType;
import org.apache.hugegraph.store.grpc.Graphpb.Vertex;
import org.apache.hugegraph.structure.HugeEdge;
import org.apache.hugegraph.structure.HugeElement;
import org.apache.hugegraph.structure.HugeProperty;
import org.apache.hugegraph.structure.HugeVertex;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.util.Blob;
import org.apache.tinkerpop.gremlin.structure.Property;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.codehaus.groovy.jsr223.GroovyScriptEngineImpl;
import com.google.protobuf.ByteString;
import com.google.protobuf.Descriptors;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class GraphStoreIterator<T> extends AbstractSelectIterator
implements ScanIterator {
private static final Descriptors.FieldDescriptor propertiesDesEdge =
Edge.getDescriptor().findFieldByNumber(6);
private static final Descriptors.FieldDescriptor propertiesDesVertex =
Vertex.getDescriptor().findFieldByNumber(3);
private final ScanPartitionRequest scanRequest;
private final ScanIterator iter;
private final Request request;
private final boolean isVertex;
private final HugeType type;
private final Set properties;
private Vertex.Builder vertex;
private Edge.Builder edge;
private ArrayList<BackendColumn> data;
private GroovyScriptEngineImpl engine;
private CompiledScript script;
private HugeElement current;
public GraphStoreIterator(ScanIterator iterator,
ScanPartitionRequest scanRequest) {
super();
this.iter = iterator;
this.scanRequest = scanRequest;
this.request = this.scanRequest.getScanRequest();
ScanType scanType = this.request.getScanType();
isVertex = scanType.equals(ScanType.SCAN_VERTEX);
if (isVertex) {
vertex = Vertex.newBuilder();
type = HugeType.VERTEX;
} else {
edge = Edge.newBuilder();
type = HugeType.EDGE;
}
properties = new HashSet<Long>();
List<Long> pl = request.getPropertiesList();
if (pl != null) {
for (Long i : pl) {
properties.add(i);
}
}
String condition = request.getCondition();
if (!StringUtils.isEmpty(condition)) {
ScriptEngineManager factory = new ScriptEngineManager();
engine = (GroovyScriptEngineImpl) factory.getEngineByName("groovy");
try {
script = engine.compile(condition);
} catch (ScriptException e) {
log.error("create script with error:", e);
}
}
}
private HugeElement getElement(BackendColumn next) {
BackendEntry entry = null;
BackendEntry.BackendColumn column = BackendEntry.BackendColumn.of(
next.name, next.value);
if (entry == null || !belongToMe(entry, column) || !isVertex) {
try {
entry = new BinaryBackendEntry(type, next.name);
} catch (Exception e) {
log.error("using core to new entry with error:", e);
}
}
BackendEntry.BackendColumn[] columns =
new BackendEntry.BackendColumn[]{column};
entry.columns(Arrays.asList(columns));
return this.parseEntry(entry, isVertex);
}
@Override
public boolean hasNext() {
if (current == null) {
while (iter.hasNext()) {
BackendColumn next = this.iter.next();
HugeElement element = getElement(next);
try {
boolean evalResult = true;
if (isVertex) {
HugeVertex el = (HugeVertex) element;
if (engine != null) {
Bindings bindings = engine.createBindings();
bindings.put("element", el);
evalResult = (boolean) script.eval(bindings);
}
} else {
HugeEdge el = (HugeEdge) element;
if (engine != null) {
Bindings bindings = engine.createBindings();
bindings.put("element", el);
evalResult = (boolean) script.eval(bindings);
}
}
if (!evalResult) {
continue;
}
current = element;
return true;
} catch (Exception e) {
log.error("get next with error:", e);
}
}
} else {
return true;
}
return false;
}
@Override
public boolean isValid() {
return false;
}
@Override
public T next() {
T next;
if (isVertex) {
next = (T) parseVertex(current);
} else {
next = (T) parseEdge(current);
}
current = null;
return next;
}
public T select(BackendColumn current) {
HugeElement element = getElement(current);
if (isVertex) {
return (T) parseVertex(element);
} else {
return (T) parseEdge(element);
}
}
public ArrayList<T> convert() {
ArrayList result = new ArrayList(data.size());
for (int i = 0; i < data.size(); i++) {
result.add(select(data.get(i)));
}
return result;
}
private <P extends Property<Object>> List<Graphpb.Property> buildProperties(
Builder variant,
int size,
Iterator<P> eps) {
int pSize = properties.size();
List<Graphpb.Property> props = new ArrayList<>(pSize > 0 ?
pSize : size);
Graphpb.Property.Builder pb = Graphpb.Property.newBuilder();
while (eps.hasNext()) {
HugeProperty<?> property = (HugeProperty<?>) eps.next();
PropertyKey key = property.propertyKey();
long pkId = key.id().asLong();
if (pSize > 0 && !properties.contains(pkId)) {
continue;
}
pb.clear();
variant.clear();
pb.setLabel(pkId);
Object v = property.value();
switch (key.dataType()) {
case UUID:
variant.setType(VariantType.VT_STRING)
.setValueString(v.toString());
break;
case LONG:
variant.setType(VariantType.VT_LONG)
.setValueInt64((Long) v);
break;
case INT:
variant.setType(VariantType.VT_INT)
.setValueInt32((Integer) v);
break;
case BLOB:
byte[] bytes = v instanceof byte[] ?
(byte[]) v : ((Blob) v).bytes();
variant.setType(VariantType.VT_BYTES)
.setValueBytes(ByteString.copyFrom(bytes));
break;
case BYTE:
variant.setType(VariantType.VT_BYTES)
.setValueBytes(
ByteString.copyFrom(new byte[]{(Byte) v}));
break;
case DATE:
Date date = (Date) v;
variant.setType(VariantType.VT_DATETIME)
.setValueDatetime(date.toString());
break;
case FLOAT:
variant.setType(VariantType.VT_FLOAT)
.setValueFloat((Float) v);
break;
case TEXT:
variant.setType(VariantType.VT_STRING)
.setValueString((String) v);
break;
case DOUBLE:
variant.setType(VariantType.VT_DOUBLE)
.setValueDouble((Double) v);
break;
case OBJECT:
case UNKNOWN:
variant.setType(VariantType.VT_UNKNOWN)
.setValueString(v.toString());
break;
case BOOLEAN:
variant.setType(VariantType.VT_BOOLEAN)
.setValueBoolean((Boolean) v);
break;
default:
break;
}
pb.setValue(variant.build());
props.add(pb.build());
}
return props;
}
private void buildId(Builder variant, Id id) {
switch (id.type()) {
case STRING:
case UUID:
variant.setType(VariantType.VT_STRING)
.setValueString(id.asString());
break;
case LONG:
variant.setType(VariantType.VT_LONG)
.setValueInt64(id.asLong());
break;
case EDGE:
// TODO
break;
case UNKNOWN:
variant.setType(VariantType.VT_UNKNOWN)
.setValueBytes(ByteString.copyFrom(id.asBytes()));
break;
default:
break;
}
}
private Edge parseEdge(HugeElement element) {
HugeEdge e = (HugeEdge) element;
edge.clear();
EdgeLabel label = e.schemaLabel();
edge.setLabel(label.longId());
edge.setSourceLabel(e.sourceVertex().schemaLabel().id().asLong());
edge.setTargetLabel(e.targetVertex().schemaLabel().id().asLong());
Builder variant = Graphpb.Variant.newBuilder();
buildId(variant, e.sourceVertex().id());
edge.setSourceId(variant.build());
variant.clear();
buildId(variant, e.targetVertex().id());
edge.setTargetId(variant.build());
int size = e.sizeOfProperties();
Iterator<Property<Object>> eps = e.properties();
List<Graphpb.Property> props = buildProperties(variant, size, eps);
edge.setField(propertiesDesEdge, props);
return edge.build();
}
private Vertex parseVertex(HugeElement element) {
HugeVertex v = (HugeVertex) element;
vertex.clear();
VertexLabel label = v.schemaLabel();
vertex.setLabel(label.longId());
Builder variant = Graphpb.Variant.newBuilder();
buildId(variant, v.id());
vertex.setId(variant.build());
int size = v.sizeOfProperties();
Iterator<VertexProperty<Object>> vps = v.properties();
List<Graphpb.Property> props = buildProperties(variant, size, vps);
vertex.setField(propertiesDesVertex, props);
return vertex.build();
}
@Override
public void close() {
iter.close();
}
}

117
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/InnerKeyCreator.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.business;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hugegraph.store.meta.GraphIdManager;
import org.apache.hugegraph.store.term.Bits;
import org.apache.hugegraph.store.util.HgStoreException;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class InnerKeyCreator {
final BusinessHandler businessHandler;
private final Map<Integer, GraphIdManager> graphIdCache = new ConcurrentHashMap<>();
public InnerKeyCreator(BusinessHandler businessHandler) {
this.businessHandler = businessHandler;
}
public int getGraphId(Integer partId, String graphName) throws HgStoreException {
try {
GraphIdManager manager;
if ((manager = graphIdCache.get(partId)) == null) {
manager = new GraphIdManager(businessHandler, partId);
graphIdCache.put(partId, manager);
}
return (int) manager.getGraphId(graphName);
} catch (Exception e) {
throw new HgStoreException(HgStoreException.EC_RKDB_PD_FAIL, e.getMessage());
}
}
public void delGraphId(Integer partId, String graphName) {
if (graphIdCache.containsKey(partId)) {
graphIdCache.get(partId).releaseGraphId(graphName);
} else {
new GraphIdManager(businessHandler, partId).releaseGraphId(graphName);
}
}
public void clearCache(Integer partId) {
graphIdCache.remove(partId);
}
/**
* 从key中解析出keyCode
*/
public int parseKeyCode(byte[] innerKey) {
return Bits.getShort(innerKey, innerKey.length - Short.BYTES);
}
public byte[] getKey(Integer partId, String graph, int code, byte[] key) {
int graphId = getGraphId(partId, graph);
byte[] buf = new byte[Short.BYTES + key.length + Short.BYTES];
Bits.putShort(buf, 0, graphId);
Bits.put(buf, Short.BYTES, key);
Bits.putShort(buf, key.length + Short.BYTES, code);
return buf;
}
public byte[] getStartKey(Integer partId, String graph) {
int graphId = getGraphId(partId, graph);
byte[] buf = new byte[Short.BYTES];
Bits.putShort(buf, 0, graphId);
return buf;
}
public byte[] getStartKey(Integer partId, String graph, byte[] key) {
int graphId = getGraphId(partId, graph);
byte[] buf = new byte[Short.BYTES + key.length];
Bits.putShort(buf, 0, graphId);
Bits.put(buf, Short.BYTES, key);
return buf;
}
public byte[] getEndKey(Integer partId, String graph) {
int graphId = getGraphId(partId, graph);
byte[] buf = new byte[Short.BYTES];
Bits.putShort(buf, 0, graphId + 1);
return buf;
}
public byte[] getEndKey(Integer partId, String graph, byte[] key) {
int graphId = getGraphId(partId, graph);
byte[] buf = new byte[Short.BYTES + key.length];
Bits.putShort(buf, 0, graphId);
Bits.put(buf, Short.BYTES, key);
return buf;
}
public byte[] getPrefixKey(Integer partId, String graph, byte[] prefix) {
int graphId = getGraphId(partId, graph);
byte[] buf = new byte[Short.BYTES + prefix.length];
Bits.putShort(buf, 0, graphId);
Bits.put(buf, Short.BYTES, prefix);
return buf;
}
}

105
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/InnerKeyFilter.java Normal file
View File

@ -0,0 +1,105 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.business;
import java.util.Arrays;
import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.term.Bits;
public class InnerKeyFilter<T extends BackendColumn> implements ScanIterator {
final int codeFrom;
final int codeTo;
//是否进行code过滤，启动该选项，返回key的尾部包含code
final boolean codeFilter;
ScanIterator iterator;
T current = null;
public InnerKeyFilter(ScanIterator iterator) {
this.iterator = iterator;
this.codeFrom = Integer.MIN_VALUE;
this.codeTo = Integer.MAX_VALUE;
this.codeFilter = false;
moveNext();
}
public InnerKeyFilter(ScanIterator iterator, int codeFrom, int codeTo) {
this.iterator = iterator;
this.codeFrom = codeFrom;
this.codeTo = codeTo;
this.codeFilter = true;
moveNext();
}
private void moveNext() {
current = null;
if (codeFilter) {
while (iterator.hasNext()) {
T t = iterator.next();
int code = Bits.getShort(t.name, t.name.length - Short.BYTES);
if (code >= codeFrom && code < codeTo) {
current = t;
break;
}
}
} else {
if (iterator.hasNext()) {
current = iterator.next();
}
}
}
@Override
public boolean hasNext() {
return current != null;
}
@Override
public boolean isValid() {
return iterator.isValid();
}
@Override
public T next() {
T column = current;
if (!codeFilter)
// 去掉图ID和hash后缀
{
column.name = Arrays.copyOfRange(column.name, Short.BYTES,
column.name.length - Short.BYTES);
} else// 去掉图ID
{
column.name = Arrays.copyOfRange(column.name, Short.BYTES,
column.name.length);
}
moveNext();
return column;
}
@Override
public void close() {
iterator.close();
}
@Override
public long count() {
return iterator.count();
}
}

201
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/MultiPartitionIterator.java Normal file
View File

@ -0,0 +1,201 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.business;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Queue;
import java.util.function.BiFunction;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import lombok.extern.slf4j.Slf4j;
/**
* created on 2021/11/2
*
* @version 1.1.0 implements position method to pass partition-id on 2022/03/10
*/
@Slf4j
public class MultiPartitionIterator implements ScanIterator {
public final static byte[] EMPTY_BYTES = new byte[0];
private final Queue<Integer> partitions;
private final BiFunction<Integer, byte[], ScanIterator> supplier;
private ScanIterator iterator;
private Integer curPartitionId;
private Integer positionPartitionId;
private byte[] positionKey;
private MultiPartitionIterator(List<Integer> partitionIds,
BiFunction<Integer, byte[], ScanIterator> supplier) {
/*****************************************************************************
** CAUTION: MAKE SURE IT SORTED IN A FIXED ORDER! TO DO THIS IS FOR PAGING. **
*****************************************************************************/
Collections.sort(partitionIds);
this.partitions = new LinkedList<>(partitionIds);
this.supplier = supplier;
}
public static MultiPartitionIterator of(List<Integer> partitionIdList,
BiFunction<Integer, byte[], ScanIterator> supplier) {
return new MultiPartitionIterator(partitionIdList, supplier);
}
private static byte[] toBytes(final int i) {
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
buffer.putInt(i);
return buffer.array();
}
public static int toInt(byte[] bytes) {
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
buffer.put(bytes);
buffer.flip();//need flip
return buffer.getInt();
}
private ScanIterator getIterator() {
if (this.partitions.isEmpty()) {
return null;
}
ScanIterator buf = null;
while (!partitions.isEmpty()) {
this.curPartitionId = partitions.poll();
if (!this.inPosition(this.curPartitionId)) {
continue;
}
buf = supplier.apply(this.curPartitionId, getPositionKey(this.curPartitionId));
if (buf == null) {
continue;
}
if (buf.hasNext()) {
break;
}
}
if (buf == null) {
return null;
}
if (!buf.hasNext()) {
buf.close();
buf = null;
}
return buf;
}
private void init() {
if (this.iterator == null) {
this.iterator = this.getIterator();
}
}
@Override
public boolean hasNext() {
this.init();
return this.iterator != null;
}
@Override
public boolean isValid() {
this.init();
return this.iterator != null;
}
@Override
public <T> T next() {
this.init();
if (this.iterator == null) {
throw new NoSuchElementException();
}
T t = this.iterator.next();
if (!this.iterator.hasNext()) {
this.iterator.close();
this.iterator = null;
}
return t;
}
@Override
public long count() {
long count = 0;
this.iterator = this.getIterator();
while (this.iterator != null) {
count += this.iterator.count();
// this.iterator.close();
this.iterator = this.getIterator();
}
return count;
}
/**
* @return the current partition-id in bytes form.
*/
@Override
public byte[] position() {
if (this.curPartitionId == null) {
return EMPTY_BYTES;
}
return toBytes(this.curPartitionId.shortValue());
}
@Override
public void seek(byte[] position) {
if (position == null || position.length < Integer.BYTES) {
return;
}
byte[] buf = new byte[Integer.BYTES];
System.arraycopy(position, 0, buf, 0, Integer.BYTES);
this.positionPartitionId = toInt(buf);
this.positionKey = new byte[position.length - Integer.BYTES];
System.arraycopy(position, Integer.BYTES, this.positionKey, 0, this.positionKey.length);
}
@Override
public void close() {
if (this.iterator != null) {
this.iterator.close();
}
}
private boolean inPosition(int partitionId) {
if (this.positionPartitionId == null) {
return true;
}
return partitionId >= this.positionPartitionId;
}
private byte[] getPositionKey(int partitionId) {
if (this.positionKey == null || this.positionKey.length == 0) {
return null;
}
if (this.positionPartitionId == null) {
return null;
}
if (this.positionPartitionId.intValue() == partitionId) {
return this.positionKey;
} else {
return null;
}
}
}

107
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/SelectIterator.java Normal file
View File

@ -0,0 +1,107 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.business;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.serializer.BytesBuffer;
import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.type.define.DataType;
import org.apache.hugegraph.type.define.SerialEnum;
public class SelectIterator implements ScanIterator {
ScanIterator iter;
Set<Integer> properties;
public SelectIterator(ScanIterator iterator, List<Integer> properties) {
this.iter = iterator;
this.properties = new HashSet<>(properties);
}
public BackendColumn select(BackendColumn column) {
int size;
if (properties == null || (size = properties.size()) == 0) {
return column;
}
byte[] name = column.name;
byte[] value = column.value;
BytesBuffer buffer = BytesBuffer.wrap(value);
Id labelId = buffer.readId(); // label
int bpSize = buffer.readVInt(); // property
if (size == bpSize) {
return column;
}
BytesBuffer allocate = BytesBuffer.allocate(8 + 16 * size);
allocate.writeId(labelId);
allocate.writeVInt(size);
for (int i = 0; i < bpSize; i++) {
int propertyId = buffer.readVInt();
byte cat = buffer.read(); // cardinality and type
byte code = BytesBuffer.getType(cat);
DataType dataType = SerialEnum.fromCode(DataType.class, code);
Object bpValue = buffer.readProperty(dataType);
if (properties.contains(propertyId)) {
allocate.writeVInt(propertyId);
allocate.write(cat);
allocate.writeProperty(dataType, bpValue);
}
}
return BackendColumn.of(name, allocate.bytes());
}
@Override
public boolean hasNext() {
return this.iter.hasNext();
}
@Override
public boolean isValid() {
return this.iter.isValid();
}
@Override
public BackendColumn next() {
BackendColumn value = this.iter.next();
return select(value);
}
@Override
public long count() {
return this.iter.count();
}
@Override
public byte[] position() {
return this.iter.position();
}
@Override
public void seek(byte[] position) {
this.iter.seek(position);
}
@Override
public void close() {
this.iter.close();
}
}

52
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/BatchPutRequest.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
import java.io.Serializable;
import java.util.List;
import lombok.Data;
@Data
public class BatchPutRequest extends HgCmdBase.BaseRequest {
private List<KV> entries;
@Override
public byte magic() {
return HgCmdBase.BATCH_PUT;
}
@Data
public static class KV implements Serializable {
private String table;
private int code;
private byte[] key;
private byte[] value;
public static KV of(String table, int code, byte[] key, byte[] value) {
KV kv = new KV();
kv.table = table;
kv.code = code;
kv.key = key;
kv.value = value;
return kv;
}
}
}

22
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/BatchPutResponse.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
public class BatchPutResponse extends HgCmdBase.BaseResponse {
}

70
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CleanDataRequest.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
import org.apache.hugegraph.pd.grpc.pulse.CleanPartition;
import org.apache.hugegraph.pd.grpc.pulse.CleanType;
import org.apache.hugegraph.store.meta.Partition;
import lombok.Data;
@Data
public class CleanDataRequest extends HgCmdBase.BaseRequest {
private long keyStart;
private long keyEnd;
private CleanType cleanType;
private boolean deletePartition;
private long taskId;
public static CleanDataRequest fromCleanPartitionTask(CleanPartition task, Partition partition,
long taskId) {
return fromCleanPartitionTask(partition.getGraphName(), partition.getId(), taskId, task);
}
public static CleanDataRequest fromCleanPartitionTask(String graphName, int partitionId,
long taskId,
CleanPartition task) {
CleanDataRequest request = new CleanDataRequest();
request.setGraphName(graphName);
request.setPartitionId(partitionId);
request.setCleanType(task.getCleanType());
request.setKeyStart(task.getKeyStart());
request.setKeyEnd(task.getKeyEnd());
request.setDeletePartition(task.getDeletePartition());
request.setTaskId(taskId);
return request;
}
public static CleanPartition toCleanPartitionTask(CleanDataRequest request) {
return CleanPartition.newBuilder()
.setKeyStart(request.keyStart)
.setKeyEnd(request.keyEnd)
.setDeletePartition(request.deletePartition)
.setCleanType(request.cleanType)
.build();
}
@Override
public byte magic() {
return HgCmdBase.CLEAN_DATA;
}
}

22
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CleanDataResponse.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
public class CleanDataResponse extends HgCmdBase.BaseResponse {
}

72
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CreateRaftRequest.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
import java.util.ArrayList;
import java.util.List;
import org.apache.hugegraph.pd.grpc.Metapb;
import com.alipay.sofa.jraft.conf.Configuration;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class CreateRaftRequest extends HgCmdBase.BaseRequest {
List<byte[]> values = new ArrayList<>();
String peers;
public List<Metapb.Partition> getPartitions() {
try {
List<Metapb.Partition> partitions = new ArrayList<>();
for (byte[] partition : values) {
partitions.add(Metapb.Partition.parseFrom(partition));
}
return partitions;
} catch (InvalidProtocolBufferException e) {
log.error("CreateRaftNodeProcessor parse partition exception }", e);
}
return new ArrayList<>();
}
public void addPartition(Metapb.Partition partition) {
values.add(partition.toByteArray());
}
public Configuration getConf() {
Configuration conf = null;
if (peers != null) {
conf = new Configuration();
conf.parse(this.peers);
}
return conf;
}
public void setConf(Configuration conf) {
if (conf != null) {
this.peers = conf.toString();
}
}
@Override
public byte magic() {
return HgCmdBase.CREATE_RAFT;
}
}

22
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CreateRaftResponse.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
public class CreateRaftResponse extends HgCmdBase.BaseResponse {
}

31
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DbCompactionRequest.java Normal file
View File

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
import lombok.Data;
@Data
public class DbCompactionRequest extends HgCmdBase.BaseRequest {
private String tableName;
@Override
public byte magic() {
return HgCmdBase.ROCKSDB_COMPACTION;
}
}

22
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DbCompactionResponse.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
public class DbCompactionResponse extends HgCmdBase.BaseResponse {
}

38
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DestroyRaftRequest.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
@Data
public class DestroyRaftRequest extends HgCmdBase.BaseRequest {
private final List<String> graphNames = new ArrayList<>();
public void addGraphName(String graphName) {
graphNames.add(graphName);
}
@Override
public byte magic() {
return HgCmdBase.DESTROY_RAFT;
}
}

22
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DestroyRaftResponse.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
public class DestroyRaftResponse extends HgCmdBase.BaseResponse {
}

49
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/FutureClosureAdapter.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
import java.util.concurrent.CompletableFuture;
import com.alipay.sofa.jraft.Closure;
import com.alipay.sofa.jraft.Status;
public class FutureClosureAdapter<T> implements Closure {
public final CompletableFuture<T> future = new CompletableFuture<>();
private T resp;
public T getResponse() {
return this.resp;
}
public void setResponse(T resp) {
this.resp = resp;
future.complete(resp);
run(Status.OK());
}
public void failure(Throwable t) {
future.completeExceptionally(t);
run(new Status(-1, t.getMessage()));
}
@Override
public void run(Status status) {
}
}

26
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/GetStoreInfoRequest.java Normal file
View File

@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
public class GetStoreInfoRequest extends HgCmdBase.BaseRequest {
@Override
public byte magic() {
return HgCmdBase.GET_STORE_INFO;
}
}

44
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/GetStoreInfoResponse.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.meta.Store;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class GetStoreInfoResponse extends HgCmdBase.BaseResponse {
private byte[] store;
public Store getStore() {
try {
return new Store(Metapb.Store.parseFrom(this.store));
} catch (InvalidProtocolBufferException e) {
log.error("GetStoreResponse parse exception {}", e);
}
return null;
}
public void setStore(Store store) {
this.store = store.getProtoObj().toByteArray();
}
}

78
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdBase.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
public class HgCmdBase {
public static final byte GET_STORE_INFO = 0x01;
public static final byte BATCH_PUT = 0x02;
public static final byte CLEAN_DATA = 0x03;
public static final byte RAFT_UPDATE_PARTITION = 0x04;
public static final byte ROCKSDB_COMPACTION = 0x05;
public static final byte CREATE_RAFT = 0x06;
public static final byte DESTROY_RAFT = 0x07;
@Data
public abstract static class BaseRequest implements Serializable {
private String graphName;
private int partitionId;
public abstract byte magic();
}
@Data
public abstract static class BaseResponse implements Serializable {
List<PartitionLeader> partitionLeaders;
private HgCmdProcessor.Status status;
public synchronized BaseResponse addPartitionLeader(PartitionLeader ptLeader) {
if (partitionLeaders == null) {
partitionLeaders = new ArrayList<>();
}
partitionLeaders.add(ptLeader);
return this;
}
public static class PartitionLeader implements Serializable {
private final Integer partId;
private final Long storeId;
public PartitionLeader(Integer partId, Long storeId) {
this.partId = partId;
this.storeId = storeId;
}
public Long getStoreId() {
return storeId;
}
public Integer getPartId() {
return partId;
}
}
}
}

262
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdClient.java Normal file
View File

@ -0,0 +1,262 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.meta.Store;
import com.alipay.sofa.jraft.Closure;
import com.alipay.sofa.jraft.JRaftUtils;
import com.alipay.sofa.jraft.Status;
import com.alipay.sofa.jraft.conf.Configuration;
import com.alipay.sofa.jraft.option.RpcOptions;
import com.alipay.sofa.jraft.rpc.InvokeCallback;
import com.alipay.sofa.jraft.rpc.InvokeContext;
import com.alipay.sofa.jraft.rpc.RaftRpcFactory;
import com.alipay.sofa.jraft.rpc.RpcClient;
import com.alipay.sofa.jraft.util.Endpoint;
import com.alipay.sofa.jraft.util.RpcFactoryHelper;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class HgCmdClient {
private static final int MAX_RETRY_TIMES = 5;
protected volatile RpcClient rpcClient;
private RpcOptions rpcOptions;
private PartitionAgent ptAgent;
public synchronized boolean init(final RpcOptions rpcOptions, PartitionAgent ptAgent) {
this.ptAgent = ptAgent;
this.rpcOptions = rpcOptions;
final RaftRpcFactory factory = RpcFactoryHelper.rpcFactory();
this.rpcClient =
factory.createRpcClient(factory.defaultJRaftClientConfigHelper(this.rpcOptions));
return this.rpcClient.init(rpcOptions);
}
public <T> Future<T> createRaftNode(final String address, final List<Partition> partitions,
final Closure done) {
CreateRaftRequest request = new CreateRaftRequest();
partitions.forEach(partition -> {
request.addPartition(partition.getProtoObj());
});
log.info("Send to {} CreateRaftNode rpc call {} ", address, request.getPartitions().get(0));
return internalCallAsyncWithRpc(JRaftUtils.getEndPoint(address), request, done);
}
public <T> Future<T> createRaftNode(final String address, final List<Partition> partitions,
Configuration conf, final Closure done) {
CreateRaftRequest request = new CreateRaftRequest();
partitions.forEach(partition -> {
request.addPartition(partition.getProtoObj());
});
request.setConf(conf);
log.info("Send to {} CreateRaftNode rpc call {} ", address, request.getPartitions().get(0));
return internalCallAsyncWithRpc(JRaftUtils.getEndPoint(address), request, done);
}
public <T> Future<T> destroyRaftNode(final String peer, final List<Partition> partitions,
final Closure done) {
DestroyRaftRequest request = new DestroyRaftRequest();
partitions.forEach(partition -> {
request.setPartitionId(partition.getId());
request.addGraphName(partition.getGraphName());
});
log.info("Send to {} DestroyRaftNode rpc call partitionId={} ", peer,
request.getPartitionId());
return internalCallAsyncWithRpc(JRaftUtils.getEndPoint(peer), request, done);
}
public Store getStoreInfo(final String address) {
GetStoreInfoRequest request = new GetStoreInfoRequest();
request.setGraphName("");
request.setPartitionId(0);
GetStoreInfoResponse response = null;
try {
response = internalCallSyncWithRpc(JRaftUtils.getEndPoint(address), request);
} catch (Exception e) {
return null;
}
return response != null ? response.getStore() : null;
}
/**
* 批量插入数据
*
* @param request
* @return
*/
public BatchPutResponse batchPut(BatchPutRequest request) {
return (BatchPutResponse) tryInternalCallSyncWithRpc(request);
}
/**
* 清理无效数据
*
* @param request
* @return
*/
public CleanDataResponse cleanData(CleanDataRequest request) {
return (CleanDataResponse) tryInternalCallSyncWithRpc(request);
}
/**
* 通过raft更新本地分区信息
*
* @param request
* @return
*/
public UpdatePartitionResponse raftUpdatePartition(UpdatePartitionRequest request) {
return (UpdatePartitionResponse) tryInternalCallSyncWithRpc(request);
}
/**
* 查找Leader，错误重试，处理Leader重定向
*
* @param request
* @return
*/
public HgCmdBase.BaseResponse tryInternalCallSyncWithRpc(HgCmdBase.BaseRequest request) {
HgCmdBase.BaseResponse response = null;
for (int i = 0; i < MAX_RETRY_TIMES; i++) {
try {
Endpoint leader = ptAgent.getPartitionLeader(request.getGraphName(),
request.getPartitionId());
if (leader == null) {
log.error("get leader of graph {} - {} is null", request.getGraphName(),
request.getPartitionId());
Thread.sleep(i * 1000);
continue;
}
response = internalCallSyncWithRpc(leader, request);
if (response != null) {
if (response.getStatus().isOK()) {
break;
} else if (HgCmdProcessor.Status.LEADER_REDIRECT == response.getStatus()
&& response.partitionLeaders != null
) {
// 当返回leader 漂移，并且partitionLeaders 不为空时，需要重新设置leader
} else {
log.error(
"HgCmdClient tryInternalCallSyncWithRpc error msg {} leaders is {}",
response.getStatus().getMsg(), response.getPartitionLeaders());
}
}
// break;
} catch (Exception e) {
if (i + 1 >= MAX_RETRY_TIMES) {
log.error("tryInternalCallSyncWithRpc Exception {}", e);
}
}
}
return response;
}
private <V> V internalCallSyncWithRpc(final Endpoint endpoint,
final HgCmdBase.BaseRequest request)
throws ExecutionException, InterruptedException, TimeoutException {
FutureClosureAdapter<V> response = new FutureClosureAdapter<>();
internalCallAsyncWithRpc(endpoint, request, response);
try {
return response.future.get(5000, TimeUnit.MILLISECONDS);
} catch (Exception e) {
throw e;
}
}
private <V> Future<V> internalCallAsyncWithRpc(final Endpoint endpoint,
final HgCmdBase.BaseRequest request,
final Closure done) {
final InvokeContext invokeCtx = null;
int[] retryCount = new int[]{0};
FutureClosureAdapter<V> response = new FutureClosureAdapter<>() {
@Override
public void run(Status status) {
done.run(status);
}
};
tryWithTimes(endpoint, request, response, invokeCtx, retryCount);
return response.future;
}
private <V> void internalCallAsyncWithRpc(final Endpoint endpoint,
final HgCmdBase.BaseRequest request,
final FutureClosureAdapter<V> closure) {
final InvokeContext invokeCtx = null;
int[] retryCount = new int[]{0};
tryWithTimes(endpoint, request, closure, invokeCtx, retryCount);
}
private <V> void tryWithTimes(Endpoint endpoint, HgCmdBase.BaseRequest request,
FutureClosureAdapter<V> closure,
InvokeContext invokeCtx,
int[] retryCount) {
InvokeCallback invokeCallback = (result, err) -> {
if (err == null) {
final HgCmdBase.BaseResponse response = (HgCmdBase.BaseResponse) result;
closure.setResponse((V) response);
} else {
tryWithThrowable(endpoint, request, closure, invokeCtx, retryCount, err);
}
};
try {
this.rpcClient.invokeAsync(endpoint, request, invokeCtx, invokeCallback,
this.rpcOptions.getRpcDefaultTimeout());
} catch (final Throwable err) {
tryWithThrowable(endpoint, request, closure, invokeCtx, retryCount, err);
}
}
private <V> void tryWithThrowable(Endpoint endpoint,
HgCmdBase.BaseRequest request,
FutureClosureAdapter<V> closure,
InvokeContext invokeCtx,
int[] retryCount, Throwable err) {
if (retryCount[0] >= MAX_RETRY_TIMES) {
closure.failure(err);
closure.run(new Status(-1, err.getMessage()));
} else {
retryCount[0]++;
try {
Thread.sleep(100L * retryCount[0]);
} catch (InterruptedException e) {
closure.run(new Status(-1, e.getMessage()));
}
tryWithTimes(endpoint, request, closure, invokeCtx, retryCount);
}
}
public interface PartitionAgent {
Endpoint getPartitionLeader(String graph, int partitionId);
}
}

228
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdProcessor.java Normal file
View File

@ -0,0 +1,228 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
import java.io.Serializable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.raft.RaftClosure;
import org.apache.hugegraph.store.raft.RaftOperation;
import org.apache.hugegraph.store.util.HgRaftError;
import com.alipay.sofa.jraft.rpc.RpcContext;
import com.alipay.sofa.jraft.rpc.RpcProcessor;
import com.alipay.sofa.jraft.rpc.RpcServer;
import lombok.extern.slf4j.Slf4j;
/**
* 快照同步rpc处理器，leader批量入库完成后，基于seqnum读取新增的kv,批量发送给follower.
*
* @param <T>
*/
@Slf4j
public class HgCmdProcessor<T extends HgCmdBase.BaseRequest> implements RpcProcessor<T> {
private final Class<?> requestClass;
private final HgStoreEngine engine;
public HgCmdProcessor(Class<?> requestClass, HgStoreEngine engine) {
this.requestClass = requestClass;
this.engine = engine;
}
public static void registerProcessor(final RpcServer rpcServer, final HgStoreEngine engine) {
rpcServer.registerProcessor(new HgCmdProcessor<>(GetStoreInfoRequest.class, engine));
rpcServer.registerProcessor(new HgCmdProcessor<>(BatchPutRequest.class, engine));
rpcServer.registerProcessor(new HgCmdProcessor<>(CleanDataRequest.class, engine));
rpcServer.registerProcessor(new HgCmdProcessor<>(UpdatePartitionRequest.class, engine));
rpcServer.registerProcessor(new HgCmdProcessor<>(CreateRaftRequest.class, engine));
rpcServer.registerProcessor(new HgCmdProcessor<>(DestroyRaftRequest.class, engine));
}
@Override
public void handleRequest(RpcContext rpcCtx, T request) {
HgCmdBase.BaseResponse response = null;
switch (request.magic()) {
case HgCmdBase.GET_STORE_INFO: {
response = new GetStoreInfoResponse();
handleGetStoreInfo((GetStoreInfoRequest) request, (GetStoreInfoResponse) response);
break;
}
case HgCmdBase.BATCH_PUT: {
response = new BatchPutResponse();
handleBatchPut((BatchPutRequest) request, (BatchPutResponse) response);
break;
}
case HgCmdBase.CLEAN_DATA: {
response = new CleanDataResponse();
handleCleanData((CleanDataRequest) request, (CleanDataResponse) response);
break;
}
case HgCmdBase.RAFT_UPDATE_PARTITION: {
response = new UpdatePartitionResponse();
handleUpdatePartition((UpdatePartitionRequest) request,
(UpdatePartitionResponse) response);
break;
}
case HgCmdBase.CREATE_RAFT: {
response = new CreateRaftResponse();
handleCreateRaft((CreateRaftRequest) request, (CreateRaftResponse) response);
break;
}
case HgCmdBase.DESTROY_RAFT: {
response = new DestroyRaftResponse();
handleDestroyRaft((DestroyRaftRequest) request, (DestroyRaftResponse) response);
break;
}
default: {
log.warn("HgCmdProcessor magic {} is not recognized ", request.magic());
}
}
rpcCtx.sendResponse(response);
}
@Override
public String interest() {
return this.requestClass.getName();
}
public void handleGetStoreInfo(GetStoreInfoRequest request, GetStoreInfoResponse response) {
response.setStore(engine.getPartitionManager().getStore());
response.setStatus(Status.OK);
}
public void handleUpdatePartition(UpdatePartitionRequest request,
UpdatePartitionResponse response) {
raftSyncTask(request, response, RaftOperation.RAFT_UPDATE_PARTITION);
}
public void handleBatchPut(BatchPutRequest request, BatchPutResponse response) {
raftSyncTask(request, response, RaftOperation.IN_WRITE_OP);
}
public void handleCleanData(CleanDataRequest request, CleanDataResponse response) {
raftSyncTask(request, response, RaftOperation.IN_CLEAN_OP);
}
public void handleCreateRaft(CreateRaftRequest request, CreateRaftResponse response) {
log.info("CreateRaftNode rpc call received, {}, {}", request.getPartitions(),
request.getConf());
request.getPartitions().forEach(partition -> {
engine.createPartitionEngine(new Partition(partition), request.getConf());
});
response.setStatus(Status.OK);
}
public void handleDestroyRaft(DestroyRaftRequest request, DestroyRaftResponse response) {
log.info("DestroyRaftNode rpc call received, partitionId={}", request.getPartitionId());
engine.destroyPartitionEngine(request.getPartitionId(), request.getGraphNames());
response.setStatus(Status.OK);
}
/**
* raft 通知副本同步执行
*
* @param request
* @param response
* @param op
*/
private void raftSyncTask(HgCmdBase.BaseRequest request, HgCmdBase.BaseResponse response,
final byte op) {
CountDownLatch latch = new CountDownLatch(1);
engine.addRaftTask(request.getGraphName(), request.getPartitionId(),
RaftOperation.create(op, request), new RaftClosure() {
@Override
public void run(com.alipay.sofa.jraft.Status status) {
Status responseStatus = Status.UNKNOWN;
switch (HgRaftError.forNumber(status.getCode())) {
case OK:
responseStatus = Status.OK;
break;
case NOT_LEADER:
responseStatus = Status.LEADER_REDIRECT;
break;
case NOT_LOCAL:
responseStatus = Status.NO_PARTITION;
break;
case WAIT_LEADER_TIMEOUT:
responseStatus = Status.WAIT_LEADER_TIMEOUT;
break;
default:
responseStatus.setMsg(status.getErrorMsg());
}
response.setStatus(responseStatus);
latch.countDown();
}
@Override
public void onLeaderChanged(Integer partId, Long storeId) {
RaftClosure.super.onLeaderChanged(partId, storeId);
response.addPartitionLeader(
new HgCmdBase.BaseResponse.PartitionLeader(partId, storeId));
}
});
try {
latch.await(1, TimeUnit.MINUTES);
} catch (InterruptedException e) {
log.info("handleBatchPut InterruptedException {}", e);
}
}
public enum Status implements Serializable {
UNKNOWN(-1, "unknown"),
OK(0, "ok"),
COMPLETE(0, "Transmission completed"),
INCOMPLETE(1, "Incomplete transmission"),
NO_PARTITION(10, "Partition not found"),
IO_ERROR(11, "io error"),
EXCEPTION(12, "exception"),
DOWN_SNAPSHOT_ERROR(13, "download snapshot error"),
LEADER_REDIRECT(14, "leader redirect"),
WAIT_LEADER_TIMEOUT(15, "Waiting for leader timeout"),
ABORT(100, "Transmission aborted");
private final int code;
private String msg;
Status(int code, String msg) {
this.code = code;
this.msg = msg;
}
public int getCode() {
return this.code;
}
public String getMsg() {
return this.msg;
}
public Status setMsg(String msg) {
this.msg = msg;
return this;
}
public boolean isOK() {
return this.code == 0;
}
}
}

36
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/UpdatePartitionRequest.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
import org.apache.hugegraph.pd.grpc.Metapb;
import lombok.Data;
@Data
public class UpdatePartitionRequest extends HgCmdBase.BaseRequest {
private int startKey;
private int endKey;
private Metapb.PartitionState workState;
@Override
public byte magic() {
return HgCmdBase.RAFT_UPDATE_PARTITION;
}
}

22
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/UpdatePartitionResponse.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.cmd;
public class UpdatePartitionResponse extends HgCmdBase.BaseResponse {
}

66
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/DeletedFileManager.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.store.meta.base.GlobalMetaStore;
import org.apache.hugegraph.store.options.MetadataOptions;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DeletedFileManager extends GlobalMetaStore {
public DeletedFileManager(MetadataOptions options) {
super(options);
}
public void load() {
byte[] key = MetadataKeyHelper.getDeletedFilePrefix();
List<RocksDBSession.BackendColumn> columns = scan(key);
for (RocksDBSession.BackendColumn column : columns) {
String filePath = new String(column.value, StandardCharsets.UTF_8);
try {
if (new File(filePath).exists()) {
FileUtils.deleteDirectory(new File(filePath));
log.warn("Delete legacy files {}", filePath);
removeDeletedFile(filePath);
}
} catch (IOException e) {
log.error("Delete legacy files {} exception", filePath, e);
}
}
}
public void addDeletedFile(String path) {
byte[] key = MetadataKeyHelper.getDeletedFileKey(path);
put(key, path.getBytes(StandardCharsets.UTF_8));
}
public void removeDeletedFile(String path) {
byte[] key = MetadataKeyHelper.getDeletedFileKey(path);
delete(key);
}
}

63
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Graph.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import org.apache.hugegraph.pd.grpc.Metapb;
import lombok.Data;
@Data
public class Graph implements Cloneable {
private String graphName;
private Metapb.Graph graph;
public Graph() {
}
public Graph(Metapb.Graph protoObj) {
graphName = protoObj.getGraphName();
this.graph = protoObj;
}
public Metapb.Graph getProtoObj() {
// return Metapb.Graph.newBuilder()
// .setGraphName(graphName)
// .build();
return this.graph;
}
public void setProtoObj(Metapb.Graph protoObj) {
// return Metapb.Graph.newBuilder()
// .setGraphName(graphName)
// .build();
this.graph = protoObj;
}
@Override
public Graph clone() {
try {
return (Graph) super.clone();
} catch (CloneNotSupportedException e) {
e.printStackTrace();
}
return null;
}
}

180
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphIdManager.java Normal file
View File

@ -0,0 +1,180 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hugegraph.store.meta.base.DBSessionBuilder;
import org.apache.hugegraph.store.meta.base.PartitionMetaStore;
import org.apache.hugegraph.store.util.HgStoreException;
import com.google.protobuf.Int64Value;
/**
* GraphId管理器，维护一个自增循环ID，负责管理GraphName和GraphId的映射
*/
public class GraphIdManager extends PartitionMetaStore {
protected static final String GRAPH_ID_PREFIX = "@GRAPH_ID@";
protected static int maxGraphID = 65535;
static Object graphIdLock = new Object();
static Object cidLock = new Object();
final DBSessionBuilder sessionBuilder;
final int partitionId;
// public long getGraphId(String graphName) {
// if (!graphIdCache.containsKey(graphName)) {
// synchronized (graphIdLock) {
// if (!graphIdCache.containsKey(graphName)) {
// byte[] key = MetadataKeyHelper.getGraphIDKey(graphName);
// Int64Value id = get(Int64Value.parser(), key);
// if (id == null) {
// id = Int64Value.of(getCId(GRAPH_ID_PREFIX, maxGraphID));
// if (id.getValue() == -1) {
// throw new HgStoreException(HgStoreException.EC_FAIL,
// "The number of graphs exceeds the maximum 65535");
// }
// put(key, id);
// flush();
// }
// graphIdCache.put(graphName, id.getValue());
// }
// }
// }
// return graphIdCache.get(graphName);
// }
private final Map<String, Long> graphIdCache = new ConcurrentHashMap<>();
public GraphIdManager(DBSessionBuilder sessionBuilder, int partitionId) {
super(sessionBuilder, partitionId);
this.sessionBuilder = sessionBuilder;
this.partitionId = partitionId;
}
/**
* 获取一个图的id
*/
public long getGraphId(String graphName) {
Long l = graphIdCache.get(graphName);
if (l == null) {
synchronized (graphIdLock) {
if ((l = graphIdCache.get(graphName)) == null) {
byte[] key = MetadataKeyHelper.getGraphIDKey(graphName);
Int64Value id = get(Int64Value.parser(), key);
if (id == null) {
id = Int64Value.of(getCId(GRAPH_ID_PREFIX, maxGraphID));
if (id.getValue() == -1) {
throw new HgStoreException(HgStoreException.EC_FAIL,
"The number of graphs exceeds the maximum " +
"65535");
}
put(key, id);
flush();
}
l = id.getValue();
graphIdCache.put(graphName, l);
}
}
}
return l;
}
/**
* 释放一个图id
*/
public long releaseGraphId(String graphName) {
long gid = getGraphId(graphName);
synchronized (graphIdLock) {
graphIdCache.remove(graphName);
byte[] key = MetadataKeyHelper.getGraphIDKey(graphName);
delete(key);
delCId(GRAPH_ID_PREFIX, gid);
flush();
}
return gid;
}
/**
* 获取自增循环不重复id, 达到上限后从0开始自增
*
* @param key key
* @param max id上限，达到该值后，重新从0开始自增
* @return id
*/
protected long getCId(String key, long max) {
byte[] cidNextKey = MetadataKeyHelper.getCidKey(key);
synchronized (cidLock) {
Int64Value value = get(Int64Value.parser(), cidNextKey);
long current = value != null ? value.getValue() : 0L;
long last = current == 0 ? max - 1 : current - 1;
// 查找一个未使用的cid
List<Int64Value> ids =
scan(Int64Value.parser(), genCIDSlotKey(key, current), genCIDSlotKey(key, max));
for (Int64Value id : ids) {
if (current == id.getValue()) {
current++;
} else {
break;
}
}
if (current == max) {
current = 0;
ids = scan(Int64Value.parser(), genCIDSlotKey(key, current),
genCIDSlotKey(key, last));
for (Int64Value id : ids) {
if (current == id.getValue()) {
current++;
} else {
break;
}
}
}
if (current == last) {
return -1;
}
// 保存当前id，标记已被使用
put(genCIDSlotKey(key, current), Int64Value.of(current));
// 保存下一次遍历的id
put(cidNextKey, Int64Value.of(current + 1));
return current;
}
}
/**
* 返回已使用Cid的key
*/
private byte[] genCIDSlotKey(String key, long value) {
byte[] keySlot = MetadataKeyHelper.getCidSlotKeyPrefix(key);
ByteBuffer buf = ByteBuffer.allocate(keySlot.length + Long.SIZE);
buf.put(keySlot);
buf.putLong(value);
return buf.array();
}
/**
* 删除一个循环id，释放id值
*/
protected void delCId(String key, long cid) {
delete(genCIDSlotKey(key, cid));
}
}

86
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphManager.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.meta.base.GlobalMetaStore;
import org.apache.hugegraph.store.options.MetadataOptions;
import org.apache.hugegraph.store.pd.PdProvider;
public class GraphManager extends GlobalMetaStore {
private final PdProvider pdProvider;
private final Map<String, Graph> graphs;
public GraphManager(MetadataOptions options, PdProvider pdProvider) {
super(options);
this.graphs = new ConcurrentHashMap<>();
this.pdProvider = pdProvider;
this.pdProvider.setGraphManager(this);
}
/**
* 修改图
* 此处不加锁，要求graph是被克隆的，进制修改原始对象
*
* @param graph
* @return
*/
public Graph updateGraph(Graph graph) {
this.graphs.put(graph.getGraphName(), graph);
byte[] key = MetadataKeyHelper.getGraphKey(graph.getGraphName());
if (graph.getProtoObj() != null) {
put(key, graph.getProtoObj().toByteArray());
}
return graph;
}
public void load() {
byte[] key = MetadataKeyHelper.getGraphKeyPrefix();
List<Metapb.Graph> values = scan(Metapb.Graph.parser(), key);
values.forEach(graph -> {
graphs.put(graph.getGraphName(), new Graph(graph));
});
}
public Map<String, Graph> getGraphs() {
return graphs;
}
public Graph getGraph(String graphName) {
return graphs.get(graphName);
}
public Graph getCloneGraph(String graphName) {
if (graphs.containsKey(graphName)) {
return graphs.get(graphName).clone();
}
return new Graph();
}
public Graph removeGraph(String graphName) {
byte[] key = MetadataKeyHelper.getGraphKey(graphName);
delete(key);
return graphs.remove(graphName);
}
}

317
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/MetadataKeyHelper.java Normal file
View File

@ -0,0 +1,317 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import java.nio.charset.StandardCharsets;
public class MetadataKeyHelper {
private static final char DELIMITER = '/';
private static final String HUGEGRAPH = "HUGEGRAPH";
private static final String STORE = "STORE";
private static final String PARTITION = "PARTITION";
private static final String TASK = "TASK";
private static final String ASYNC_TASK = "A_TASK";
private static final String INSTRUCTION_TASK = "INS_TASK";
private static final String TASK_DONE = "TASK_DONE";
private static final String GRAPH = "GRAPH";
private static final String PARTITION_STORE = "PARTITION_STORE";
private static final String PARTITION_RAFT = "PARTITION_Raft";
private static final String DELETED_FILE = "DELETED_FILE";
private static final String SHARD_GROUP = "SHARDGROUP";
private static final String CID_PREFIX = "CID";
private static final String CID_SLOT_PREFIX = "CID_SLOT";
private static final String GRAPH_ID_PREFIX = "GRAPH_ID";
public static byte[] getPartitionKey(String graph, Integer partId) {
// HUGEGRAPH/Partition/{graph}/partId
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(PARTITION).append(DELIMITER)
.append(graph).append(DELIMITER)
.append(partId)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getPartitionPrefixKey(String graph) {
// HUGEGRAPH/Partition/{graph}/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(PARTITION).append(DELIMITER)
.append(graph).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
/**
* 查询分区内的所有partition prefix， 不包含 graph name
*
* @return
*/
public static byte[] getPartitionPrefixKey() {
// HUGEGRAPH/Partition/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(PARTITION).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getShardGroupKey(int partitionId) {
// HUGEGRAPH/SHARDGROUP/{partition_id}
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(SHARD_GROUP).append(DELIMITER)
.append(partitionId)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getGraphKey(String graph) {
// HUGEGRAPH/Graph/{graph}
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(GRAPH).append(DELIMITER)
.append(graph)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getGraphKeyPrefix() {
// HUGEGRAPH/Graph/{graph}
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(GRAPH).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getStoreKey() {
// HUGEGRAPH/STORE/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(STORE)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getTaskKey(int partId, String type, long taskId) {
// HUGEGRAPH/TASK/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(TASK).append(DELIMITER)
.append(partId).append(DELIMITER)
.append(type).append(DELIMITER)
.append(String.format("%016x", taskId))
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getInstructionIdKey(long taskId) {
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(INSTRUCTION_TASK).append(DELIMITER)
.append(taskId).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getAsyncTaskKey(int partId, String graphName, String taskId) {
// HUGEGRAPH/A_TASK/ part id / graphName / task id
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(ASYNC_TASK).append(DELIMITER)
.append(partId).append(DELIMITER)
.append(graphName).append(DELIMITER)
.append(taskId)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getAsyncTaskPrefix(int partId, String graphName) {
// HUGEGRAPH/A_TASK/ part id / graphName / task id
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(ASYNC_TASK).append(DELIMITER)
.append(partId).append(DELIMITER)
.append(graphName).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getTaskPrefix(int partId, String type) {
// HUGEGRAPH/TASK/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(TASK).append(DELIMITER)
.append(partId).append(DELIMITER)
.append(type).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getTaskPrefix(int partId) {
// HUGEGRAPH/TASK/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(TASK).append(DELIMITER)
.append(partId).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getTaskPrefix() {
// HUGEGRAPH/TASK/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(TASK).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getDoneTaskKey(long taskId) {
// HUGEGRAPH/TASK/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(TASK_DONE).append(DELIMITER)
.append(String.format("%016x", taskId))
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getPartitionStoreKey(int partId) {
// HUGEGRAPH/TASK/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(PARTITION_STORE).append(DELIMITER)
.append(partId)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getPartitionStorePrefix() {
// HUGEGRAPH/TASK/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(PARTITION_STORE).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getPartitionRaftKey(int partId) {
// HUGEGRAPH/TASK/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(PARTITION_RAFT).append(DELIMITER)
.append(partId)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getPartitionRaftPrefix() {
// HUGEGRAPH/TASK/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(PARTITION_RAFT).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getDeletedFileKey(String filePath) {
// HUGEGRAPH/TASK/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(DELETED_FILE).append(DELIMITER)
.append(filePath)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getDeletedFilePrefix() {
// HUGEGRAPH/TASK/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(DELETED_FILE).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getCidKey(String name) {
// HUGEGRAPH/CID/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(CID_PREFIX).append(DELIMITER)
.append(name)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getCidSlotKeyPrefix(String name) {
// HUGEGRAPH/CID_SLOT/
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(CID_SLOT_PREFIX).append(DELIMITER)
.append(name).append(DELIMITER)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
public static byte[] getGraphIDKey(String graph) {
// HUGEGRAPH/Graph/{graph}
String key = StringBuilderHelper.get()
.append(HUGEGRAPH).append(DELIMITER)
.append(GRAPH_ID_PREFIX).append(DELIMITER)
.append(graph)
.toString();
return key.getBytes(StandardCharsets.UTF_8);
}
static class StringBuilderHelper {
private static final int DISCARD_LIMIT = 1024 << 3; // 8k
private static final ThreadLocal<StringBuilderHolder> holderThreadLocal = ThreadLocal
.withInitial(StringBuilderHolder::new);
public static StringBuilder get() {
final StringBuilderHolder holder = holderThreadLocal.get();
return holder.getStringBuilder();
}
public static void truncate() {
final StringBuilderHolder holder = holderThreadLocal.get();
holder.truncate();
}
private static class StringBuilderHolder {
private final StringBuilder buf = new StringBuilder();
private StringBuilder getStringBuilder() {
truncate();
return buf;
}
private void truncate() {
buf.setLength(0);
}
}
}
}

95
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Partition.java Normal file
View File

@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.PartitionEngine;
import lombok.Data;
@Data
public class Partition implements Cloneable {
private int id; // region id
private String graphName;
// Region key range [startKey, endKey)
private long startKey;
private long endKey;
private long version;
// shardlist版本，shardlist每次改变加1
// private long confVer;
private Metapb.PartitionState workState;
// private PartitionRole role;
// private List<Metapb.Shard> shardsList;
// exclusive
public Partition() {
workState = Metapb.PartitionState.PState_Normal;
}
public Partition(Metapb.Partition protoObj) {
id = protoObj.getId();
graphName = protoObj.getGraphName();
startKey = protoObj.getStartKey();
endKey = protoObj.getEndKey();
// shardsList = protoObj.getShardsList();
workState = protoObj.getState();
version = protoObj.getVersion();
// confVer = protoObj.getConfVer();
if (workState == Metapb.PartitionState.UNRECOGNIZED ||
workState == Metapb.PartitionState.PState_None) {
workState = Metapb.PartitionState.PState_Normal;
}
}
public boolean isLeader() {
PartitionEngine engine = HgStoreEngine.getInstance().getPartitionEngine(id);
return engine != null && engine.isLeader();
}
public Metapb.Partition getProtoObj() {
return Metapb.Partition.newBuilder()
.setId(id)
.setVersion(version)
// .setConfVer(confVer)
.setGraphName(graphName)
.setStartKey(startKey)
.setEndKey(endKey)
.setState(workState)
// .addAllShards(shardsList)
.build();
}
@Override
public Partition clone() {
Partition obj = null;
try {
obj = (Partition) super.clone();
} catch (CloneNotSupportedException e) {
e.printStackTrace();
}
return obj;
}
@Override
public String toString() {
return getProtoObj().toString();
}
}

921
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionManager.java Normal file
View File

@ -0,0 +1,921 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import java.io.File;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Consumer;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.MetaTask;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.business.BusinessHandlerImpl;
import org.apache.hugegraph.store.cmd.UpdatePartitionRequest;
import org.apache.hugegraph.store.cmd.UpdatePartitionResponse;
import org.apache.hugegraph.store.meta.base.GlobalMetaStore;
import org.apache.hugegraph.store.options.HgStoreEngineOptions;
import org.apache.hugegraph.store.options.MetadataOptions;
import org.apache.hugegraph.store.pd.PdProvider;
import org.apache.hugegraph.store.util.PartitionMetaStoreWrapper;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
import com.alipay.sofa.jraft.core.ElectionPriority;
import lombok.extern.slf4j.Slf4j;
/**
* Partition对象管理策略，每次修改需要克隆一份，并且版本号递增
*/
@Slf4j
public class PartitionManager extends GlobalMetaStore {
private static final Logger LOG = Log.logger(PartitionManager.class);
private final PdProvider pdProvider;
private final GraphManager graphManager;
private final StoreMetadata storeMetadata;
private final DeletedFileManager deletedFileManager;
private final boolean useRaft;
private final HgStoreEngineOptions options;
private final List<PartitionChangedListener> partitionChangedListeners;
// 读写锁对象
private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
private final PartitionMetaStoreWrapper wrapper = new PartitionMetaStoreWrapper();
// 记录本机所有的分区信息，与rocksdb存储保持一致
private Map<String, Map<Integer, Partition>> partitions;
public PartitionManager(PdProvider pdProvider, HgStoreEngineOptions options) {
super(new MetadataOptions() {{
setDataPath(options.getDataPath());
setRaftPath(options.getRaftPath());
}});
this.options = options;
this.pdProvider = pdProvider;
partitions = new ConcurrentHashMap<>();
storeMetadata = new StoreMetadata(getOptions());
graphManager = new GraphManager(getOptions(), pdProvider);
deletedFileManager = new DeletedFileManager(getOptions());
this.useRaft = true;
partitionChangedListeners = new ArrayList<>();
}
public void load() {
storeMetadata.load();
graphManager.load();
deletedFileManager.load();
}
public void loadPartition() {
loadPartitions();
}
public DeletedFileManager getDeletedFileManager() {
return deletedFileManager;
}
public PdProvider getPdProvider() {
return pdProvider;
}
public StoreMetadata getStoreMetadata() {
return storeMetadata;
}
public void addPartitionChangedListener(PartitionChangedListener listener) {
partitionChangedListeners.add(listener);
}
/**
* 判断存储路径为分区id或者分区id_开头
*
* @param detections dir list
* @param partitionId partition id
* @param checkLogDir : 是否包含子目录 log ( raft snapshot 和log 分离，需要进一步检查)
* @return true if contains partition id, otherwise false
*/
private Boolean checkPathContains(File[] detections, int partitionId, boolean checkLogDir) {
String partitionDirectory = String.format("%05d", partitionId);
for (int x = 0; x < detections.length; x++) {
// 一定是以分区id命名的文件夹下
if (detections[x].isDirectory()) {
String tmp = detections[x].getName();
if (tmp.equals(partitionDirectory) || tmp.startsWith(partitionDirectory + "_")) {
if (checkLogDir) {
String logDir = detections[x].getAbsolutePath() + "/log";
if (new File(logDir).exists()) {
return true;
}
} else {
return true;
}
}
}
}
return false;
}
/**
* 根据配置文件根目录，循环查找分区的存储路径
* 根据约定db数据在dataPath/db/分区id目录，raft数据在dataPath/raft/分区id目录
* 检测分区存储文件夹是否存在
*/
private Boolean resetPartitionPath(int partitionId) {
List<String> dataPaths = Arrays.asList(this.options.getDataPath().split(","));
List<String> raftPaths = Arrays.asList(this.options.getRaftPath().split(","));
boolean isDataOk = false;
boolean isRaftOk = false;
// 检查 db 目录
for (int i = 0; i < dataPaths.size(); i++) {
String dbPath = Paths.get(dataPaths.get(i),
HgStoreEngineOptions.DB_Path_Prefix).toAbsolutePath()
.toString();
File dbFile = new File(dbPath);
if (dbFile.exists()) {
File[] dbFiles = dbFile.listFiles();
if (this.checkPathContains(dbFiles, partitionId, false)) {
Metapb.PartitionStore location = storeMetadata.getPartitionStore(partitionId);
if (!location.getStoreLocation().equals(dataPaths.get(i))) {
Metapb.PartitionStore newLocation = location.toBuilder()
.setStoreLocation(
dataPaths.get(i))
.build();
storeMetadata.savePartitionStore(newLocation);
}
isDataOk = true;
break;
}
}
}
// 检查 raft目录
for (int i = 0; i < raftPaths.size(); i++) {
String raftPath = Paths.get(raftPaths.get(i),
HgStoreEngineOptions.Raft_Path_Prefix).toAbsolutePath()
.toString();
File raftFile = new File(raftPath);
if (raftFile.exists()) {
File[] raftFiles = raftFile.listFiles();
if (this.checkPathContains(raftFiles, partitionId, true)) {
Metapb.PartitionRaft location = storeMetadata.getPartitionRaft(partitionId);
// 兼容版本升级
if (location == null ||
!Objects.equals(location.getRaftLocation(), raftPaths.get(i))) {
Metapb.PartitionRaft newLocation = Metapb.PartitionRaft.newBuilder()
.setPartitionId(
partitionId)
.setRaftLocation(
raftPaths.get(
i))
.build();
storeMetadata.savePartitionRaft(newLocation);
}
isRaftOk = true;
break;
}
}
}
return isDataOk && isRaftOk;
}
/**
* 从本地storage中读取分区
*/
private void loadPartitions() {
byte[] key = MetadataKeyHelper.getPartitionPrefixKey();
long storeId = getStore().getId();
// 从data path中读取 partition
// 记录有哪些分区
var partIds = new HashSet<Integer>();
for (String path : this.options.getDataPath().split(",")) {
File[] dirs = new File(path + "/" + HgStoreEngineOptions.DB_Path_Prefix).listFiles();
if (dirs == null) {
continue;
}
for (File f : dirs) {
if (f.isDirectory()) {
try {
partIds.add(Integer.parseInt(f.getName().split("_")[0]));
} catch (Exception e) {
log.error("find illegal dir {} in data path, error:{}", f.getName(),
e.getMessage());
}
}
}
}
// 一次按照分区读取
for (int partId : partIds) {
if (!resetPartitionPath(partId)) {
log.error("partition " + partId + " Directory not exists,options " +
this.options.getDataPath());
continue;
}
for (var metaPart : wrapper.scan(partId, Metapb.Partition.parser(), key)) {
var graph = metaPart.getGraphName();
var pdPartition = pdProvider.getPartitionByID(graph, metaPart.getId());
boolean isLegeal = false;
var shards = pdProvider.getShardGroup(metaPart.getId()).getShardsList();
if (pdPartition != null) {
// 判断是否包含本store id
if (shards.stream().anyMatch(s -> s.getStoreId() == storeId)) {
isLegeal = true;
}
}
if (isLegeal) {
if (!partitions.containsKey(graph)) {
partitions.put(graph, new ConcurrentHashMap<>());
}
Partition partition = new Partition(metaPart);
partition.setWorkState(Metapb.PartitionState.PState_Normal); // 启动恢复工作状态
partitions.get(graph).put(partition.getId(), partition);
log.info("load partition : {} -{}", partition.getGraphName(),
partition.getId());
} else {
// 无效
// removePartitionFromLocalDb(graph, partId);
// var businessHandler = HgStoreEngine.getInstance().getBusinessHandler();
// businessHandler.truncate(graph, partId);
// businessHandler.dbCompaction(graph, partId);
log.error("partition {}-{} is illegal. store id {} not in valid shard group:{}",
graph, partId, getStore().getId(), shards2Peers(shards));
System.exit(0);
}
}
}
}
public List<Metapb.Partition> loadPartitionsFromDb(int partitionId) {
byte[] key = MetadataKeyHelper.getPartitionPrefixKey();
return wrapper.scan(partitionId, Metapb.Partition.parser(), key);
}
/**
* 从PD同步分区，并删除本地多余的分区
* 同步过程中，新增分区需要保存到本地，已有的分区信息与本地进行合并
*/
public void syncPartitionsFromPD(Consumer<Partition> delCallback) throws PDException {
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
try {
List<Partition> partListFrPD =
pdProvider.getPartitionsByStore(storeMetadata.getStore().getId());
Map<String, Map<Integer, Partition>> graphPtFrpd = new HashMap<>();
partListFrPD.forEach(partition -> {
if (!graphPtFrpd.containsKey(partition.getGraphName())) {
graphPtFrpd.put(partition.getGraphName(), new HashMap<>());
}
if (isLocalPartition(partition)) {
graphPtFrpd.get(partition.getGraphName()).put(partition.getId(), partition);
}
});
// 遍历本地图，删除本地多余，追加新的。
partitions.forEach((graphName, v) -> {
Map<Integer, Partition> partitionsFrpd = graphPtFrpd.get(graphName);
v.forEach((id, pt) -> {
if (partitionsFrpd == null || !partitionsFrpd.containsKey(id)) {
// 本地的分区，pd已不存在，需要删除
delCallback.accept(pt);
removePartition(pt.getGraphName(), pt.getId());
} else {
// 修改shard信息
// Partition ptFrpd = partitionsFrpd.get(id);
// pt.setShardsList(ptFrpd.getShardsList());
savePartition(pt, true, true);
}
});
if (partitionsFrpd != null) {
partitionsFrpd.forEach((id, pt) -> {
if (!v.containsKey(id)) {
// 新增的分区
savePartition(pt, true);
}
});
}
});
partitions = graphPtFrpd;
} finally {
writeLock.unlock();
}
}
public Partition changeState(Partition partition, Metapb.PartitionState state) {
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
try {
partition = findPartition(partition.getGraphName(), partition.getId());
partition.setWorkState(state);
savePartition(partition, false);
} finally {
writeLock.unlock();
}
return partition;
}
public Partition changeKeyRange(Partition partition, int startKey, int endKey) {
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
try {
partition = findPartition(partition.getGraphName(), partition.getId());
partition.setStartKey(startKey);
partition.setEndKey(endKey);
savePartition(partition, false, true);
} finally {
writeLock.unlock();
}
return partition;
}
public Partition updatePartition(Metapb.Partition partition, boolean updateRange) {
return updatePartition(new Partition(partition), updateRange);
}
/**
* 增加partition对象
*
* @param partition
* @return
*/
public Partition updatePartition(Partition partition, boolean updateRange) {
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
try {
savePartition(partition, true, updateRange);
} finally {
writeLock.unlock();
}
return partition;
}
public void updatePartitionRangeOrState(UpdatePartitionRequest req) {
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
try {
Partition partition = findPartition(req.getGraphName(), req.getPartitionId());
if (req.getStartKey() >= 0 && req.getEndKey() > 0
&& partition.getStartKey() != req.getStartKey() &&
partition.getEndKey() != req.getEndKey()) {
changeKeyRange(partition, req.getStartKey(), req.getEndKey());
}
if (req.getWorkState() != null) {
changeState(partition, req.getWorkState());
}
} finally {
writeLock.unlock();
}
}
/**
* 强制更新 partition，不校验 version
*
* @param partition
* @return
*/
public Partition loadPartitionFromSnapshot(Partition partition) {
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
try {
savePartition(partition, true, true);
} finally {
writeLock.unlock();
}
return partition;
}
/**
* 查找属于本机的Partiton，优先从本地查找，本地未找到，询问pd。
*
* @param graph
* @param partId
* @return
*/
public Partition findPartition(String graph, Integer partId) {
Partition partition = null;
if (partitions.containsKey(graph)) {
partition = partitions.get(graph).get(partId);
}
if (partition == null) {
partition = pdProvider.getPartitionByID(graph, partId);
if (partition != null) {
if (isLocalPartition(partition)) {
// 属于本机的partion，保存partition
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
try {
savePartition(partition, true);
} finally {
writeLock.unlock();
}
} else {
LOG.error("Partition {}-{} does not belong to local store! store id{} \n {}",
graph, partId,
storeMetadata.getStore().getId(), partition.getProtoObj());
return null;
}
} else {
LOG.error("Partition {}-{} is not Found! ", graph, partId);
return null;
}
}
return partitions.get(graph).get(partId);
}
public int getPartitionIdByCode(String graph, int code) {
return pdProvider.getPartitionByCode(graph, code).getId();
}
/**
* 从pd获取拉取分区信息，并和本地的分区信息进行合并。leader和shardList取自本地
*/
public Partition getPartitionFromPD(String graph, int partId) {
pdProvider.invalidPartitionCache(graph, partId);
Partition partition = pdProvider.getPartitionByID(graph, partId);
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
try {
if (partitions.containsKey(graph)) {
Partition local = partitions.get(graph).get(partId);
if (local != null) {
//更新本地的key范围，保证pd和本地分区信息的一致性
local.setStartKey(partition.getStartKey());
local.setEndKey(partition.getEndKey());
savePartition(local, true, true);
}
partition = local;
}
} finally {
writeLock.unlock();
}
return partition;
}
/**
* 是否是本地的分区
* 对于批处理入库，只有leader才属于本地
*
* @param partition
* @return
*/
public boolean isLocalPartition(Partition partition) {
boolean isLocal = false;
var shardGroup = getShardGroup(partition.getId());
if (shardGroup != null) {
for (Shard shard : shardGroup.getShards()) {
if (shard.getStoreId() == storeMetadata.getStore().getId()) {
isLocal = true;
break;
}
}
}
return isLocal;
}
/**
* 是否是本地的分区
* 对于批处理入库，只有leader才属于本地
*
* @return
*/
public boolean isLocalPartition(int partId) {
return pdProvider.isLocalPartition(storeMetadata.getStore().getId(), partId);
}
/**
* 存储partition信息,同步保存到内存和rocksdb
* 不更新key range
*/
private void savePartition(Partition partition, Boolean changeLeader) {
savePartition(partition, changeLeader, false);
}
/**
* 保存partition 信息
*
* @param partition partition
* @param changeLeader is change leader
* @param changeRange update start and end key if yes.
* using key range in local if no and partition key exists
*/
private void savePartition(Partition partition, Boolean changeLeader, Boolean changeRange) {
String graphName = partition.getGraphName();
Integer partId = partition.getId();
byte[] key = MetadataKeyHelper.getPartitionKey(graphName, partId);
if (!changeRange) {
var local = wrapper.get(partId, key, Metapb.Partition.parser());
if (local != null) {
partition.setStartKey(local.getStartKey());
partition.setEndKey(local.getEndKey());
}
}
if (!partitions.containsKey(graphName)) {
partitions.put(graphName, new ConcurrentHashMap<>());
}
partitions.get(graphName).put(partition.getId(), partition);
// put(key, partition.getProtoObj().toByteArray());
wrapper.put(partId, key, partition.getProtoObj().toByteArray());
Graph graph = new Graph();
graph.setGraphName(partition.getGraphName());
graphManager.updateGraph(graph);
// 更新PD cache，后序优化，store不依赖pdclient cache
pdProvider.updatePartitionCache(partition, changeLeader);
partitionChangedListeners.forEach(listener -> {
listener.onChanged(partition); // 通知raft，进行同步分区信息同步
});
}
/**
* 更新shard group到db, 同时更新shardGroups对象
*
* @param shardGroup
*/
public void updateShardGroup(ShardGroup shardGroup) {
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
wrapper.put(shardGroup.getId(),
MetadataKeyHelper.getShardGroupKey(shardGroup.getId()),
shardGroup.getProtoObj().toByteArray());
writeLock.unlock();
}
/**
* 查找 partition id对应的shard group。
* 依次从 raft node/local db/ pd 读取。
*
* @param partitionId
* @return
*/
public ShardGroup getShardGroup(int partitionId) {
var partitionEngine = HgStoreEngine.getInstance().getPartitionEngine(partitionId);
if (partitionEngine != null) {
return partitionEngine.getShardGroup();
}
Metapb.ShardGroup shardGroup =
wrapper.get(partitionId, MetadataKeyHelper.getShardGroupKey(partitionId),
Metapb.ShardGroup.parser());
if (shardGroup == null) {
shardGroup = pdProvider.getShardGroup(partitionId);
if (shardGroup != null) {
// local not found, write back to db from pd
wrapper.put(partitionId, MetadataKeyHelper.getShardGroupKey(partitionId),
shardGroup.toByteArray());
} else {
log.error("get shard group {} from pd failed", partitionId);
}
}
return ShardGroup.from(shardGroup);
}
public Partition removePartition(String graphName, Integer partId) {
log.info("partition manager: remove partition : {}-{}", graphName, partId);
if (partitions.containsKey(graphName)) {
pdProvider.invalidPartitionCache(graphName, partId);
removePartitionFromLocalDb(graphName, partId);
Partition partition = partitions.get(graphName).remove(partId);
log.info("partition manager: remove partition, partition: {}", partition);
if (partitions.get(graphName).size() == 0) {
log.info("remove graph {}", graphName);
graphManager.removeGraph(graphName);
}
return partition;
}
return null;
}
private void removePartitionFromLocalDb(String graphName, Integer partId) {
byte[] key = MetadataKeyHelper.getPartitionKey(graphName, partId);
// delete(key);
wrapper.delete(partId, key);
}
/**
* 删除图数据，删除本地数据，并删除PD上的分区信息
*/
public Partition deletePartition(String graphName, Integer partId) {
removePartition(graphName, partId);
return pdProvider.delPartition(graphName, partId);
}
// 获取本地Store信息
public Store getStore() {
return storeMetadata.getStore();
}
// 注册会修改StoreId，需要重置
public void setStore(Store store) {
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
try {
storeMetadata.save(store);
} finally {
writeLock.unlock();
}
}
public Store getStore(Long storeId) {
return pdProvider.getStoreByID(storeId);
}
public Map<String, Map<Integer, Partition>> getPartitions() {
return partitions;
}
public Map<String, Partition> getPartitions(int partitionId) {
Map<String, Partition> result = new HashMap<>();
this.partitions.forEach((k, v) -> {
v.forEach((k1, v1) -> {
if (k1 == partitionId) {
result.put(k, v1);
}
});
}
);
return result;
}
public Partition getPartition(String graphName, int partitionId) {
return this.partitions.getOrDefault(graphName, new HashMap<>())
.getOrDefault(partitionId, null);
}
public List<Partition> getPartitionList(int partitionId) {
List<Partition> pts = new ArrayList<>();
getPartitions(partitionId).forEach((k, v) -> {
pts.add(findPartition(k, v.getId()));
});
return pts;
}
public boolean hasPartition(String graphName, int partitionId) {
return this.partitions.getOrDefault(graphName, new HashMap<>()).containsKey(partitionId);
}
/**
* 获取图在本机中所有Leader 分区
*
* @param graph
* @return
*/
public List<Integer> getLeaderPartitionIds(String graph) {
List<Integer> ids = new ArrayList<>();
if (partitions.containsKey(graph)) {
partitions.get(graph).forEach((k, v) -> {
if (!useRaft || v.isLeader()) {
ids.add(k);
}
});
}
return ids;
}
/**
* 生成分区peer字符串，包含优先级信息 *
*
* @param shardGroup
* @return
*/
public List<String> getPartitionPeers(ShardGroup shardGroup) {
List<String> peers = new ArrayList<>();
final int decayPriorityGap = 10;
int priority = 100;
if (shardGroup != null) {
for (Shard shard : shardGroup.getShards()) {
Store store = getStore(shard.getStoreId());
if (store != null && !store.getRaftAddress().isEmpty()) {
peers.add(store.getRaftAddress() + "::" + priority);
final int gap = Math.max(decayPriorityGap, (priority / 5));
priority = Math.max(ElectionPriority.MinValue, (priority - gap));
}
}
}
return peers;
}
public List<String> shards2Peers(List<Metapb.Shard> shards) {
List<String> peers = new ArrayList<>();
shards.forEach(s -> {
peers.add(getStore(s.getStoreId()).getRaftAddress());
});
return peers;
}
/**
* 是否是本地store
*
* @param store
* @return
*/
public boolean isLocalStore(Store store) {
return storeMetadata.getStore().getId() == store.getId();
}
public PartitionRole getLocalRoleFromShard(Partition partition) {
return partition.isLeader() ? PartitionRole.LEADER : PartitionRole.FOLLOWER;
}
/**
* 修改分区角色
*/
public Partition changeLeader(Partition pt, List<Metapb.Shard> shards, long term) {
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
try {
Partition partition = findPartition(pt.getGraphName(), pt.getId());
if (partition != null) {
// partition.setShardsList(shards);
partition.setVersion(term);
savePartition(partition, true);
}
return partition;
} finally {
writeLock.unlock();
}
}
/**
* 根据raft peers清单，重建shardList
*/
public Partition changeShards(Partition pt, List<Metapb.Shard> shards) {
Lock writeLock = readWriteLock.writeLock();
writeLock.lock();
try {
Partition partition = findPartition(pt.getGraphName(), pt.getId());
if (partition != null) {
// partition.setShardsList(shards);
// partition.setConfVer(partition.getConfVer() + 1);
savePartition(partition, true);
}
return partition;
} finally {
writeLock.unlock();
}
}
/**
* 拆分partition对象
*/
public List<Metapb.Partition> updatePartitionToPD(List<Metapb.Partition> partitions) throws
PDException {
// 更新本地分区信息，以及cache信息
return pdProvider.updatePartition(partitions);
}
/**
* 根据raft address查找Store
*/
public Store getStoreByRaftEndpoint(ShardGroup group, String endpoint) {
final Store[] result = {new Store()};
group.getShards().forEach((shard) -> {
Store store = getStore(shard.getStoreId());
if (store != null && store.getRaftAddress().equalsIgnoreCase(endpoint)) {
result[0] = store;
}
});
return result[0];
}
public Shard getShardByRaftEndpoint(ShardGroup group, String endpoint) {
final Shard[] result = {new Shard()};
group.getShards().forEach((shard) -> {
Store store = getStore(shard.getStoreId());
if (store != null && store.getRaftAddress().equalsIgnoreCase(endpoint)) {
result[0] = shard;
}
});
return result[0];
}
/**
* raft存储路径
*
* @param groupId
* @return location/raft/groupId/
*/
public String getRaftDataPath(int groupId) {
String location = storeMetadata.getPartitionRaftLocation(groupId);
location = Paths.get(location,
HgStoreEngineOptions.Raft_Path_Prefix,
String.format("%05d", groupId)).toAbsolutePath().toString();
return location;
}
/**
* raft snapshot 的路径，要和 db同一个盘上，便于hard link
*
* @param groupId raft group id
* @return location/snapshot/0000x/
*/
public String getRaftSnapShotPath(int groupId) {
String dbName = BusinessHandlerImpl.getDbName(groupId);
String location = storeMetadata.getPartitionStoreLocation(groupId, dbName);
location = Paths.get(location,
HgStoreEngineOptions.Raft_Path_Prefix,
dbName).toAbsolutePath().toString();
return location;
}
/**
* db存储路径
*
* @return location/db
*/
public String getDbDataPath(int partitionId, String dbName) {
String location = storeMetadata.getPartitionStoreLocation(partitionId, dbName);
location = Paths.get(location,
HgStoreEngineOptions.DB_Path_Prefix).toAbsolutePath().toString();
return location;
}
public void reportTask(MetaTask.Task task) {
try {
pdProvider.reportTask(task);
} catch (Exception e) {
LOG.error("reportTask exception {}, {}", e, task);
}
}
/**
* 修改partion的state状态
*/
public List<Metapb.Partition> changePartitionToOnLine(List<Metapb.Partition> partitions) {
List<Metapb.Partition> newPartitions = new ArrayList<>();
partitions.forEach(e -> {
newPartitions.add(e.toBuilder().setState(Metapb.PartitionState.PState_Normal).build());
});
return newPartitions;
}
public PartitionMetaStoreWrapper getWrapper() {
return wrapper;
}
/**
* Partition对象被修改消息
*/
public interface PartitionChangedListener {
void onChanged(Partition partition);
UpdatePartitionResponse rangeOrStateChanged(UpdatePartitionRequest request);
}
}

77
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionRole.java Normal file
View File

@ -0,0 +1,77 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import org.apache.hugegraph.pd.grpc.Metapb;
public enum PartitionRole {
UNKNOWN(0, "unknown"),
LEADER(1, "leader"),
FOLLOWER(2, "follower"),
LEARNER(3, "learner"),
CANDIDATE(4, "candidate");
private final int role;
private final String name;
PartitionRole(int role, String name) {
this.role = role;
this.name = name;
}
public static PartitionRole fromShardRole(Metapb.ShardRole shard) {
PartitionRole role = PartitionRole.FOLLOWER;
switch (shard) {
case Leader:
role = PartitionRole.LEADER;
break;
case Follower:
role = PartitionRole.FOLLOWER;
break;
case Learner:
role = PartitionRole.LEARNER;
break;
}
return role;
}
@Override
public String toString() {
return this.ordinal() + "_" + this.name;
}
public String getName() {
return this.name;
}
public Metapb.ShardRole toShardRole() {
Metapb.ShardRole shardRole = Metapb.ShardRole.None;
switch (this) {
case LEADER:
shardRole = Metapb.ShardRole.Leader;
break;
case FOLLOWER:
shardRole = Metapb.ShardRole.Follower;
break;
case LEARNER:
shardRole = Metapb.ShardRole.Learner;
break;
}
return shardRole;
}
}

59
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionStats.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import java.util.ArrayList;
import java.util.List;
import org.apache.hugegraph.pd.grpc.Metapb;
import lombok.Data;
@Data
public class PartitionStats {
// 分区leader所在shard
Metapb.Shard leader;
// 分区离线的shard
List<Metapb.Shard> offlineShards = new ArrayList<>();
long committedIndex;
long leaderTerm;
long approximateSize;
long approximateKeys;
// 分区ID
private int id;
private String namespace;
private String graphName;
public PartitionStats addOfflineShard(Metapb.Shard shard) {
offlineShards.add(shard);
return this;
}
public Metapb.PartitionStats getProtoObj() {
return Metapb.PartitionStats.newBuilder()
.setId(id)
.addGraphName(graphName)
.setLeader(leader)
.addAllShard(offlineShards)
.setApproximateKeys(approximateKeys)
.setApproximateSize(approximateSize)
.setLeaderTerm(leaderTerm)
.build();
}
}

43
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Shard.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import org.apache.hugegraph.pd.grpc.Metapb;
import lombok.Data;
/**
* 一个分片
*/
@Data
public class Shard {
private long storeId;
private Metapb.ShardRole role;
public static Shard fromMetaPbShard(Metapb.Shard shard) {
Shard s = new Shard();
s.setRole(shard.getRole());
s.setStoreId(shard.getStoreId());
return s;
}
public Metapb.Shard toMetaPbShard() {
return Metapb.Shard.newBuilder().setStoreId(storeId).setRole(role).build();
}
}

115
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/ShardGroup.java Normal file
View File

@ -0,0 +1,115 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Collectors;
import org.apache.hugegraph.pd.grpc.Metapb;
import lombok.Data;
/**
* 分片副本组
*/
@Data
public class ShardGroup {
private List<Shard> shards = new CopyOnWriteArrayList<>();
private int id;
/**
* Leader 任期，leader 切换后递增 = raftNode.leader_term
* 无实际用处
*/
private long version;
/**
* shards 版本号，每次改变后递增
*/
private long confVersion;
public static ShardGroup from(Metapb.ShardGroup meta) {
if (meta == null) {
return null;
}
ShardGroup shardGroup = new ShardGroup();
shardGroup.setId(meta.getId());
shardGroup.setVersion(meta.getVersion());
shardGroup.setConfVersion(meta.getConfVer());
shardGroup.setShards(meta.getShardsList().stream().map(Shard::fromMetaPbShard)
.collect(Collectors.toList()));
return shardGroup;
}
public synchronized ShardGroup changeLeader(long storeId) {
shards.forEach(shard -> {
shard.setRole(shard.getStoreId() == storeId ? Metapb.ShardRole.Leader :
Metapb.ShardRole.Follower);
});
return this;
}
public synchronized ShardGroup changeShardList(List<Long> peerIds, List<Long> learners,
long leaderId) {
if (!peerIds.isEmpty()) {
shards.clear();
peerIds.forEach(id -> {
shards.add(new Shard() {{
setStoreId(id);
setRole(id == leaderId ? Metapb.ShardRole.Leader : Metapb.ShardRole.Follower);
}});
});
learners.forEach(id -> {
shards.add(new Shard() {{
setStoreId(id);
setRole(Metapb.ShardRole.Learner);
}});
});
confVersion = confVersion + 1;
}
return this;
}
public synchronized List<Metapb.Shard> getMetaPbShard() {
List<Metapb.Shard> shardList = new ArrayList<>();
shards.forEach(shard -> {
shardList.add(shard.toMetaPbShard());
});
return shardList;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
shards.forEach(e -> {
builder.append(String.format("{ id:%s,role:%s },", e.getStoreId(), e.getRole()));
});
return builder.length() > 0 ? builder.substring(0, builder.length() - 1) : "";
}
public Metapb.ShardGroup getProtoObj() {
return Metapb.ShardGroup.newBuilder()
.setId(this.id)
.setVersion(this.version)
.setConfVer(this.confVersion)
.addAllShards(getMetaPbShard())
.build();
}
}

96
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Store.java Normal file
View File

@ -0,0 +1,96 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import java.util.Map;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.util.Asserts;
import org.apache.hugegraph.store.util.Version;
import lombok.Data;
@Data
public class Store {
private final String version;
private long id = 0;
private String storeAddress;
private String pdAddress;
private String raftAddress;
private String deployPath;
private String dataPath; // 数据存储路径
private int dataVersion;
private int partitionCount;
private int startTime;
private int usedSize; //rocksdb存储大小
private int pdHeartbeatInterval;
private Metapb.StoreState state;
private Map<String, String> labels;
private int cores;
public Store() {
this.id = 0;
this.version = Version.getVersion();
}
public Store(int dataVersion) {
this.id = 0;
this.dataVersion = dataVersion;
this.version = Version.getVersion();
}
public Store(Metapb.Store protoObj) {
if (protoObj != null) {
this.id = protoObj.getId();
this.raftAddress = protoObj.getRaftAddress();
this.storeAddress = protoObj.getAddress();
this.dataVersion = protoObj.getDataVersion();
} else {
this.id = 0;
}
this.version = Version.getVersion();
}
public Metapb.Store getProtoObj() {
Asserts.isNonNull(storeAddress);
Asserts.isNonNull(raftAddress);
Metapb.Store.Builder builder = Metapb.Store.newBuilder()
.setId(id).setVersion(version)
.setDataVersion(dataVersion)
.setAddress(storeAddress)
.setRaftAddress(raftAddress)
.setState(Metapb.StoreState.Up)
.setCores(cores)
.setDeployPath(deployPath)
.setDataPath(dataPath);
if (labels != null) {
labels.forEach((k, v) -> {
builder.addLabels(Metapb.StoreLabel.newBuilder().setKey(k).setValue(v).build());
});
}
return builder.build();
}
public boolean checkState(Metapb.StoreState state) {
return this.state == state;
}
}

231
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/StoreMetadata.java Normal file
View File

@ -0,0 +1,231 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import java.io.File;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.meta.base.GlobalMetaStore;
import org.apache.hugegraph.store.options.MetadataOptions;
import org.apache.hugegraph.store.util.HgStoreException;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class StoreMetadata extends GlobalMetaStore {
protected final static int Store_DataFmt_Version = 1;
private final List<String> dataLocations;
private final List<String> raftLocations;
private Store store = null;
public StoreMetadata(MetadataOptions options) {
super(options);
dataLocations = Arrays.asList(options.getDataPath().split(","));
raftLocations = Arrays.asList(options.getRaftPath().split(","));
}
public List<String> getDataLocations() {
return dataLocations;
}
public List<String> getRaftLocations() {
return raftLocations;
}
public Store load() {
// 针对多目录存储的情况下，预先创建文件夹，方便pd端统计文件存储
dataLocations.forEach(path -> {
String strPath = Paths.get(path).toAbsolutePath().toString();
File dbFile = new File(strPath);
if (!dbFile.exists()) {
dbFile.mkdir();
}
});
raftLocations.forEach(path -> {
String strPath = Paths.get(path).toAbsolutePath().toString();
File dbFile = new File(strPath);
if (!dbFile.exists()) {
dbFile.mkdir();
}
});
byte[] key = MetadataKeyHelper.getStoreKey();
byte[] value = get(key);
if (value != null) {
try {
Metapb.Store protoObj = Metapb.Store.parseFrom(value);
if (protoObj != null) {
store = new Store(protoObj);
}
} catch (Exception e) {
throw new HgStoreException(HgStoreException.EC_FAIL, e);
}
}
if (store == null) {
store = new Store(Store_DataFmt_Version);
}
checkDataFmtCompatible();
return store;
}
public Store getStore() {
return store;
}
public void save(Store store) {
byte[] key = MetadataKeyHelper.getStoreKey();
put(key, store.getProtoObj().toByteArray());
this.store = store;
}
public void checkDataFmtCompatible() {
if (store == null || store.getDataVersion() != Store_DataFmt_Version) {
throw new HgStoreException(HgStoreException.EC_DATAFMT_NOT_SUPPORTED,
String.format(
"Incompatible data format, data format version is " +
"%d, supported version is %d",
store.getDataVersion(), Store_DataFmt_Version));
}
}
public Metapb.PartitionStore getPartitionStore(int partitionId) {
byte[] key = MetadataKeyHelper.getPartitionStoreKey(partitionId);
return get(Metapb.PartitionStore.parser(), key);
}
public List<Metapb.PartitionStore> getPartitionStores() {
byte[] key = MetadataKeyHelper.getPartitionStorePrefix();
return scan(Metapb.PartitionStore.parser(), key);
}
public void savePartitionStore(Metapb.PartitionStore partitionStore) {
byte[] key = MetadataKeyHelper.getPartitionStoreKey(partitionStore.getPartitionId());
put(key, partitionStore.toByteArray());
}
public Metapb.PartitionRaft getPartitionRaft(int partitionId) {
byte[] key = MetadataKeyHelper.getPartitionRaftKey(partitionId);
return get(Metapb.PartitionRaft.parser(), key);
}
public List<Metapb.PartitionRaft> getPartitionRafts() {
byte[] key = MetadataKeyHelper.getPartitionRaftPrefix();
return scan(Metapb.PartitionRaft.parser(), key);
}
public void savePartitionRaft(Metapb.PartitionRaft partitionRaft) {
byte[] key = MetadataKeyHelper.getPartitionRaftKey(partitionRaft.getPartitionId());
put(key, partitionRaft.toByteArray());
}
private String getMinDataLocation() {
Map<String, Integer> counter = new HashMap<>();
dataLocations.forEach(l -> {
counter.put(l, Integer.valueOf(0));
});
getPartitionStores().forEach(ptStore -> {
if (counter.containsKey(ptStore.getStoreLocation())) {
counter.put(ptStore.getStoreLocation(),
counter.get(ptStore.getStoreLocation()) + 1);
}
});
int min = Integer.MAX_VALUE;
String location = "";
for (String k : counter.keySet()) {
if (counter.get(k) < min) {
min = counter.get(k);
location = k;
}
}
return location;
}
private String getMinRaftLocation() {
Map<String, Integer> counter = new HashMap<>();
raftLocations.forEach(l -> {
counter.put(l, Integer.valueOf(0));
});
getPartitionRafts().forEach(ptRaft -> {
if (counter.containsKey(ptRaft.getRaftLocation())) {
counter.put(ptRaft.getRaftLocation(), counter.get(ptRaft.getRaftLocation()) + 1);
}
});
int min = Integer.MAX_VALUE;
String location = "";
for (String k : counter.keySet()) {
if (counter.get(k) < min) {
min = counter.get(k);
location = k;
}
}
return location;
}
/**
* 获取分区数据存储的位置，如果分布数据不存在，自动创建新的位置
*
* @param partitionId
* @return
*/
public String getPartitionStoreLocation(int partitionId, String dbName) {
Metapb.PartitionStore location = getPartitionStore(partitionId);
if (location == null) {
synchronized (this) {
location = getPartitionStore(partitionId);
if (location == null) {
// 查找分区数最少的存储
location = Metapb.PartitionStore.newBuilder()
.setPartitionId(partitionId)
.setStoreLocation(getMinDataLocation())
.build();
// TODO 选择分区数最小的路径
savePartitionStore(location);
}
}
}
return location.getStoreLocation();
}
public String getPartitionRaftLocation(int partitionId) {
Metapb.PartitionRaft location = getPartitionRaft(partitionId);
if (location == null) {
synchronized (this) {
location = getPartitionRaft(partitionId);
if (location == null) {
// 查找分区数最少的存储
location = Metapb.PartitionRaft.newBuilder()
.setPartitionId(partitionId)
.setRaftLocation(getMinRaftLocation())
.build();
// TODO 选择分区数最小的路径
savePartitionRaft(location);
}
}
}
return location.getRaftLocation();
}
}

146
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/TaskManager.java Normal file
View File

@ -0,0 +1,146 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta;
import java.util.ArrayList;
import java.util.List;
import org.apache.hugegraph.pd.grpc.MetaTask;
import org.apache.hugegraph.store.meta.asynctask.AbstractAsyncTask;
import org.apache.hugegraph.store.meta.asynctask.AsyncTask;
import org.apache.hugegraph.store.meta.asynctask.AsyncTaskState;
import org.apache.hugegraph.store.meta.base.DBSessionBuilder;
import org.apache.hugegraph.store.meta.base.PartitionMetaStore;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class TaskManager extends PartitionMetaStore {
public TaskManager(DBSessionBuilder sessionBuilder, int partId) {
super(sessionBuilder, partId);
}
public void updateTask(int partId, String type, long taskId, byte[] task) {
byte[] key = MetadataKeyHelper.getTaskKey(partId, type, taskId);
put(key, task);
}
public void updateTask(MetaTask.Task task) {
if (task.getState().compareTo(MetaTask.TaskState.Task_Stop) < 0) {
updateTask(task.getPartition().getId(),
task.getType().name(), task.getId(), task.toByteArray());
} else {
deleteTask(task);
}
}
public MetaTask.Task getOneTask(int partId, String type) {
byte[] key = MetadataKeyHelper.getTaskPrefix(partId, type);
List<MetaTask.Task> tasks = scan(MetaTask.Task.parser(), key);
if (tasks.size() > 0) {
return tasks.get(tasks.size() - 1);
}
return get(MetaTask.Task.parser(), key);
}
public MetaTask.Task getOneTask(int partId, MetaTask.TaskType taskType) {
return getOneTask(partId, taskType.name());
}
public MetaTask.Task getOneTask(MetaTask.Task task) {
return getOneTask(task.getPartition().getId(), task.getType());
}
public void deleteTask(MetaTask.Task task) {
byte[] key = MetadataKeyHelper.getTaskKey(task.getPartition().getId(),
task.getType().name(), task.getId());
delete(key);
}
public void deleteTask(int partId, String type) {
deletePrefix(MetadataKeyHelper.getTaskPrefix(partId, type));
}
public boolean taskExists(int partId, String graphName, String taskTypeName) {
return partitionTaskRepeat(partId, graphName, taskTypeName, 0);
}
public boolean taskExists(MetaTask.Task task) {
return null != getOneTask(task);
}
/*
* 判断相同分区下相同任务是否重复
* partId 分区id
* TaskTypeName 任务类型名称
* graphName
*/
public boolean partitionTaskRepeat(int partId, String graphName, String taskTypeName) {
return partitionTaskRepeat(partId, graphName, taskTypeName, 1);
}
private boolean partitionTaskRepeat(int partId, String graphName, String taskTypeName,
int checkCount) {
byte[] key = MetadataKeyHelper.getTaskPrefix(partId, taskTypeName);
List<MetaTask.Task> tasks = scan(MetaTask.Task.parser(), key);
if (tasks.size() > 1) {
int graphCount = 0;
for (MetaTask.Task task : tasks) {
if (task.getPartition().getGraphName().equals(graphName) &&
task.getState().getNumber() < MetaTask.TaskState.Task_Stop_VALUE) {
graphCount++;
}
}
return graphCount > checkCount;
}
return false;
}
public void putAsyncTask(AsyncTask task) {
put(MetadataKeyHelper.getAsyncTaskKey(task.getPartitionId(),
task.getGraphName(), task.getId()), task.toBytes());
}
public AsyncTask getOneAsyncTask(int partId, String graphName, String taskId) {
var bytes = get(MetadataKeyHelper.getAsyncTaskKey(partId, graphName, taskId));
if (bytes != null) {
return AbstractAsyncTask.fromBytes(bytes);
}
return null;
}
public void updateAsyncTaskState(int partId, String graphName, String taskId,
AsyncTaskState state) {
var task = getOneAsyncTask(partId, graphName, taskId);
if (task != null) {
task.setState(state);
putAsyncTask(task);
}
}
public List<AsyncTask> scanAsyncTasks(int partitionId, String graphName) {
var list = new ArrayList<AsyncTask>();
for (var task : scan(MetadataKeyHelper.getAsyncTaskPrefix(partitionId, graphName))) {
list.add(AbstractAsyncTask.fromBytes(task.value));
}
return list;
}
}

146
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AbstractAsyncTask.java Normal file
View File

@ -0,0 +1,146 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta.asynctask;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractAsyncTask implements AsyncTask, Serializable {
private final String id;
private final int partitionId;
private final String graphName;
private final String type;
/**
* 任务额外需要的参数
*/
private final Object extra;
private AsyncTaskState state;
public AbstractAsyncTask(int partitionId, String graphName, AsyncTaskState state,
Object extra) {
this.id = getNextId();
this.partitionId = partitionId;
this.graphName = graphName;
this.state = state;
this.type = getType();
this.extra = extra;
}
private static String getNextId() {
return UUID.randomUUID().toString().replace("-", "");
}
public static AsyncTask fromBytes(byte[] bytes) {
AsyncTask obj = null;
try {
ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
ObjectInputStream ois = new ObjectInputStream(bis);
obj = (AsyncTask) ois.readObject();
ois.close();
bis.close();
} catch (IOException e) {
log.error("AsyncTask deserialized failed,{}", e.getMessage());
} catch (ClassNotFoundException e) {
log.error("AsyncTask deserialized failed,{}", e.getMessage());
}
return obj;
}
@Override
public String getId() {
return this.id;
}
@Override
public int getPartitionId() {
return this.partitionId;
}
public AsyncTaskState getState() {
return this.state;
}
@Override
public void setState(AsyncTaskState newState) {
this.state = newState;
}
@Override
public String getGraphName() {
return this.graphName;
}
public Object getExtra() {
return this.extra;
}
public abstract String getType();
@Override
public byte[] toBytes() {
byte[] bytes = null;
ByteArrayOutputStream bos = new ByteArrayOutputStream();
try {
ObjectOutputStream oos = new ObjectOutputStream(bos);
oos.writeObject(this);
oos.flush();
bytes = bos.toByteArray();
oos.close();
bos.close();
} catch (IOException e) {
log.error("AsyncTask serialized failed, {}", e.getMessage());
e.printStackTrace();
}
return bytes;
}
@Override
public void handleTask() {
if (this.getState() == AsyncTaskState.FAILED) {
onError();
} else if (this.getState() == AsyncTaskState.START) {
onNotFinished();
}
}
protected abstract void onError();
protected abstract void onNotFinished();
@Override
public String toString() {
return "AbstractAsyncTask{" +
"id='" + id + '\'' +
", partitionId=" + partitionId +
", graphName='" + graphName + '\'' +
", state=" + state +
", type='" + type + '\'' +
", extra=" + getExtra() +
'}';
}
}

55
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AsyncTask.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta.asynctask;
public interface AsyncTask {
/**
* 需要检查异步任务时候，检查当前的状态，根据状态去做对应的处理
*/
void handleTask();
/**
* 任务ID
*/
String getId();
/**
* 针对哪个图的
*/
String getGraphName();
/**
* 针对哪个分区的
*/
int getPartitionId();
/**
* 用来进行序列化
*
* @return
*/
byte[] toBytes();
/**
* 设置执行状态
*
* @param newState
*/
void setState(AsyncTaskState newState);
}

24
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AsyncTaskState.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta.asynctask;
public enum AsyncTaskState {
START,
SUCCESS,
FAILED
}

75
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/CleanTask.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta.asynctask;
import org.apache.hugegraph.pd.grpc.pulse.CleanType;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.cmd.CleanDataRequest;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class CleanTask extends AbstractAsyncTask {
public CleanTask(int partitionId, String graphName, AsyncTaskState state, Object attach) {
super(partitionId, graphName, state, attach);
}
@Override
public String getType() {
return "CLEAN_TYPE";
}
@Override
protected void onError() {
cleanTask();
}
@Override
protected void onNotFinished() {
cleanTask();
}
private void cleanTask() {
log.info("CleanTask begin to run:{}", this);
var storeEngine = HgStoreEngine.getInstance();
if (storeEngine != null) {
if (getExtra() != null) {
CleanDataRequest request = (CleanDataRequest) getExtra();
var partition = storeEngine.getPartitionManager()
.getPartition(getGraphName(), getPartitionId());
// 只允许清理本分区之外的数据。 缩容等任务会造成干扰, 而且不能删除分区
if (request.getKeyEnd() == partition.getStartKey() &&
request.getKeyEnd() == partition.getEndKey() &&
request.getCleanType() == CleanType.CLEAN_TYPE_EXCLUDE_RANGE &&
!request.isDeletePartition()) {
storeEngine.getBusinessHandler()
.cleanPartition(getGraphName(), getPartitionId(),
request.getKeyStart(), request.getKeyEnd(),
request.getCleanType());
}
} else {
storeEngine.getBusinessHandler().cleanPartition(getGraphName(), getPartitionId());
}
storeEngine.getPartitionEngine(getPartitionId()).getTaskManager()
.updateAsyncTaskState(getPartitionId(), getGraphName(), getId(),
AsyncTaskState.SUCCESS);
}
}
}

26
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/DBSessionBuilder.java Normal file
View File

@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta.base;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.store.util.HgStoreException;
public interface DBSessionBuilder {
RocksDBSession getSession(int partId) throws HgStoreException;
}

58
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/GlobalMetaStore.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta.base;
import java.util.Arrays;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.store.options.MetadataOptions;
public class GlobalMetaStore extends MetaStoreBase {
public static final String HSTORE_METADATA_GRAPH_NAME = "hgstore-metadata";
public static final String HSTORE_CF_NAME = "default";
private final MetadataOptions options;
private final String dataPath;
public GlobalMetaStore(MetadataOptions options) {
this.options = options;
dataPath = Arrays.asList(options.getDataPath().split(",")).get(0);
}
public MetadataOptions getOptions() {
return options;
}
@Override
protected RocksDBSession getRocksDBSession() {
RocksDBFactory rocksDBFactory = RocksDBFactory.getInstance();
RocksDBSession dbSession = rocksDBFactory.queryGraphDB(HSTORE_METADATA_GRAPH_NAME);
if (dbSession == null) {
dbSession = rocksDBFactory.createGraphDB(dataPath, HSTORE_METADATA_GRAPH_NAME);
}
return dbSession;
}
@Override
protected String getCFName() {
return HSTORE_CF_NAME;
}
}

174
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/MetaStoreBase.java Normal file
View File

@ -0,0 +1,174 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta.base;
import java.io.Closeable;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.SessionOperator;
import org.apache.hugegraph.store.util.Asserts;
import org.apache.hugegraph.store.util.HgStoreException;
import com.google.protobuf.GeneratedMessageV3;
import com.google.protobuf.InvalidProtocolBufferException;
/**
* Store、Partition等元数据存储到hgstore-metadata图下
*/
public abstract class MetaStoreBase implements Closeable {
protected abstract RocksDBSession getRocksDBSession();
protected abstract String getCFName();
@Override
public void close() throws IOException {
}
public void put(byte[] key, byte[] value) {
try (RocksDBSession dbSession = getRocksDBSession()) {
Asserts.isTrue(dbSession != null, "DB session is null.");
SessionOperator op = dbSession.sessionOp();
try {
op.prepare();
op.put(getCFName(), key, value);
op.commit();
} catch (Exception e) {
op.rollback();
throw e;
}
}
}
public void put(byte[] key, GeneratedMessageV3 value) {
put(key, value.toByteArray());
}
public byte[] get(byte[] key) {
try (RocksDBSession dbSession = getRocksDBSession()) {
SessionOperator op = dbSession.sessionOp();
return op.get(getCFName(), key);
}
}
public <E> E get(com.google.protobuf.Parser<E> parser, byte[] key) {
byte[] value = get(key);
try {
if (value != null) {
return parser.parseFrom(value);
}
} catch (Exception e) {
throw new HgStoreException(HgStoreException.EC_FAIL, e);
}
return null;
}
public List<RocksDBSession.BackendColumn> scan(byte[] prefix) {
List<RocksDBSession.BackendColumn> values = new LinkedList<>();
try (RocksDBSession dbSession = getRocksDBSession()) {
SessionOperator op = dbSession.sessionOp();
ScanIterator iterator = op.scan(getCFName(), prefix);
while (iterator.hasNext()) {
values.add(iterator.next());
}
}
return values;
}
public <E> List<E> scan(com.google.protobuf.Parser<E> parser, byte[] prefix) {
try (RocksDBSession dbSession = getRocksDBSession()) {
SessionOperator op = dbSession.sessionOp();
ScanIterator iterator = op.scan(getCFName(), prefix);
List<E> values = new LinkedList<>();
try {
while (iterator.hasNext()) {
RocksDBSession.BackendColumn col = iterator.next();
values.add(parser.parseFrom(col.value));
}
} catch (InvalidProtocolBufferException e) {
throw new HgStoreException(HgStoreException.EC_FAIL, e);
}
return values;
}
}
public List<RocksDBSession.BackendColumn> scan(byte[] start, byte[] end) {
List<RocksDBSession.BackendColumn> values = new LinkedList<>();
try (RocksDBSession dbSession = getRocksDBSession()) {
SessionOperator op = dbSession.sessionOp();
ScanIterator iterator = op.scan(getCFName(), start, end,
ScanIterator.Trait.SCAN_GTE_BEGIN |
ScanIterator.Trait.SCAN_LT_END);
while (iterator.hasNext()) {
values.add(iterator.next());
}
}
return values;
}
public <E> List<E> scan(com.google.protobuf.Parser<E> parser, byte[] start, byte[] end) {
try (RocksDBSession dbSession = getRocksDBSession()) {
SessionOperator op = dbSession.sessionOp();
ScanIterator iterator = op.scan(getCFName(), start, end,
ScanIterator.Trait.SCAN_GTE_BEGIN |
ScanIterator.Trait.SCAN_LT_END);
List<E> values = new LinkedList<>();
try {
while (iterator.hasNext()) {
RocksDBSession.BackendColumn col = iterator.next();
values.add(parser.parseFrom(col.value));
}
} catch (InvalidProtocolBufferException e) {
throw new HgStoreException(HgStoreException.EC_FAIL, e);
}
return values;
}
}
public void delete(byte[] key) {
try (RocksDBSession dbSession = getRocksDBSession()) {
SessionOperator op = dbSession.sessionOp();
try {
op.prepare();
op.delete(getCFName(), key);
op.commit();
} catch (Exception e) {
op.rollback();
throw e;
}
}
}
public void deletePrefix(byte[] key) {
try (RocksDBSession dbSession = getRocksDBSession()) {
SessionOperator op = dbSession.sessionOp();
try {
op.prepare();
op.deletePrefix(getCFName(), key);
op.commit();
} catch (Exception e) {
op.rollback();
throw e;
}
}
}
}

52
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/PartitionMetaStore.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.meta.base;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
/**
* 元数据存储在分区的default cf中
*/
public class PartitionMetaStore extends MetaStoreBase {
public static final String DEFAULT_CF_NAME = "default";
private final DBSessionBuilder sessionBuilder;
private final Integer partitionId;
public PartitionMetaStore(DBSessionBuilder sessionBuilder, int partId) {
this.sessionBuilder = sessionBuilder;
this.partitionId = partId;
}
@Override
protected RocksDBSession getRocksDBSession() {
return sessionBuilder.getSession(this.partitionId);
}
@Override
protected String getCFName() {
return DEFAULT_CF_NAME;
}
protected void flush() {
try (RocksDBSession dbSession = getRocksDBSession()) {
dbSession.flush(true);
}
}
}

200
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/metric/HgMetricService.java Normal file
View File

@ -0,0 +1,200 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.metric;
import java.io.File;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.io.FileUtils;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.PartitionEngine;
import org.apache.hugegraph.store.util.Lifecycle;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class HgMetricService implements Lifecycle<Void> {
private final static HgMetricService instance = new HgMetricService();
private final static AtomicLong bytesWritten = new AtomicLong();
private final static AtomicLong bytesRead = new AtomicLong();
private final static AtomicLong keysWritten = new AtomicLong();
private final static AtomicLong keysRead = new AtomicLong();
private final static long startTime = Instant.now().getEpochSecond();
private static long lastQueryTime = 0;
private final SystemMetricService systemMetricService = new SystemMetricService();
private HgStoreEngine storeEngine;
private Map<String, Long> systemMetrics = new HashMap<>();
private HgMetricService() {
}
public static HgMetricService getInstance() {
return instance;
}
@Override
public boolean init(final Void v) {
resetMetrics();
return true;
}
@Override
public void shutdown() {
}
public HgMetricService setHgStoreEngine(HgStoreEngine storeEngine) {
this.storeEngine = storeEngine;
this.systemMetricService.setStoreEngine(storeEngine);
return this;
}
public Metapb.StoreStats.Builder getMetrics() {
Metapb.StoreStats.Builder builder = Metapb.StoreStats.newBuilder();
try {
getStoreMetrics(builder);
getRaftMetrics(builder);
getDiskMetrics(builder);
getSystemMetrics(builder);
} catch (Exception e) {
log.error("HgMetricService getMetrics {}", e);
}
return builder;
}
private Metapb.StoreStats.Builder getDiskMetrics(Metapb.StoreStats.Builder builder) {
try {
long capacity = 0L;
long available = 0L;
long used = 0L;
HashSet<String> fileStoreSet = new HashSet<>();
for (String dbPath : this.storeEngine.getDataLocations()) {
FileStore fs = Files.getFileStore(Paths.get(dbPath));
if (fileStoreSet.contains(fs.name())) {
continue;
}
fileStoreSet.add(fs.name());
capacity += fs.getTotalSpace();
available += fs.getUsableSpace();
used += FileUtils.sizeOfDirectory(new File(dbPath));
}
builder.setCapacity(capacity);
builder.setAvailable(available);
builder.setUsedSize(used);
} catch (Exception e) {
log.error("Failed to get disk metrics. {}", e.toString());
}
return builder;
}
private Metapb.StoreStats.Builder getRaftMetrics(Metapb.StoreStats.Builder builder) {
Map<Integer, PartitionEngine> partitionEngines = this.storeEngine.getPartitionEngines();
builder.setPartitionCount(partitionEngines.size());
partitionEngines.forEach((partId, engine) -> {
builder.addRaftStats(Metapb.RaftStats.newBuilder()
.setPartitionId(partId)
.setCommittedIndex(engine.getCommittedIndex())
.build());
});
return builder;
}
private Metapb.StoreStats.Builder getStoreMetrics(Metapb.StoreStats.Builder builder) {
builder.setStoreId(this.storeEngine.getHeartbeatService().getStoreInfo().getId());
builder.setStartTime((int) startTime);
this.storeEngine.getPartitionManager().getPartitions().forEach((graphName, partitions) -> {
partitions.forEach((partId, partition) -> {
HgStoreMetric.Graph graphMetric =
this.storeEngine.getBusinessHandler().getGraphMetric(graphName, partId);
if ((graphMetric != null) &&
(storeEngine.getPartitionManager().getLocalRoleFromShard(partition) != null)) {
builder.addGraphStats(Metapb.GraphStats.newBuilder()
.setGraphName(graphName)
.setPartitionId(partId)
.setApproximateKeys(
graphMetric.getApproxKeyCount())
.setApproximateSize(
graphMetric.getApproxDataSize())
.setRole(
storeEngine.getPartitionManager()
.getLocalRoleFromShard(
partition)
.toShardRole())
.setWorkState(partition.getWorkState())
.build());
}
});
});
return builder;
}
/**
* get system metrics each 1 minute
*
* @param builder
* @return
*/
private Metapb.StoreStats.Builder getSystemMetrics(Metapb.StoreStats.Builder builder) {
// load each 1 minute
if (systemMetrics.isEmpty() || System.currentTimeMillis() - lastQueryTime >= 60000) {
systemMetrics = systemMetricService.getSystemMetrics();
lastQueryTime = System.currentTimeMillis();
}
for (Map.Entry<String, Long> entry : systemMetrics.entrySet()) {
if (entry.getValue() != null) {
builder.addSystemMetrics(Metapb.RecordPair.newBuilder()
.setKey(entry.getKey())
.setValue(entry.getValue())
.build());
}
}
return builder;
}
private void resetMetrics() {
bytesWritten.set(0);
bytesRead.set(0);
keysWritten.set(0);
keysRead.set(0);
}
public void increaseWriteCount(long keys, long bytes) {
keysWritten.addAndGet(keys);
bytesWritten.addAndGet(bytes);
}
public void increaseReadCount(long keys, long bytes) {
keysRead.addAndGet(keys);
bytesRead.addAndGet(bytes);
}
}

47
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/metric/HgStoreMetric.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.metric;
import java.util.List;
import lombok.Data;
public class HgStoreMetric {
@Data
public static class Table {
private String tableName;
private long keyCount;
private String dataSize;
}
@Data
public static class Partition {
private int partitionId;
private List<Table> tables;
}
@Data
public static class Graph {
private long approxDataSize;
private long approxKeyCount;
}
}

448
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/metric/SystemMetricService.java Normal file
View File

@ -0,0 +1,448 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.metric;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryUsage;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Deque;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.store.HgStoreEngine;
import org.rocksdb.MemoryUsageType;
import org.rocksdb.Statistics;
import org.rocksdb.TickerType;
import com.sun.management.OperatingSystemMXBean;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class SystemMetricService {
private static final long MIB = 1024 * 1024;
private final Deque<Map<String, List<Long>>> deque = new LinkedList<>();
HgStoreEngine storeEngine;
public void setStoreEngine(HgStoreEngine hgStoreEngine) {
this.storeEngine = hgStoreEngine;
}
public HgStoreEngine getStorageEngine() {
return this.storeEngine;
}
public Map<String, Long> getSystemMetrics() {
Map<String, Long> systemMetrics = new HashMap<>();
try {
// cpu
loadCpuInfo(systemMetrics);
// memory
loadMemInfo(systemMetrics);
// disk
loadDiskInfo(systemMetrics);
// disk io
loadDiskIo(systemMetrics);
// network
loadNetFlowInfo(systemMetrics);
// rocksdb
loadRocksDbInfo(systemMetrics);
} catch (Exception e) {
log.error("get system metric failed, {}", e.toString());
}
return systemMetrics;
}
private void loadCpuInfo(Map<String, Long> map) {
OperatingSystemMXBean osBean =
(OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
Double cpuLoad = osBean.getSystemLoadAverage();
map.put("cpu.load", cpuLoad.longValue());
}
private void loadMemInfo(Map<String, Long> map) {
OperatingSystemMXBean osBean =
(OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
map.put("mem.physical_total", osBean.getTotalPhysicalMemorySize());
map.put("mem.physical_free", osBean.getFreePhysicalMemorySize());
map.put("mem.swap_total", osBean.getTotalSwapSpaceSize());
map.put("mem.swap_free", osBean.getFreeSwapSpaceSize());
Runtime runtime = Runtime.getRuntime();
map.put("mem.heap_total", runtime.totalMemory());
map.put("mem.heap_used", runtime.totalMemory() - runtime.freeMemory());
MemoryUsage memoryUsage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage();
map.put("mem.non_heap_total", memoryUsage.getCommitted());
map.put("mem.non_heap_used", memoryUsage.getUsed());
}
private void loadDiskInfo(Map<String, Long> map) {
// sum up all disk space
File[] rootDrive = File.listRoots();
long total = 0;
long free = 0;
long usable = 0;
if (rootDrive != null) {
for (File d : rootDrive) {
total += d.getTotalSpace();
free += d.getFreeSpace();
usable += d.getUsableSpace();
}
}
map.put("disk.total_size", total / MIB);
map.put("disk.free_size", free / MIB);
map.put("disk.usable_size", usable / MIB);
}
private void loadDiskIo(Map<String, Long> map) {
for (Map.Entry<String, Float> entry : getDiskIoData().entrySet()) {
map.put(entry.getKey(), entry.getValue().longValue());
}
}
private void loadNetFlowInfo(Map<String, Long> map) {
for (Map.Entry<String, List<Long>> entry : getTraffic().entrySet()) {
// exclude none-functional network interface
map.put("network." + entry.getKey() + ".sent_bytes",
entry.getValue().get(0) / 1024 / 1024);
map.put("network." + entry.getKey() + ".recv_bytes",
entry.getValue().get(1) / 1024 / 1024);
map.put("network." + entry.getKey() + ".sent_rates",
entry.getValue().get(2) / 1024 / 1024);
map.put("network." + entry.getKey() + ".recv_rates",
entry.getValue().get(3) / 1024 / 1024);
}
}
private void loadRocksDbInfo(Map<String, Long> map) {
Map<MemoryUsageType, Long> dbMem =
storeEngine.getBusinessHandler().getApproximateMemoryUsageByType(null);
map.put("rocksdb.table.reader.total", dbMem.get(MemoryUsageType.kTableReadersTotal));
map.put("rocksdb.mem.table.total", dbMem.get(MemoryUsageType.kMemTableTotal));
map.put("rocksdb.cache.total", dbMem.get(MemoryUsageType.kCacheTotal));
map.put("rocksdb.mem.table.un_flushed", dbMem.get(MemoryUsageType.kMemTableUnFlushed));
RocksDBFactory dbFactory = RocksDBFactory.getInstance();
Set<String> names = dbFactory.getGraphNames();
if (names != null) {
for (String name : names) {
try {
RocksDBSession session = dbFactory.queryGraphDB(name);
Statistics statistics = session.getRocksDbStats();
map.put(
"rocksdb.graph." + name + "." +
TickerType.NUMBER_KEYS_WRITTEN.name().toLowerCase(),
statistics.getTickerCount(TickerType.NUMBER_KEYS_WRITTEN));
map.put(
"rocksdb.graph." + name + "." +
TickerType.NUMBER_KEYS_READ.name().toLowerCase(),
statistics.getTickerCount(TickerType.NUMBER_KEYS_READ));
map.put(
"rocksdb.graph." + name + "." +
TickerType.NUMBER_KEYS_UPDATED.name().toLowerCase(),
statistics.getTickerCount(TickerType.NUMBER_KEYS_UPDATED));
map.put(
"rocksdb.graph." + name + "." +
TickerType.BYTES_WRITTEN.name().toLowerCase(),
statistics.getTickerCount(TickerType.BYTES_WRITTEN));
map.put(
"rocksdb.graph." + name + "." +
TickerType.BYTES_READ.name().toLowerCase(),
statistics.getTickerCount(TickerType.BYTES_READ));
} catch (Exception e) {
}
}
}
}
/**
* get all network interface traffic(delta from last invoke).
* -sent bytes
* -receive bytes
* -in rates
* -out rates
*
* @return
*/
private Map<String, List<Long>> getTraffic() {
deque.add(loadTrafficData());
if (deque.size() < 2) {
return new HashMap<>();
}
// keep 2 copies
while (deque.size() > 2) {
deque.removeFirst();
}
// compare
Map<String, List<Long>> result = new HashMap<>();
Map<String, List<Long>> currentFlows = deque.getLast();
Map<String, List<Long>> preFlows = deque.getFirst();
for (Map.Entry<String, List<Long>> entry : currentFlows.entrySet()) {
if (preFlows.containsKey(entry.getKey())) {
List<Long> prev = preFlows.get(entry.getKey());
List<Long> now = preFlows.get(entry.getKey());
// no traffic
if (now.get(0) == 0) {
continue;
}
long diff = now.get(2) - prev.get(2);
diff = diff > 0 ? diff : 1L;
result.put(
entry.getKey(),
Arrays.asList(
now.get(0) - prev.get(0),
now.get(1) - prev.get(1),
// rate rate
(now.get(0) - prev.get(0)) / diff,
// recv rate
(now.get(1) - prev.get(1)) / diff));
}
}
return result;
}
/**
* load traffic according to os, now only support mac os and linux
*
* @return
*/
private Map<String, List<Long>> loadTrafficData() {
String osName = System.getProperty("os.name").toLowerCase();
if (osName.startsWith("linux")) {
return loadLinuxTrafficData();
} else if (osName.startsWith("mac")) {
return loadMacOsTrafficData();
}
return new HashMap<>();
}
/**
* read the result of "netstat -ib". (lo is ignored)
*
* @return
*/
private Map<String, List<Long>> loadMacOsTrafficData() {
Map<String, List<Long>> flows = new HashMap<>();
Long current = System.currentTimeMillis() / 1000;
for (String line : executeCmd("netstat -ib")) {
if (line.startsWith("Name") || line.startsWith("lo")) {
// first table header line
continue;
}
List<String> arr = Arrays.stream(line.split(" ")).filter(x -> x.length() > 0)
.collect(Collectors.toList());
long sentBytes = Long.parseLong(arr.get(arr.size() - 2));
long recvBytes = Long.parseLong(arr.get(arr.size() - 5));
String name = arr.get(0);
// log.debug("mac: {}, -> {},{},{}", line, sentBytes, recvBytes, name);
if (sentBytes > 0 && recvBytes > 0) {
flows.put(name, Arrays.asList(sentBytes, recvBytes, current));
}
}
return flows;
}
/**
* read the statistics file for network interface
* cat /sys/class/net/NETWORK_INTERFACE_NAME/statistics/tx_bytes
* cat /sys/class/net/NETWORK_INTERFACE_NAME/statistics/rx_bytes
*
* @return
*/
private Map<String, List<Long>> loadLinuxTrafficData() {
Long current = System.currentTimeMillis() / 1000;
Map<String, List<Long>> flows = new HashMap<>();
try {
for (String name : getAllNetworkInterfaces()) {
long sentBytes = getUnsignedLongFromFile(
String.format("/sys/class/net/%s/statistics/tx_bytes", name));
long recvBytes = getUnsignedLongFromFile(
String.format("/sys/class/net/%s/statistics/rx_bytes", name));
flows.put(name, Arrays.asList(sentBytes, recvBytes, current));
}
} catch (Exception e) {
}
return flows;
}
/**
* read file and parse to long
*
* @param filename
* @return
*/
private long getUnsignedLongFromFile(String filename) throws IOException {
List<String> lines = Files.readAllLines(Paths.get(filename), StandardCharsets.UTF_8);
if (!lines.isEmpty()) {
return Long.parseLong(lines.get(0));
}
return 0L;
}
/**
* get all network interface names. (lo is ignored)
*
* @return
* @throws SocketException
*/
private List<String> getAllNetworkInterfaces() throws SocketException {
Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
List<String> names = new ArrayList<>();
while (interfaces.hasMoreElements()) {
String name = interfaces.nextElement().getName();
if (!"lo".equals(name)) {
names.add(name);
}
}
return names;
}
private Map<String, Float> getDiskIoData() {
String osName = System.getProperty("os.name").toLowerCase();
if (osName.startsWith("linux")) {
return loadLinuxDiskIoData();
} else if (osName.startsWith("mac")) {
return loadMacDiskIoData();
}
return new HashMap<>();
}
/**
* get io data using iostat -d -x -k
*
* @return
*/
private Map<String, Float> loadLinuxDiskIoData() {
Map<String, Float> result = new HashMap<>();
boolean contentFlag = false;
for (String line : executeCmd("iostat -d -x -k")) {
// header
if (line.startsWith("Device")) {
contentFlag = true;
continue;
}
if (contentFlag) {
List<String> arr =
Arrays.stream(line.split(" ")).filter(x -> x.length() > 0)
.collect(Collectors.toList());
try {
// util%
result.put("disk.io." + arr.get(0) + ".util",
Float.valueOf(arr.get(arr.size() - 1)) * 100);
// wait
result.put("disk.io." + arr.get(0) + ".wait",
Float.valueOf(arr.get(arr.size() - 5)) * 100);
} catch (Exception e) {
log.debug("error get disk io data {}", line);
}
}
}
return result;
}
/**
* get io data using iostat
*
* @return
*/
private Map<String, Float> loadMacDiskIoData() {
Map<String, Float> result = new HashMap<>();
List<String> lines = executeCmd("iostat -oK");
// disks
List<String> disks =
Arrays.stream(lines.get(0).split(" "))
.filter(x -> x.length() > 0 && x.startsWith("disk"))
.collect(Collectors.toList());
// datas
List<String> data =
Arrays.stream(lines.get(2).split(" ")).filter(x -> x.length() > 0)
.collect(Collectors.toList());
// zip data
for (int i = 0; i < disks.size(); i++) {
try {
// msps
result.put("disk.io." + disks.get(i) + ".wait",
Float.valueOf(data.get(i * 3 + 2)) * 100);
// no such value
result.put("disk.io." + disks.get(i) + ".util", 0.0F);
} catch (Exception e) {
log.debug("error get io data {}", data.get(i));
}
}
return result;
}
/**
* execute cmd and get the output
*
* @param cmd
* @return
*/
private List<String> executeCmd(String cmd) {
List<String> result = new ArrayList<>();
try {
Process pr = Runtime.getRuntime().exec(cmd);
BufferedReader in = new BufferedReader(new InputStreamReader(pr.getInputStream()));
String line;
while ((line = in.readLine()) != null) {
if (line.length() > 0) {
result.add(line);
}
}
pr.waitFor();
in.close();
} catch (IOException | InterruptedException e) {
}
return result;
}
}

148
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java Normal file
View File

@ -0,0 +1,148 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.options;
import java.util.Map;
import org.apache.hugegraph.store.business.DataMover;
import org.apache.hugegraph.store.pd.PdProvider;
import org.apache.hugegraph.store.raft.RaftTaskHandler;
import com.alipay.sofa.jraft.util.Utils;
import lombok.Data;
/**
* Storage engine configuration
*/
@Data
public class HgStoreEngineOptions {
public static String Raft_Path_Prefix = "raft";
public static String DB_Path_Prefix = "db";
public static String Snapshot_Path_Prefix = "snapshot";
// store心跳间隔，单位秒
private final int storeHBInterval = 30;
// 分区心跳间隔，单位秒
private final int partitionHBInterval = 5;
// 等待leader超时时间，单位秒
private final int waitLeaderTimeout = 30;
private final int raftRpcThreadPoolSize = Utils.cpus() * 6;
// 没有PD模式，用于开发调试使用
private boolean fakePD = false;
// fakePd配置项
private FakePdOptions fakePdOptions = new FakePdOptions();
private RaftOptions raftOptions = new RaftOptions();
// pd 服务器地址
private String pdAddress;
// 对外服务地址
private String grpcAddress;
// Raft 对外服务地址
private String raftAddress;
// 存储路径，支持多个位置，逗号分割
private String dataPath;
private String raftPath;
private Map<String, Object> rocksdbConfig;
// 自定义的标签，传给pd
private Map<String, String> labels;
// Raft任务处理器
private RaftTaskHandler taskHandler;
private PdProvider pdProvider;
// 数据迁移服务
private DataMover dataTransfer;
@Data
public static class FakePdOptions {
private int partitionCount = 0;
private int shardCount = 0;
private String storeList;
private String peersList;
}
@Data
public static class RaftOptions {
/*
* Rpc connect timeout in milliseconds
* The time should be less than electionTimeoutMs, otherwise the election will timeout
*/
private final int rpcConnectTimeoutMs = 1000;
private final int electionTimeoutMs = 3000;
// A follower would become a candidate if it doesn't receive any message
// from the leader in |election_timeout_ms| milliseconds
/**
* Install snapshot RPC request default timeout in milliseconds
*/
private final int rpcInstallSnapshotTimeout = 60 * 60 * 1000;
// 等待leader超时时间，单位秒
private final int waitLeaderTimeout = 30;
/**
* The maximum number of entries in AppendEntriesRequest
*/
private final int maxEntriesSize = 256;
/**
* Raft集群发生数据积压后，限速等待时间 单位毫秒
**/
private final int overloadRateLimit = 100;
private final int keepInMemorySegmentCount = 2;
private final int preAllocateSegmentCount = 1;
private final int splitPartitionLogIndexMargin = 10;
/**
* RPC request default timeout in milliseconds
*/
private int rpcDefaultTimeout = 5000;
// A snapshot saving would be triggered every |snapshot_interval_s| seconds
// if this was reset as a positive number
// If |snapshot_interval_s| <= 0, the time based snapshot would be disabled.
//
// Default: 3600 (1 hour)
private int snapshotIntervalSecs = 3600;
// A snapshot saving would be triggered every |snapshot_interval_s| seconds,
// and at this moment when state machine's lastAppliedIndex value
// minus lastSnapshotId value is greater than snapshotLogIndexMargin value,
// the snapshot action will be done really.
// If |snapshotLogIndexMargin| <= 0, the distance based snapshot would be disable.
//
// Default: 0
private int snapshotLogIndexMargin = 1024;
private boolean metrics = true;
/**
* Internal disruptor buffers size for Node/FSMCaller/LogManager etc.
*/
private int disruptorBufferSize = 4096;
/**
* The maximum replicator pipeline in-flight requests/responses, only valid when enable
* replicator pipeline.
*/
private int maxReplicatorInflightMsgs = 256;
/**
* The maximum byte size of log allowed by user.
*/
private long maxLogSize = 100 * 1024 * 1024;
/**
* The ratio of exponential approximation for average size of log entry.
*/
private double aveLogEntrySizeRatio = 0.95;
private boolean useRocksDBSegmentLogStorage = true;
private int maxSegmentFileSize = 64 * 1024 * 1024;
}
}

27
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/MetadataOptions.java Normal file
View File

@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.options;
import lombok.Data;
@Data
public class MetadataOptions {
private String dataPath;
private String raftPath;
}

48
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/PartitionEngineOptions.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.options;
import java.util.List;
import org.apache.hugegraph.store.raft.RaftTaskHandler;
import com.alipay.sofa.jraft.conf.Configuration;
import lombok.Data;
/**
* Partition engine configuration
*/
@Data
public class PartitionEngineOptions {
// 异步任务执行时间间隔, 单位秒
private final int taskScheduleTime = 60;
// 分裂过程，等待数据对齐超时时间
private final long splitPartitionTimeout = 30 * 60 * 1000;
HgStoreEngineOptions.RaftOptions raftOptions;
// raft存储路径
private String raftDataPath;
private String raftSnapShotPath;
private Integer groupId;
private String raftAddress;
private List<String> peerList;
private Configuration conf;
// raft 任务处理器
private RaftTaskHandler taskHandler;
}

209
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RaftRocksdbOptions.java Normal file
View File

@ -0,0 +1,209 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.options;
import java.util.Map;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.rocksdb.access.RocksDBOptions;
import org.apache.hugegraph.store.business.BusinessHandlerImpl;
import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.BloomFilter;
import org.rocksdb.Cache;
import org.rocksdb.ColumnFamilyOptions;
import org.rocksdb.CompressionType;
import org.rocksdb.DBOptions;
import org.rocksdb.Env;
import org.rocksdb.IndexType;
import org.rocksdb.LRUCache;
import org.rocksdb.RocksDB;
import org.rocksdb.WriteBufferManager;
import org.rocksdb.util.SizeUnit;
import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage;
import com.alipay.sofa.jraft.util.StorageOptionsFactory;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class RaftRocksdbOptions {
private static RocksdbConfig rocksdbConfig = null;
private static RocksdbConfig getRocksdbConfig(HugeConfig options) {
if (rocksdbConfig == null) {
synchronized (RocksdbConfig.class) {
rocksdbConfig = new RocksdbConfig(options);
}
}
return rocksdbConfig;
}
private static void registerRaftRocksdbConfig(HugeConfig options) {
Cache blockCache = new LRUCache(SizeUnit.GB);
BlockBasedTableConfig tableConfig = new BlockBasedTableConfig()
.setIndexType(IndexType.kTwoLevelIndexSearch)
.setPartitionFilters(true) //
.setMetadataBlockSize(8 * SizeUnit.KB) //
.setCacheIndexAndFilterBlocks(
options.get(RocksDBOptions.PUT_FILTER_AND_INDEX_IN_CACHE))
.setCacheIndexAndFilterBlocksWithHighPriority(true)
.setPinL0FilterAndIndexBlocksInCache(
options.get(RocksDBOptions.PIN_L0_FILTER_AND_INDEX_IN_CACHE))
.setBlockSize(4 * SizeUnit.KB)
.setBlockCache(blockCache);
StorageOptionsFactory.registerRocksDBTableFormatConfig(RocksDBLogStorage.class,
tableConfig);
DBOptions dbOptions = StorageOptionsFactory.getDefaultRocksDBOptions();
dbOptions.setEnv(rocksdbConfig.getEnv());
// raft rocksdb数量固定，通过max_write_buffer_number可以控制
//dbOptions.setWriteBufferManager(rocksdbConfig.getBufferManager());
dbOptions.setUnorderedWrite(true);
StorageOptionsFactory.registerRocksDBOptions(RocksDBLogStorage.class,
dbOptions);
ColumnFamilyOptions cfOptions =
StorageOptionsFactory.getDefaultRocksDBColumnFamilyOptions();
cfOptions.setTargetFileSizeBase(256 * SizeUnit.MB);
cfOptions.setWriteBufferSize(8 * SizeUnit.MB);
cfOptions.setNumLevels(3);
cfOptions.setMaxWriteBufferNumber(3);
cfOptions.setCompressionType(CompressionType.NO_COMPRESSION);
cfOptions.setMaxBytesForLevelBase(2048 * SizeUnit.GB);
StorageOptionsFactory.registerRocksDBColumnFamilyOptions(RocksDBLogStorage.class,
cfOptions);
}
public static void initRocksdbGlobalConfig(Map<String, Object> config) {
HugeConfig hugeConfig = BusinessHandlerImpl.initRocksdb(config, null);
RocksdbConfig rocksdbConfig = getRocksdbConfig(hugeConfig);
registerRaftRocksdbConfig(hugeConfig);
config.put(RocksDBOptions.ENV, rocksdbConfig.getEnv());
config.put(RocksDBOptions.WRITE_BUFFER_MANAGER, rocksdbConfig.getBufferManager());
config.put(RocksDBOptions.BLOCK_TABLE_CONFIG, rocksdbConfig.getTableConfig());
config.put(RocksDBOptions.BLOCK_CACHE, rocksdbConfig.getBlockCache());
config.put(RocksDBOptions.WRITE_CACHE, rocksdbConfig.getWriteCache());
}
public static WriteBufferManager getWriteBufferManager() {
return rocksdbConfig.getBufferManager();
}
public static Env getEnv() {
return rocksdbConfig.getEnv();
}
public static Cache getWriteCache() {
return rocksdbConfig.getWriteCache();
}
public static Cache getBlockCache() {
return rocksdbConfig.getBlockCache();
}
public static long getWriteCacheCapacity() {
return rocksdbConfig.getWriteCacheCapacity();
}
public static long getBlockCacheCapacity() {
return rocksdbConfig.getBlockCacheCapacity();
}
static class RocksdbConfig {
private final Env env;
private final LRUCache blockCache;
private final LRUCache writeCache;
private final WriteBufferManager bufferManager;
private final BlockBasedTableConfig tableConfig;
private final long blockCacheCapacity;
private final long writeCacheCapacity;
public RocksdbConfig(HugeConfig options) {
RocksDB.loadLibrary();
this.env = Env.getDefault();
double writeBufferRatio = options.get(RocksDBOptions.WRITE_BUFFER_RATIO);
this.writeCacheCapacity =
(long) (options.get(RocksDBOptions.TOTAL_MEMORY_SIZE) * writeBufferRatio);
this.blockCacheCapacity =
options.get(RocksDBOptions.TOTAL_MEMORY_SIZE) - writeCacheCapacity;
this.writeCache = new LRUCache(writeCacheCapacity);
this.blockCache = new LRUCache(blockCacheCapacity);
this.bufferManager = new WriteBufferManager(writeCacheCapacity, writeCache,
options.get(
RocksDBOptions.WRITE_BUFFER_ALLOW_STALL));
this.tableConfig = new BlockBasedTableConfig() //
.setIndexType(
IndexType.kTwoLevelIndexSearch) //
.setPartitionFilters(true) //
.setMetadataBlockSize(8 * SizeUnit.KB) //
.setCacheIndexAndFilterBlocks(
options.get(
RocksDBOptions.PUT_FILTER_AND_INDEX_IN_CACHE)) //
.setCacheIndexAndFilterBlocksWithHighPriority(
true) //
.setPinL0FilterAndIndexBlocksInCache(
options.get(
RocksDBOptions.PIN_L0_FILTER_AND_INDEX_IN_CACHE)) //
.setBlockSize(4 * SizeUnit.KB)//
.setBlockCache(blockCache);
int bitsPerKey = options.get(RocksDBOptions.BLOOM_FILTER_BITS_PER_KEY);
if (bitsPerKey >= 0) {
tableConfig.setFilterPolicy(new BloomFilter(bitsPerKey,
options.get(
RocksDBOptions.BLOOM_FILTER_MODE)));
}
tableConfig.setWholeKeyFiltering(
options.get(RocksDBOptions.BLOOM_FILTER_WHOLE_KEY));
log.info("RocksdbConfig {}", options.get(RocksDBOptions.BLOOM_FILTER_BITS_PER_KEY));
}
public Env getEnv() {
return env;
}
public LRUCache getBlockCache() {
return blockCache;
}
public LRUCache getWriteCache() {
return writeCache;
}
public WriteBufferManager getBufferManager() {
return bufferManager;
}
public BlockBasedTableConfig getTableConfig() {
return tableConfig;
}
public long getBlockCacheCapacity() {
return blockCacheCapacity;
}
public long getWriteCacheCapacity() {
return writeCacheCapacity;
}
}
}

472
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/DefaultPdProvider.java Normal file
View File

@ -0,0 +1,472 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.pd;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.client.PDConfig;
import org.apache.hugegraph.pd.client.PDPulse;
import org.apache.hugegraph.pd.client.PDPulseImpl;
import org.apache.hugegraph.pd.common.KVPair;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.MetaTask;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatRequest;
import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatResponse;
import org.apache.hugegraph.pd.grpc.pulse.PdInstructionType;
import org.apache.hugegraph.pd.grpc.pulse.PulseResponse;
import org.apache.hugegraph.pd.grpc.watch.WatchGraphResponse;
import org.apache.hugegraph.pd.grpc.watch.WatchResponse;
import org.apache.hugegraph.pd.pulse.PulseServerNotice;
import org.apache.hugegraph.pd.watch.NodeEvent;
import org.apache.hugegraph.pd.watch.PartitionEvent;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.meta.Graph;
import org.apache.hugegraph.store.meta.GraphManager;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.meta.Store;
import org.apache.hugegraph.store.metric.HgMetricService;
import org.apache.hugegraph.store.util.Asserts;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DefaultPdProvider implements PdProvider {
private static final Logger LOG = Log.logger(DefaultPdProvider.class);
private final PDClient pdClient;
private final String pdServerAddress;
private final PDPulse pulseClient;
private Consumer<Throwable> hbOnError = null;
private List<PartitionInstructionListener> partitionCommandListeners;
private PDPulse.Notifier<PartitionHeartbeatRequest.Builder> pdPulse;
private GraphManager graphManager = null;
PDClient.PDEventListener listener = new PDClient.PDEventListener() {
// 监听pd变更信息的listener
@Override
public void onStoreChanged(NodeEvent event) {
if (event.getEventType() == NodeEvent.EventType.NODE_RAFT_CHANGE) {
log.info("store raft group changed!, {}", event);
pdClient.invalidStoreCache(event.getNodeId());
HgStoreEngine.getInstance().rebuildRaftGroup(event.getNodeId());
} else if (event.getEventType() == NodeEvent.EventType.NODE_PD_LEADER_CHANGE) {
log.info("pd leader changed!, {}. restart heart beat", event);
if (pulseClient.resetStub(event.getGraph(), pdPulse)) {
startHeartbeatStream(hbOnError);
}
}
}
@Override
public void onPartitionChanged(PartitionEvent event) {
}
@Override
public void onGraphChanged(WatchResponse event) {
WatchGraphResponse graphResponse = event.getGraphResponse();
Metapb.Graph graph = graphResponse.getGraph();
if (graphManager != null) {
graphManager.updateGraph(new Graph(graph));
}
}
};
public DefaultPdProvider(String pdAddress) {
this.pdClient = PDClient.create(PDConfig.of(pdAddress).setEnableCache(true));
this.pdClient.addEventListener(listener);
this.pdServerAddress = pdAddress;
partitionCommandListeners = Collections.synchronizedList(new ArrayList());
log.info("pulse client connect to {}", pdClient.getLeaderIp());
this.pulseClient = new PDPulseImpl(pdClient.getLeaderIp());
}
@Override
public long registerStore(Store store) throws PDException {
Asserts.isTrue(this.pdClient != null, "pd client is null");
LOG.info("registerStore pd={} storeId={}, store={}", this.pdServerAddress, store.getId(),
store);
long storeId = 0;
Metapb.Store protoObj = store.getProtoObj();
try {
storeId = pdClient.registerStore(protoObj);
store.setId(storeId);
if (pdClient.getStore(storeId).getState() != Metapb.StoreState.Up) {
LOG.warn("Store {} is not activated, state is {}", storeId,
pdClient.getStore(storeId).getState());
}
} catch (PDException e) {
LOG.error(
"Exception in storage registration, StoreID= {} pd= {} exceptCode= {} except=" +
" {}.",
protoObj.getId(), this.pdServerAddress, e.getErrorCode(), e.getMessage());
storeId = 0;
throw e;
} catch (Exception e) {
LOG.error(
"Exception in storage registration, StoreID= {} pd= {} except= {}, Please " +
"check your network settings.",
protoObj.getId(), this.pdServerAddress, e.getMessage());
handleCommonException(e);
storeId = 0;
}
return storeId;
}
@Override
public Partition getPartitionByID(String graph, int partId) {
try {
KVPair<Metapb.Partition, Metapb.Shard> pair = pdClient.getPartitionById(
graph, partId);
if (null != pair) {
return new Partition(pair.getKey());
}
} catch (PDException e) {
log.error("Partition {}-{} getPartitionByID exception {}", graph, partId, e);
}
return null;
}
@Override
public Metapb.Shard getPartitionLeader(String graph, int partId) {
try {
KVPair<Metapb.Partition, Metapb.Shard> pair = pdClient.getPartitionById(
graph, partId);
if (null != pair) {
return pair.getValue();
}
} catch (PDException e) {
log.error("Partition {}-{} getPartitionByID exception {}", graph, partId, e);
}
return null;
}
@Override
public Metapb.Partition getPartitionByCode(String graph, int code) {
try {
KVPair<Metapb.Partition, Metapb.Shard> pair = pdClient.getPartitionByCode(
graph, code);
if (null != pair) {
return pair.getKey();
}
} catch (PDException e) {
log.error("Partition {} getPartitionByCode {} exception {}", graph, code, e);
}
return null;
}
@Override
public Partition delPartition(String graph, int partId) {
log.info("Partition {}-{} send delPartition to PD", graph, partId);
try {
Metapb.Partition partition = pdClient.delPartition(graph, partId);
if (null != partition) {
return new Partition(partition);
}
} catch (PDException e) {
log.error("Partition {}-{} remove exception {}", graph, partId, e);
}
return null;
}
@Override
public List<Metapb.Partition> updatePartition(List<Metapb.Partition> partitions) throws
PDException {
try {
List<Metapb.Partition> results = pdClient.updatePartition(partitions);
return results;
} catch (PDException e) {
throw e;
}
}
@Override
public List<Partition> getPartitionsByStore(long storeId) throws PDException {
List<Partition> partitions = new ArrayList<>();
List<Metapb.Partition> parts = pdClient.getPartitionsByStore(storeId);
parts.forEach(e -> {
partitions.add(new Partition(e));
});
return partitions;
}
@Override
public void updatePartitionCache(Partition partition, Boolean changeLeader) {
Metapb.Shard leader = null;
var shardGroup = getShardGroup(partition.getId());
if (shardGroup != null) {
for (Metapb.Shard shard : shardGroup.getShardsList()) {
if (shard.getRole() == Metapb.ShardRole.Leader) {
leader = shard;
}
}
}
if (!changeLeader) {
try {
leader = pdClient.getPartitionById(partition.getGraphName(), partition.getId())
.getValue();
} catch (PDException e) {
log.error("find leader error,leader changed to storeId:{}", leader.getStoreId());
} catch (Exception e1) {
log.error("exception ", e1);
}
}
pdClient.updatePartitionCache(partition.getProtoObj(), leader);
}
@Override
public void invalidPartitionCache(String graph, int partId) {
pdClient.invalidPartitionCache(graph, partId);
}
/**
* 启动partition心跳流式传输
*
* @return
*/
@Override
public boolean startHeartbeatStream(Consumer<Throwable> onError) {
this.hbOnError = onError;
pdPulse = pulseClient.connectPartition(new PDPulse.Listener<>() {
@Override
public void onNotice(PulseServerNotice<PulseResponse> response) {
PulseResponse content = response.getContent();
// 消息消费应答，能够正确消费消息，调用accept返回状态码，否则不要调用accept
Consumer<Integer> consumer = integer -> {
LOG.debug("Partition heartbeat accept instruction: {}", content);
// LOG.info("accept notice id : {}, ts:{}", response.getNoticeId(), System
// .currentTimeMillis());
// http2 并发问题，需要加锁
// synchronized (pdPulse) {
response.ack();
// }
};
if (content.hasInstructionResponse()) {
var pdInstruction = content.getInstructionResponse();
consumer.accept(0);
// 当前的链接变成了follower，重新链接
if (pdInstruction.getInstructionType() ==
PdInstructionType.CHANGE_TO_FOLLOWER) {
onCompleted();
log.info("got pulse instruction, change leader to {}",
pdInstruction.getLeaderIp());
if (pulseClient.resetStub(pdInstruction.getLeaderIp(), pdPulse)) {
startHeartbeatStream(hbOnError);
}
}
return;
}
PartitionHeartbeatResponse instruct = content.getPartitionHeartbeatResponse();
LOG.debug("Partition heartbeat receive instruction: {}", instruct);
Partition partition = new Partition(instruct.getPartition());
for (PartitionInstructionListener event : partitionCommandListeners) {
if (instruct.hasChangeShard()) {
event.onChangeShard(instruct.getId(), partition, instruct
.getChangeShard(),
consumer);
}
if (instruct.hasSplitPartition()) {
event.onSplitPartition(instruct.getId(), partition,
instruct.getSplitPartition(), consumer);
}
if (instruct.hasTransferLeader()) {
event.onTransferLeader(instruct.getId(), partition,
instruct.getTransferLeader(), consumer);
}
if (instruct.hasDbCompaction()) {
event.onDbCompaction(instruct.getId(), partition,
instruct.getDbCompaction(), consumer);
}
if (instruct.hasMovePartition()) {
event.onMovePartition(instruct.getId(), partition,
instruct.getMovePartition(), consumer);
}
if (instruct.hasCleanPartition()) {
event.onCleanPartition(instruct.getId(), partition,
instruct.getCleanPartition(),
consumer);
}
if (instruct.hasKeyRange()) {
event.onPartitionKeyRangeChanged(instruct.getId(), partition,
instruct.getKeyRange(),
consumer);
}
}
}
@Override
public void onError(Throwable throwable) {
LOG.error("Partition heartbeat stream error. {}", throwable);
pulseClient.resetStub(pdClient.getLeaderIp(), pdPulse);
onError.accept(throwable);
}
@Override
public void onCompleted() {
LOG.info("Partition heartbeat stream complete");
}
});
return true;
}
/**
* 添加服务端消息监听
*
* @param listener
* @return
*/
@Override
public boolean addPartitionInstructionListener(PartitionInstructionListener listener) {
partitionCommandListeners.add(listener);
return true;
}
@Override
public boolean partitionHeartbeat(List<Metapb.PartitionStats> statsList) {
for (Metapb.PartitionStats stats : statsList) {
PartitionHeartbeatRequest.Builder request = PartitionHeartbeatRequest.newBuilder()
.setStates(stats);
pdPulse.notifyServer(request);
}
return false;
}
@Override
public boolean isLocalPartition(long storeId, int partitionId) {
try {
return !pdClient.queryPartitions(storeId, partitionId).isEmpty();
} catch (PDException e) {
log.error("isLocalPartition exception ", e);
}
return false;
}
@Override
public Metapb.Graph getGraph(String graphName) throws PDException {
return pdClient.getGraph(graphName);
}
@Override
public void reportTask(MetaTask.Task task) throws PDException {
pdClient.reportTask(task);
}
@Override
public PDClient getPDClient() {
return this.pdClient;
}
@Override
public boolean updatePartitionLeader(String graphName, int partId, long leaderStoreId) {
this.pdClient.updatePartitionLeader(graphName, partId, leaderStoreId);
return true;
}
@Override
public Store getStoreByID(Long storeId) {
try {
return new Store(pdClient.getStore(storeId));
} catch (PDException e) {
log.error("getStoreByID exception {}", e);
}
return null;
}
@Override
public Metapb.ClusterStats getClusterStats() {
try {
return pdClient.getClusterStats();
} catch (PDException e) {
log.error("getClusterStats exception {}", e);
return Metapb.ClusterStats.newBuilder()
.setState(Metapb.ClusterState.Cluster_Fault).build();
}
}
@Override
public Metapb.ClusterStats storeHeartbeat(Store node) throws PDException {
LOG.debug("storeHeartbeat node id: {}", node.getId());
try {
Metapb.StoreStats.Builder stats = HgMetricService.getInstance().getMetrics();
LOG.debug("storeHeartbeat StoreStats: {}", stats);
stats.setCores(node.getCores());
return pdClient.storeHeartbeat(stats.build());
} catch (PDException e) {
throw e;
} catch (Exception e) {
LOG.warn("Store {} report heartbeat exception: {}", node.getId(), e.toString());
}
return Metapb.ClusterStats.newBuilder()
.setState(Metapb.ClusterState.Cluster_Fault).build();
}
private void handleCommonException(Exception e) {
}
@Override
public GraphManager getGraphManager() {
return graphManager;
}
@Override
public void setGraphManager(GraphManager graphManager) {
this.graphManager = graphManager;
}
@Override
public void deleteShardGroup(int groupId) throws PDException {
pdClient.deleteShardGroup(groupId);
}
@Override
public Metapb.ShardGroup getShardGroup(int partitionId) {
try {
return pdClient.getShardGroup(partitionId);
} catch (PDException e) {
log.error("get shard group :{} from pd failed: {}", partitionId, e.getMessage());
}
return null;
}
@Override
public void updateShardGroup(Metapb.ShardGroup shardGroup) throws PDException {
pdClient.updateShardGroup(shardGroup);
}
}

264
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/FakePdServiceProvider.java Normal file
View File

@ -0,0 +1,264 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.pd;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.common.PartitionUtils;
import org.apache.hugegraph.pd.grpc.MetaTask;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.pd.grpc.Pdpb;
import org.apache.hugegraph.store.meta.GraphManager;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.meta.Store;
import org.apache.hugegraph.store.options.HgStoreEngineOptions;
import lombok.extern.slf4j.Slf4j;
/**
* 内置PD服务，用于单机部署或开发调试
*/
@Slf4j
public class FakePdServiceProvider implements PdProvider {
private final Map<Long, Store> stores;
private final int shardCount = 0;
private final Map<String, Metapb.Partition> partitions = new ConcurrentHashMap<>();
private int partitionCount = 0;
private GraphManager graphManager = null;
public FakePdServiceProvider(HgStoreEngineOptions.FakePdOptions options) {
stores = new LinkedHashMap<>();
if (options != null) {
String[] storeList = options.getStoreList().split(",");
String[] peersList = options.getPeersList().split(",");
for (int i = 0; i < storeList.length; i++) {
if (!storeList[i].isEmpty()) {
addStore(storeList[i], peersList[i]);
}
}
}
this.partitionCount = options.getPartitionCount();
}
public static long makeStoreId(String storeAddress) {
return storeAddress.hashCode();
}
/**
* For unit test
*
* @return
*/
public static Store getDefaultStore() {
Store store = new Store();
store.setId(1);
store.setStoreAddress("127.0.0.1:8501");
store.setRaftAddress("127.0.0.1:8511");
store.setPartitionCount(1);
return store;
}
private void addStore(String storeAddr, String raftAddr) {
Store store = new Store() {{
setId(makeStoreId(storeAddr));
setRaftAddress(raftAddr);
setStoreAddress(storeAddr);
}};
stores.put(store.getId(), store);
}
public void addStore(Store store) {
stores.put(store.getId(), store);
}
@Override
public long registerStore(Store store) throws PDException {
log.info("registerStore storeId:{}, storeAddress:{}", store.getId(),
store.getStoreAddress());
// id 不匹配，禁止登录
if (store.getId() != 0 && store.getId() != makeStoreId(store.getStoreAddress())) {
throw new PDException(Pdpb.ErrorType.STORE_ID_NOT_EXIST_VALUE,
"Store id does not matched");
}
if (!stores.containsKey(makeStoreId(store.getStoreAddress()))) {
store.setId(makeStoreId(store.getStoreAddress()));
stores.put(store.getId(), store);
}
Store s = stores.get(makeStoreId(store.getStoreAddress()));
store.setId(s.getId());
return store.getId();
}
@Override
public Partition getPartitionByID(String graph, int partId) {
List<Store> storeList = new ArrayList(stores.values());
int shardCount = this.shardCount;
if (shardCount == 0 || shardCount >= stores.size()) {
shardCount = stores.size();
}
int storeIdx = partId % storeList.size();
List<Metapb.Shard> shards = new ArrayList<>();
for (int i = 0; i < shardCount; i++) {
Metapb.Shard shard =
Metapb.Shard.newBuilder().setStoreId(storeList.get(storeIdx).getId())
.setRole(i == 0 ? Metapb.ShardRole.Leader :
Metapb.ShardRole.Follower) //
.build();
shards.add(shard);
storeIdx = (storeIdx + 1) >= storeList.size() ? 0 : ++storeIdx; // 顺序选择
}
int partLength = getPartitionLength();
Metapb.Partition partition = Metapb.Partition.newBuilder()
.setGraphName(graph)
.setId(partId)
.setStartKey(partLength * partId)
.setEndKey(partLength * (partId + 1))
//.addAllShards(shards)
.build();
return new Partition(partition);
}
@Override
public Metapb.Shard getPartitionLeader(String graph, int partId) {
return null;
}
private int getPartitionLength() {
return PartitionUtils.MAX_VALUE / (partitionCount == 0 ? stores.size() : partitionCount) +
1;
}
@Override
public Metapb.Partition getPartitionByCode(String graph, int code) {
int partId = code / getPartitionLength();
return getPartitionByID(graph, partId).getProtoObj();
}
@Override
public Partition delPartition(String graph, int partId) {
return null;
}
@Override
public List<Metapb.Partition> updatePartition(List<Metapb.Partition> partitions) {
return partitions;
}
@Override
public List<Partition> getPartitionsByStore(long storeId) throws PDException {
return new ArrayList<>();
}
@Override
public void updatePartitionCache(Partition partition, Boolean changeLeader) {
}
@Override
public void invalidPartitionCache(String graph, int partId) {
}
@Override
public boolean startHeartbeatStream(Consumer<Throwable> onError) {
return false;
}
@Override
public boolean addPartitionInstructionListener(PartitionInstructionListener listener) {
return false;
}
@Override
public boolean partitionHeartbeat(List<Metapb.PartitionStats> statsList) {
return true;
}
@Override
public boolean isLocalPartition(long storeId, int partitionId) {
return true;
}
@Override
public Metapb.Graph getGraph(String graphName) {
return Metapb.Graph.newBuilder().setGraphName(graphName)
//.setId(PartitionUtils.calcHashcode(graphName.getBytes()))
.build();
}
@Override
public void reportTask(MetaTask.Task task) throws PDException {
}
@Override
public PDClient getPDClient() {
return null;
}
@Override
public Store getStoreByID(Long storeId) {
return stores.get(storeId);
}
@Override
public Metapb.ClusterStats getClusterStats() {
return Metapb.ClusterStats.newBuilder()
.setState(Metapb.ClusterState.Cluster_OK).build();
}
@Override
public Metapb.ClusterStats storeHeartbeat(Store node) {
return getClusterStats();
}
@Override
public boolean updatePartitionLeader(String graphName, int partId, long leaderStoreId) {
return false;
}
@Override
public GraphManager getGraphManager() {
return graphManager;
}
@Override
public void setGraphManager(GraphManager graphManager) {
this.graphManager = graphManager;
}
@Override
public void deleteShardGroup(int groupId) {
}
}

54
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/PartitionInstructionListener.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.pd;
import java.util.function.Consumer;
import org.apache.hugegraph.pd.grpc.pulse.ChangeShard;
import org.apache.hugegraph.pd.grpc.pulse.CleanPartition;
import org.apache.hugegraph.pd.grpc.pulse.DbCompaction;
import org.apache.hugegraph.pd.grpc.pulse.MovePartition;
import org.apache.hugegraph.pd.grpc.pulse.PartitionKeyRange;
import org.apache.hugegraph.pd.grpc.pulse.SplitPartition;
import org.apache.hugegraph.pd.grpc.pulse.TransferLeader;
import org.apache.hugegraph.store.meta.Partition;
public interface PartitionInstructionListener {
void onChangeShard(long taskId, Partition partition, ChangeShard changeShard,
Consumer<Integer> consumer);
void onTransferLeader(long taskId, Partition partition, TransferLeader transferLeader,
Consumer<Integer> consumer);
void onSplitPartition(long taskId, Partition partition, SplitPartition splitPartition,
Consumer<Integer> consumer);
void onDbCompaction(long taskId, Partition partition, DbCompaction rocksdbCompaction,
Consumer<Integer> consumer);
void onMovePartition(long taskId, Partition partition, MovePartition movePartition,
Consumer<Integer> consumer);
void onCleanPartition(long taskId, Partition partition, CleanPartition cleanPartition,
Consumer<Integer> consumer);
void onPartitionKeyRangeChanged(long taskId, Partition partition,
PartitionKeyRange partitionKeyRange,
Consumer<Integer> consumer);
}

92
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/PdProvider.java Normal file
View File

@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.pd;
import java.util.List;
import java.util.function.Consumer;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.MetaTask;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.meta.GraphManager;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.meta.Store;
import org.apache.hugegraph.store.util.HgStoreException;
public interface PdProvider {
long registerStore(Store store) throws PDException;
Store getStoreByID(Long storeId);
Metapb.ClusterStats getClusterStats();
Metapb.ClusterStats storeHeartbeat(Store node) throws HgStoreException, PDException;
Partition getPartitionByID(String graph, int partId);
Metapb.Shard getPartitionLeader(String graph, int partId);
Metapb.Partition getPartitionByCode(String graph, int code);
Partition delPartition(String graph, int partId);
List<Metapb.Partition> updatePartition(List<Metapb.Partition> partitions) throws PDException;
List<Partition> getPartitionsByStore(long storeId) throws PDException;
void updatePartitionCache(Partition partition, Boolean changeLeader);
void invalidPartitionCache(String graph, int partId);
boolean startHeartbeatStream(Consumer<Throwable> onError);
boolean addPartitionInstructionListener(PartitionInstructionListener listener);
boolean partitionHeartbeat(List<Metapb.PartitionStats> statsList);
boolean isLocalPartition(long storeId, int partitionId);
Metapb.Graph getGraph(String graphName) throws PDException;
void reportTask(MetaTask.Task task) throws PDException;
PDClient getPDClient();
boolean updatePartitionLeader(String graphName, int partId, long leaderStoreId);
GraphManager getGraphManager();
void setGraphManager(GraphManager graphManager);
/**
* 删除分区 shard group
*
* @param groupId
*/
void deleteShardGroup(int groupId) throws PDException;
default Metapb.ShardGroup getShardGroup(int partitionId) {
return null;
}
default void updateShardGroup(Metapb.ShardGroup shardGroup) throws PDException {
}
}

270
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/HgStoreStateMachine.java Normal file
View File

@ -0,0 +1,270 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.raft;
import java.util.Base64;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hugegraph.store.snapshot.HgSnapshotHandler;
import org.apache.hugegraph.store.util.HgStoreException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alipay.sofa.jraft.Closure;
import com.alipay.sofa.jraft.Iterator;
import com.alipay.sofa.jraft.Status;
import com.alipay.sofa.jraft.conf.Configuration;
import com.alipay.sofa.jraft.core.StateMachineAdapter;
import com.alipay.sofa.jraft.entity.LeaderChangeContext;
import com.alipay.sofa.jraft.entity.RaftOutter;
import com.alipay.sofa.jraft.error.RaftError;
import com.alipay.sofa.jraft.error.RaftException;
import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
import com.alipay.sofa.jraft.util.Utils;
/**
* Raft 状态机
*/
public class HgStoreStateMachine extends StateMachineAdapter {
private static final Logger LOG = LoggerFactory.getLogger(HgStoreStateMachine.class);
private final AtomicLong leaderTerm = new AtomicLong(-1);
private final HgSnapshotHandler snapshotHandler;
private final List<RaftTaskHandler> taskHandlers;
private final List<RaftStateListener> stateListeners;
private final Integer groupId;
private long committedIndex;
public HgStoreStateMachine(Integer groupId, HgSnapshotHandler snapshotHandler) {
this.groupId = groupId;
this.snapshotHandler = snapshotHandler;
this.stateListeners = new CopyOnWriteArrayList<>();
this.taskHandlers = new CopyOnWriteArrayList<>();
}
public void addTaskHandler(RaftTaskHandler handler) {
taskHandlers.add(handler);
}
public void addStateListener(RaftStateListener listener) {
stateListeners.add(listener);
}
public boolean isLeader() {
return this.leaderTerm.get() > 0;
}
@Override
public void onApply(Iterator inter) {
while (inter.hasNext()) {
final RaftClosureAdapter done = (RaftClosureAdapter) inter.done();
try {
for (RaftTaskHandler taskHandler : taskHandlers) {
if (done != null) {
// Leader分支，本地调用
if (taskHandler.invoke(groupId, done.op.getOp(), done.op.getReq(),
done.closure)) {
done.run(Status.OK());
break;
}
} else {
if (taskHandler.invoke(groupId, inter.getData().array(), null)) {
break;
}
}
}
} catch (Throwable t) {
LOG.info("{}", Base64.getEncoder().encode(inter.getData().array()));
LOG.error("StateMachine{} meet critical error: .", groupId, t);
if (done != null) {
LOG.error("StateMachine meet critical error: op = {} {}.", done.op.getOp(),
done.op.getReq());
// done.run(new Status(RaftError.EINTERNAL, t.getMessage()));
}
}
committedIndex = inter.getIndex();
stateListeners.forEach(listener -> {
listener.onDataCommitted(committedIndex);
});
// 清理数据
if (done != null) {
done.clear();
}
// 遍历下一条
inter.next();
}
}
public long getCommittedIndex() {
return committedIndex;
}
public long getLeaderTerm() {
return leaderTerm.get();
}
@Override
public void onError(final RaftException e) {
LOG.error("Raft {} StateMachine on error {}", groupId, e);
Utils.runInThread(() -> {
stateListeners.forEach(listener -> {
listener.onError(e);
});
});
}
@Override
public void onShutdown() {
super.onShutdown();
}
@Override
public void onLeaderStart(final long term) {
this.leaderTerm.set(term);
super.onLeaderStart(term);
Utils.runInThread(() -> {
stateListeners.forEach(listener -> {
listener.onLeaderStart(term);
});
});
LOG.info("Raft {} becomes leader ", groupId);
}
@Override
public void onLeaderStop(final Status status) {
Utils.runInThread(() -> {
stateListeners.forEach(listener -> {
listener.onLeaderStop(this.leaderTerm.get());
});
});
this.leaderTerm.set(-1);
super.onLeaderStop(status);
LOG.info("Raft {} lost leader ", groupId);
}
@Override
public void onStartFollowing(final LeaderChangeContext ctx) {
super.onStartFollowing(ctx);
Utils.runInThread(() -> {
stateListeners.forEach(listener -> {
listener.onStartFollowing(ctx.getLeaderId(), ctx.getTerm());
});
});
LOG.info("Raft {} start following: {}.", groupId, ctx);
}
@Override
public void onStopFollowing(final LeaderChangeContext ctx) {
super.onStopFollowing(ctx);
Utils.runInThread(() -> {
stateListeners.forEach(listener -> {
listener.onStopFollowing(ctx.getLeaderId(), ctx.getTerm());
});
});
LOG.info("Raft {} stop following: {}.", groupId, ctx);
}
@Override
public void onConfigurationCommitted(final Configuration conf) {
stateListeners.forEach(listener -> {
Utils.runInThread(() -> {
try {
listener.onConfigurationCommitted(conf);
} catch (Exception e) {
LOG.error("Raft {} onConfigurationCommitted {}", groupId, e);
}
});
});
LOG.info("Raft {} onConfigurationCommitted {}", groupId, conf);
}
@Override
public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
Utils.runInThread(() -> {
try {
snapshotHandler.onSnapshotSave(writer);
LOG.info("Raft {} onSnapshotSave success", groupId);
done.run(Status.OK());
} catch (HgStoreException e) {
LOG.error("Raft {} onSnapshotSave failed. {}", groupId, e.toString());
done.run(new Status(RaftError.EIO, e.toString()));
}
});
}
@Override
public boolean onSnapshotLoad(final SnapshotReader reader) {
try {
RaftOutter.SnapshotMeta meta = reader.load();
if (meta != null) {
this.committedIndex = meta.getLastIncludedIndex();
LOG.info("onSnapshotLoad committedIndex = {}", this.committedIndex);
} else {
LOG.error("onSnapshotLoad failed to get SnapshotMeta");
return false;
}
} catch (Exception e) {
LOG.error("onSnapshotLoad failed to get SnapshotMeta. {}", e.toString());
return false;
}
if (isLeader()) {
LOG.warn("Leader is not supposed to load snapshot");
return false;
}
try {
snapshotHandler.onSnapshotLoad(reader, this.committedIndex);
LOG.info("Raft {} onSnapshotLoad success", groupId);
return true;
} catch (HgStoreException e) {
LOG.error("Raft {} onSnapshotLoad failed. {}", groupId, e.toString());
return false;
}
}
public static class RaftClosureAdapter implements RaftClosure {
private final RaftClosure closure;
private RaftOperation op;
public RaftClosureAdapter(RaftOperation op, RaftClosure closure) {
this.op = op;
this.closure = closure;
}
@Override
public void run(Status status) {
closure.run(status);
}
public RaftClosure getClosure() {
return closure;
}
public void clear() {
op = null;
}
}
}

27
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftClosure.java Normal file
View File

@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.raft;
import com.alipay.sofa.jraft.Closure;
public interface RaftClosure extends Closure {
default void onLeaderChanged(Integer partId, Long storeId) {
}
}

127
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftOperation.java Normal file
View File

@ -0,0 +1,127 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.raft;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.caucho.hessian.io.Hessian2Input;
import com.caucho.hessian.io.Hessian2Output;
import com.google.protobuf.CodedOutputStream;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Data
public class RaftOperation {
public static final byte SYNC_PARTITION_TASK = 0x60;
public static final byte SYNC_PARTITION = 0x61;
public static final byte BLANK_TASK = 0x62;
public static final byte DO_SNAPSHOT = 0x63;
// 集群内部数据迁移操作
public static final byte IN_WRITE_OP = 0x64;
public static final byte IN_CLEAN_OP = 0x65;
public static final byte RAFT_UPDATE_PARTITION = 0x66;
public static final byte DB_COMPACTION = 0x67;
final static byte[] EMPTY_Bytes = new byte[0];
private static final Logger LOG = LoggerFactory.getLogger(RaftOperation.class);
private byte[] values; // req序列化的结果，用于传输给其他raft node
private Object req; // 原始对象，用于本机处理，减少一次反序列化操作
private byte op; // 操作类型
public static RaftOperation create(final byte op) {
try {
RaftOperation operation = new RaftOperation();
operation.setOp(op);
operation.setReq(null);
operation.setValues(toByteArray(op));
return operation;
} catch (Exception e) {
LOG.error("create error", e);
return null;
}
}
public static RaftOperation create(final byte op, final byte[] values, final Object req) {
RaftOperation operation = new RaftOperation();
operation.setOp(op);
operation.setReq(req);
operation.setValues(values);
return operation;
}
public static RaftOperation create(final byte op, final Object req) {
try {
RaftOperation operation = new RaftOperation();
operation.setOp(op);
operation.setReq(req);
operation.setValues(toByteArray(op, req));
return operation;
} catch (Exception e) {
log.error("exception ", e);
}
return null;
}
public static RaftOperation create(final byte op,
final com.google.protobuf.GeneratedMessageV3 req) throws
IOException {
// 序列化，
final byte[] buffer = new byte[req.getSerializedSize() + 1];
final CodedOutputStream output = CodedOutputStream.newInstance(buffer);
output.write(op);
req.writeTo(output);
output.checkNoSpaceLeft();
output.flush();
return create(op, buffer, req);
}
public static byte[] toByteArray(final byte op) throws IOException {
try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
bos.write(op);
bos.flush();
return bos.toByteArray();
}
}
public static byte[] toByteArray(final byte op, final Object obj) throws IOException {
try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
bos.write(op);
Hessian2Output output = new Hessian2Output(bos);
output.writeObject(obj);
output.flush();
return bos.toByteArray();
}
}
public static Object toObject(final byte[] bytes, int offset) throws IOException {
try (ByteArrayInputStream bis = new ByteArrayInputStream(bytes, offset + 1,
bytes.length - offset)) {
Hessian2Input input = new Hessian2Input(bis);
Object obj = input.readObject();
input.close();
return obj;
}
}
}

87
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftStateListener.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.raft;
import com.alipay.sofa.jraft.conf.Configuration;
import com.alipay.sofa.jraft.entity.PeerId;
import com.alipay.sofa.jraft.error.RaftException;
public interface RaftStateListener {
/**
* Called when current node becomes leader.
*
* @param newTerm the new term
*/
void onLeaderStart(final long newTerm);
/**
* Called when current node loses leadership.
*
* @param oldTerm the old term
*/
default void onLeaderStop(final long oldTerm) {
}
/**
* This method is called when a follower or candidate starts following a leader and its leaderId
* (should be NULL before the method is called) is set to the leader's id, situations including:
* 1. A candidate receives appendEntries request from a leader
* 2. A follower(without leader) receives appendEntries from a leader
* <p>
* The parameters gives the information(leaderId and term) about the very
* leader whom the follower starts to follow.
* User can reset the node's information as it starts to follow some leader.
*
* @param newLeaderId the new leader id whom the follower starts to follow
* @param newTerm the new term
*/
default void onStartFollowing(final PeerId newLeaderId, final long newTerm) {
}
/**
* This method is called when a follower stops following a leader and its leaderId becomes null,
* situations including:
* 1. Handle election timeout and start preVote
* 2. Receive requests with higher term such as VoteRequest from a candidate
* or appendEntries request from a new leader
* 3. Receive timeoutNow request from current leader and start request vote.
* <p>
* The parameters gives the information(leaderId and term) about the very leader
* whom the follower followed before.
* User can reset the node's information as it stops following some leader.
*
* @param oldLeaderId the old leader id whom the follower followed before
* @param oldTerm the old term
*/
default void onStopFollowing(final PeerId oldLeaderId, final long oldTerm) {
}
/**
* Invoked when a configuration has been committed to the group.
*
* @param conf committed configuration
*/
default void onConfigurationCommitted(final Configuration conf) {
}
default void onDataCommitted(long index) {
}
void onError(final RaftException e);
}

32
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftTaskHandler.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.raft;
import org.apache.hugegraph.store.util.HgStoreException;
/**
* 接收raft发送的数据
*/
public interface RaftTaskHandler {
boolean invoke(final int groupId, final byte[] request, RaftClosure response) throws
HgStoreException;
boolean invoke(final int groupId, final byte methodId, final Object req,
RaftClosure response) throws HgStoreException;
}

88
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/util/RaftUtils.java Normal file
View File

@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.raft.util;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.collections.ListUtils;
import com.alipay.sofa.jraft.Node;
import com.alipay.sofa.jraft.conf.Configuration;
public class RaftUtils {
public static List<String> getAllEndpoints(Node node) {
List<String> endPoints = new ArrayList<>();
node.listPeers().forEach(peerId -> {
endPoints.add(peerId.getEndpoint().toString());
});
node.listLearners().forEach(peerId -> {
endPoints.add(peerId.getEndpoint().toString());
});
return endPoints;
}
public static List<String> getAllEndpoints(Configuration conf) {
List<String> endPoints = new ArrayList<>();
conf.listPeers().forEach(peerId -> {
endPoints.add(peerId.getEndpoint().toString());
});
conf.listLearners().forEach(peerId -> {
endPoints.add(peerId.getEndpoint().toString());
});
return endPoints;
}
public static List<String> getPeerEndpoints(Node node) {
List<String> endPoints = new ArrayList<>();
node.listPeers().forEach(peerId -> {
endPoints.add(peerId.getEndpoint().toString());
});
return endPoints;
}
public static List<String> getPeerEndpoints(Configuration conf) {
List<String> endPoints = new ArrayList<>();
conf.listPeers().forEach(peerId -> {
endPoints.add(peerId.getEndpoint().toString());
});
return endPoints;
}
public static List<String> getLearnerEndpoints(Node node) {
List<String> endPoints = new ArrayList<>();
node.listLearners().forEach(peerId -> {
endPoints.add(peerId.getEndpoint().toString());
});
return endPoints;
}
public static List<String> getLearnerEndpoints(Configuration conf) {
List<String> endPoints = new ArrayList<>();
conf.listLearners().forEach(peerId -> {
endPoints.add(peerId.getEndpoint().toString());
});
return endPoints;
}
public static boolean configurationEquals(Configuration oldConf, Configuration newConf) {
return ListUtils.isEqualList(oldConf.listPeers(), newConf.listPeers()) &&
ListUtils.isEqualList(oldConf.listLearners(), newConf.listLearners());
}
}

225
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/snapshot/HgSnapshotHandler.java Normal file
View File

@ -0,0 +1,225 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.snapshot;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.zip.Checksum;
import org.apache.commons.io.FileUtils;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.PartitionEngine;
import org.apache.hugegraph.store.business.BusinessHandler;
import org.apache.hugegraph.store.meta.Partition;
import org.apache.hugegraph.store.util.HgStoreException;
import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
import com.alipay.sofa.jraft.storage.snapshot.Snapshot;
import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
import com.alipay.sofa.jraft.util.CRC64;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class HgSnapshotHandler {
private static final String SHOULD_NOT_LOAD = "should_not_load";
private static final String SNAPSHOT_DATA_PATH = "data";
private final PartitionEngine partitionEngine;
private final BusinessHandler businessHandler;
public HgSnapshotHandler(PartitionEngine partitionEngine) {
this.partitionEngine = partitionEngine;
this.businessHandler = partitionEngine.getStoreEngine().getBusinessHandler();
}
public static String trimStartPath(String str, String prefix) {
if (!prefix.endsWith(File.separator)) {
prefix = prefix + File.separator;
}
if (str.startsWith(prefix)) {
return (str.substring(prefix.length()));
}
return str;
}
public static void findFileList(File dir, File rootDir, List<String> files) {
if (!dir.exists() || !dir.isDirectory()) {
return;
}
File[] fs = dir.listFiles();
if (fs != null) {
for (File f : fs) {
if (f.isFile()) {
files.add(trimStartPath(dir.getPath(), rootDir.getPath()) + File.separator +
f.getName());
} else {
findFileList(f, rootDir, files);
}
}
}
}
public Map<String, Partition> getPartitions() {
return partitionEngine.getPartitions();
}
/**
* create rocksdb checkpoint
*/
public void onSnapshotSave(final SnapshotWriter writer) throws HgStoreException {
final String snapshotDir = writer.getPath();
if (partitionEngine != null) {
// rocks db snapshot
final String graphSnapshotDir = snapshotDir + File.separator + SNAPSHOT_DATA_PATH;
businessHandler.saveSnapshot(graphSnapshotDir, "", partitionEngine.getGroupId());
List<String> files = new ArrayList<>();
File dir = new File(graphSnapshotDir);
File rootDirFile = new File(writer.getPath());
// add all files in data dir
findFileList(dir, rootDirFile, files);
// load snapshot by learner ??
for (String file : files) {
String checksum = calculateChecksum(writer.getPath() + File.separator + file);
if (checksum.length() != 0) {
LocalFileMetaOutter.LocalFileMeta meta =
LocalFileMetaOutter.LocalFileMeta.newBuilder()
.setChecksum(checksum)
.build();
writer.addFile(file, meta);
} else {
writer.addFile(file);
}
}
// should_not_load wound not sync to learner
markShouldNotLoad(writer, true);
}
}
private String calculateChecksum(String path) {
// only calculate .sst and .log(wal file) file
final String emptyString = "";
if (path.endsWith(".sst") || path.endsWith(".log")) {
final int maxFullCheckLength = 8192;
final int checkLength = 4096;
try {
File file = new File(path);
long length = file.length();
Checksum checksum = new CRC64();
try (final RandomAccessFile raf = new RandomAccessFile(file, "r")) {
byte[] buf = new byte[checkLength];
if (length <= maxFullCheckLength) {
int totalReadLen = 0;
while (totalReadLen < length) {
int readLen = raf.read(buf);
checksum.update(buf, 0, readLen);
totalReadLen += readLen;
}
} else {
// head
int readLen = raf.read(buf);
checksum.update(buf, 0, readLen);
// tail
raf.seek(length - checkLength);
readLen = raf.read(buf);
checksum.update(buf, 0, readLen);
}
}
// final checksum = crc checksum + file length
return Long.toHexString(checksum.getValue()) + "_" + Long.toHexString(length);
} catch (IOException e) {
log.error("Failed to calculateChecksum for file {}. {}", path, e);
return emptyString;
}
} else {
return emptyString;
}
}
public void onSnapshotLoad(final SnapshotReader reader, long committedIndex) throws
HgStoreException {
final String snapshotDir = reader.getPath();
// 本地保存的快照没必要加载
if (shouldNotLoad(reader)) {
log.info("skip to load snapshot because of should_not_load flag");
return;
}
// 直接使用 snapshot
final String graphSnapshotDir = snapshotDir + File.separator + SNAPSHOT_DATA_PATH;
log.info("Raft {} begin loadSnapshot, {}", partitionEngine.getGroupId(), graphSnapshotDir);
businessHandler.loadSnapshot(graphSnapshotDir, "", partitionEngine.getGroupId(),
committedIndex);
log.info("Raft {} end loadSnapshot.", partitionEngine.getGroupId());
for (Metapb.Partition snapPartition : partitionEngine.loadPartitionsFromLocalDb()) {
log.info("onSnapshotLoad loaded partition from local db. Partition: {}", snapPartition);
partitionEngine.loadPartitionFromSnapshot(new Partition(snapPartition));
Partition partition = partitionEngine.getPartition(snapPartition.getGraphName());
if (partition == null) {
log.warn("skip to load snapshot for {}-{}, it is not belong to this node",
snapPartition.getGraphName(), snapPartition.getId());
continue;
}
var taskManager = partitionEngine.getTaskManager();
// async tasks
for (var task : taskManager.scanAsyncTasks(partitionEngine.getGroupId(),
snapPartition.getGraphName())) {
task.handleTask();
}
}
// mark snapshot has been loaded
markShouldNotLoad(reader, false);
}
private boolean shouldNotLoad(final Snapshot snapshot) {
String shouldNotLoadPath = getShouldNotLoadPath(snapshot);
return new File(shouldNotLoadPath).exists();
}
private void markShouldNotLoad(final Snapshot snapshot, boolean saveSnapshot) {
String shouldNotLoadPath = getShouldNotLoadPath(snapshot);
try {
FileUtils.writeStringToFile(new File(shouldNotLoadPath),
saveSnapshot ? "saved snapshot" : "loaded snapshot",
Charset.defaultCharset());
} catch (IOException e) {
log.error("Failed to create snapshot should not load flag file {}. {}",
shouldNotLoadPath, e);
}
}
private String getShouldNotLoadPath(final Snapshot snapshot) {
return snapshot.getPath() + File.separator + SHOULD_NOT_LOAD;
}
}

61
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/Asserts.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
public final class Asserts {
public static void isTrue(boolean expression, String message) {
if (message == null) {
throw new IllegalArgumentException("message is null");
}
if (!expression) {
throw new IllegalArgumentException(message);
}
}
public static void isFalse(boolean expression, String message) {
isTrue(!expression, message);
}
public static boolean isInvalid(String... strs) {
if (strs == null || strs.length == 0) {
return true;
}
for (String item : strs) {
if (item == null || "".equals(item.trim())) {
return true;
}
}
return false;
}
public static <T> T isNonNull(T obj) {
if (obj == null) {
throw new NullPointerException();
}
return obj;
}
public static <T> T isNonNull(T obj, String message) {
if (obj == null) {
throw new NullPointerException(message);
}
return obj;
}
}

150
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/CopyOnWriteCache.java Normal file
View File

@ -0,0 +1,150 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
//TODO: refer license later, 74% match, maybe refer to kafka (0.8.1)
public class CopyOnWriteCache<K, V> implements ConcurrentMap<K, V> {
ScheduledExecutorService scheduledExecutor;
private volatile Map<K, V> map;
public CopyOnWriteCache(long effectiveTime) {
this.map = Collections.emptyMap();
scheduledExecutor = Executors.newScheduledThreadPool(1);
scheduledExecutor.scheduleWithFixedDelay(() -> {
this.clear();
}, effectiveTime, effectiveTime, TimeUnit.MILLISECONDS);
}
@Override
public boolean containsKey(Object k) {
return map.containsKey(k);
}
@Override
public boolean containsValue(Object v) {
return map.containsValue(v);
}
@Override
public Set<Entry<K, V>> entrySet() {
return map.entrySet();
}
@Override
public V get(Object k) {
return map.get(k);
}
@Override
public boolean isEmpty() {
return map.isEmpty();
}
@Override
public Set<K> keySet() {
return map.keySet();
}
@Override
public int size() {
return map.size();
}
@Override
public Collection<V> values() {
return map.values();
}
@Override
public synchronized void clear() {
this.map = Collections.emptyMap();
}
@Override
public synchronized V put(K k, V v) {
Map<K, V> copy = new HashMap<>(this.map);
V prev = copy.put(k, v);
this.map = Collections.unmodifiableMap(copy);
return prev;
}
@Override
public synchronized void putAll(Map<? extends K, ? extends V> entries) {
Map<K, V> copy = new HashMap<>(this.map);
copy.putAll(entries);
this.map = Collections.unmodifiableMap(copy);
}
@Override
public synchronized V remove(Object key) {
Map<K, V> copy = new HashMap<>(this.map);
V prev = copy.remove(key);
this.map = Collections.unmodifiableMap(copy);
return prev;
}
@Override
public synchronized V putIfAbsent(K k, V v) {
if (!containsKey(k)) {
return put(k, v);
} else {
return get(k);
}
}
@Override
public synchronized boolean remove(Object k, Object v) {
if (containsKey(k) && get(k).equals(v)) {
remove(k);
return true;
} else {
return false;
}
}
@Override
public synchronized boolean replace(K k, V original, V replacement) {
if (containsKey(k) && get(k).equals(original)) {
put(k, replacement);
return true;
} else {
return false;
}
}
@Override
public synchronized V replace(K k, V v) {
if (containsKey(k)) {
return put(k, v);
} else {
return null;
}
}
}

94
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/FutureClosure.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alipay.sofa.jraft.Closure;
import com.alipay.sofa.jraft.Status;
import com.alipay.sofa.jraft.error.RaftError;
public class FutureClosure implements Closure, Future<Status> {
private static final Logger LOG = LoggerFactory.getLogger(FutureClosure.class);
private final CountDownLatch latch;
private Status status;
public FutureClosure() {
this(1);
}
public FutureClosure(int count) {
this.latch = new CountDownLatch(count);
}
public static void waitLatch(CountDownLatch latch) {
try {
latch.await();
} catch (InterruptedException e) {
LOG.error("{}", e);
}
}
@Override
public void run(Status status) {
this.status = status;
latch.countDown();
}
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
return true;
}
@Override
public boolean isCancelled() {
return false;
}
@Override
public boolean isDone() {
return false;
}
@Override
public Status get() {
try {
latch.await();
} catch (InterruptedException e) {
status = new Status(RaftError.EINTR, e.getMessage());
}
return status;
}
@Override
public Status get(long timeout, TimeUnit unit) {
try {
latch.await(timeout, unit);
} catch (InterruptedException e) {
status = new Status(RaftError.EINTR, e.getMessage());
}
return status;
}
}

69
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgRaftError.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import java.util.HashMap;
import java.util.Map;
import com.alipay.sofa.jraft.Status;
public enum HgRaftError {
UNKNOWN(-1, "unknown"),
OK(0, "OK"),
NOT_LEADER(20000, "This partition is not leader"),
WAIT_LEADER_TIMEOUT(20001, "Waiting for leader timeout"),
NOT_LOCAL(20002, "This partition is not local"),
CLUSTER_NOT_READY(20003, "The cluster is not ready, please check active stores number!"),
TASK_CONTINUE(21000, "Task is continue"),
TASK_ERROR(21001, "Task is error, need to retry"),
END(30000, "HgStore error is end");
private static final Map<Integer, HgRaftError> RAFT_ERROR_MAP = new HashMap<>();
static {
for (final HgRaftError error : HgRaftError.values()) {
RAFT_ERROR_MAP.put(error.getNumber(), error);
}
}
private final int value;
private final String msg;
HgRaftError(final int value, final String msg) {
this.value = value;
this.msg = msg;
}
public static HgRaftError forNumber(final int value) {
return RAFT_ERROR_MAP.getOrDefault(value, UNKNOWN);
}
public final int getNumber() {
return this.value;
}
public final String getMsg() {
return this.msg;
}
public Status toStatus() {
return new Status(value, msg);
}
}

37
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgStoreConst.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import java.util.Collections;
import java.util.List;
/**
* created on 2021/10/22
*/
public final class HgStoreConst {
public final static byte[] EMPTY_BYTES = new byte[0];
public static final List EMPTY_LIST = Collections.EMPTY_LIST;
public final static int SCAN_ALL_PARTITIONS_ID = -1; // means scan all partitions.
private HgStoreConst() {
}
}

80
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgStoreException.java Normal file
View File

@ -0,0 +1,80 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
public class HgStoreException extends RuntimeException {
public final static int EC_NOEXCEPT = 0;
public final static int EC_FAIL = 1000;
//存储的数据格式不支持
public final static int EC_DATAFMT_NOT_SUPPORTED = 1001;
public final static int EC_RKDB_CREATE_FAIL = 1201;
public final static int EC_RKDB_DOPUT_FAIL = 1202;
public final static int EC_RKDB_DODEL_FAIL = 1203;
public final static int EC_RDKDB_DOSINGLEDEL_FAIL = 1204;
public final static int EC_RKDB_DODELPREFIX_FAIL = 1205;
public final static int EC_RKDB_DODELRANGE_FAIL = 1206;
public final static int EC_RKDB_DOMERGE_FAIL = 1207;
public final static int EC_RKDB_DOGET_FAIL = 1208;
public final static int EC_RKDB_PD_FAIL = 1209;
public final static int EC_RKDB_TRUNCATE_FAIL = 1212;
public final static int EC_RKDB_EXPORT_SNAPSHOT_FAIL = 1214;
public final static int EC_RKDB_IMPORT_SNAPSHOT_FAIL = 1215;
public final static int EC_RKDB_TRANSFER_SNAPSHOT_FAIL = 1216;
public final static int EC_METRIC_FAIL = 1401;
private static final long serialVersionUID = 5193624480997934335L;
private final int code;
public HgStoreException() {
super();
this.code = EC_NOEXCEPT;
}
public HgStoreException(String message) {
super(message);
this.code = EC_FAIL;
}
public HgStoreException(int exceptCode, String message) {
super(message);
this.code = exceptCode;
}
public HgStoreException(int exceptCode, Throwable cause) {
super(codeToMsg(exceptCode), cause);
this.code = exceptCode;
}
public HgStoreException(int exceptCode, String message, Object... args) {
super(String.format(message, args));
this.code = exceptCode;
}
public HgStoreException(String message, Throwable cause) {
super(message, cause);
this.code = EC_FAIL;
}
public static String codeToMsg(int code) {
return "errorCode = " + code;
}
public int getCode() {
return this.code;
}
}

101
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/IpUtil.java Normal file
View File

@ -0,0 +1,101 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import java.net.Inet4Address;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class IpUtil {
/**
* 获取所有的ipv4 地址
*
* @return all ipv4 addr
* @throws SocketException io error or no network interface
*/
private static List<String> getIpAddress() throws SocketException {
List<String> list = new LinkedList<>();
Enumeration enumeration = NetworkInterface.getNetworkInterfaces();
while (enumeration.hasMoreElements()) {
NetworkInterface network = (NetworkInterface) enumeration.nextElement();
Enumeration addresses = network.getInetAddresses();
while (addresses.hasMoreElements()) {
InetAddress address = (InetAddress) addresses.nextElement();
if (address != null && (address instanceof Inet4Address)) {
list.add(address.getHostAddress());
}
}
}
return list;
}
/**
* 根据 option中的raft addr，根据本机的ip获取最相近的一个
*
* @param raftAddress raft addr
* @return raft addr that have the nearest distance with given param
*/
public static String getNearestAddress(String raftAddress) {
try {
List<String> ipv4s = getIpAddress();
String[] tmp = raftAddress.split(":");
if (ipv4s.size() == 0) {
throw new Exception("no available ipv4");
}
if (ipv4s.size() == 1) {
return ipv4s.get(0) + ":" + tmp[1];
}
var raftSeg = Arrays.stream(tmp[0].split("\\."))
.map(s -> Integer.parseInt(s))
.collect(Collectors.toList());
ipv4s.sort(Comparator.comparingLong(ip -> {
String[] ipSegments = ip.split("\\.");
long base = 256 * 256 * 256;
int i = 0;
long sum = 0;
for (String seg : ipSegments) {
sum += base * (Math.abs(raftSeg.get(i) - Integer.parseInt(seg)));
base = base / 256;
i += 1;
}
return sum;
}));
return ipv4s.get(0) + ":" + tmp[1];
} catch (SocketException e) {
log.error("getIpAddress, get ip failed, {}", e.getMessage());
} catch (Exception e) {
log.error("getRaftAddress, got exception, {}", e.getMessage());
}
return raftAddress;
}
}

33
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/Lifecycle.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
public interface Lifecycle<T> {
/**
* Initialize the service.
*
* @return true when successes.
*/
boolean init(final T opts);
/**
* Dispose the resources for service.
*/
void shutdown();
}

95
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/ManualResetEvent.java Normal file
View File

@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
/**
* Updated by RoshanF on 12/28/2015.
*/
public class ManualResetEvent {
private static final Object mutex = new Object();
private volatile CountDownLatch event;
/**
* Initializes a new instance of the System.Threading.ManualResetEvent class
* with a Boolean value indicating whether to set the initial state to signaled.
*
* @param signalled true to set the initial state to signaled; false to set the initial state
* to nonsignaled.
*/
public ManualResetEvent(boolean signalled) {
if (signalled) {
event = new CountDownLatch(0);
} else {
event = new CountDownLatch(1);
}
}
/**
* Sets the state of the event to signaled, allowing one or more waiting threads to proceed.
*/
public void set() {
event.countDown();
}
/**
* Sets the state of the event to nonsignaled, causing threads to block.
*/
public void reset() {
synchronized (mutex) {
if (event.getCount() == 0) {
event = new CountDownLatch(1);
}
}
}
/**
* Blocks the current thread until the current wait handle receives a signal.
*
* @throws InterruptedException
*/
public void waitOne() throws InterruptedException {
event.await();
}
/**
* Blocks the current thread until the current wait handle receives a signal.
*
* @param timeout the maximum time to wait
* @param unit the time unit of the {@code timeout} argument
* @return {@code true} if the count reached zero and {@code false}
* if the waiting time elapsed before the count reached zero
* @throws InterruptedException if the current thread is interrupted
* while waiting
*/
public boolean waitOne(int timeout, TimeUnit unit) throws InterruptedException {
return event.await(timeout, unit);
}
/**
* Check if the handle was signalled
*
* @return Boolean state
*/
public boolean isSignalled() {
return event.getCount() == 0;
}
}

73
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/PartitionMetaStoreWrapper.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import java.util.List;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.meta.base.MetaStoreBase;
public class PartitionMetaStoreWrapper {
private static final InnerMetaStore store = new InnerMetaStore();
public void put(int partitionId, byte[] key, byte[] value) {
store.setPartitionId(partitionId);
store.put(key, value);
}
public <T> T get(int partitionId, byte[] key, com.google.protobuf.Parser<T> parser) {
store.setPartitionId(partitionId);
return store.get(parser, key);
}
public byte[] get(int partitionId, byte[] key) {
store.setPartitionId(partitionId);
return store.get(key);
}
public void delete(int partitionId, byte[] key) {
store.setPartitionId(partitionId);
store.delete(key);
}
public <T> List<T> scan(int partitionId, com.google.protobuf.Parser<T> parser, byte[] prefix) {
store.setPartitionId(partitionId);
return store.scan(parser, prefix);
}
private static class InnerMetaStore extends MetaStoreBase {
private int partitionId;
private void setPartitionId(int partitionId) {
this.partitionId = partitionId;
}
@Override
protected RocksDBSession getRocksDBSession() {
return HgStoreEngine.getInstance().getBusinessHandler().getSession(this.partitionId);
}
@Override
protected String getCFName() {
return "default";
}
}
}

327
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/UnsafeUtf8Util.java Normal file
View File

@ -0,0 +1,327 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import static java.lang.Character.MAX_SURROGATE;
import static java.lang.Character.MIN_HIGH_SURROGATE;
import static java.lang.Character.MIN_LOW_SURROGATE;
import static java.lang.Character.MIN_SUPPLEMENTARY_CODE_POINT;
import static java.lang.Character.MIN_SURROGATE;
import static java.lang.Character.isSurrogatePair;
import static java.lang.Character.toCodePoint;
import java.util.Arrays;
public class UnsafeUtf8Util {
public static int encodedLength(CharSequence sequence) {
// Warning to maintainers: this implementation is highly optimized.
int utf16Length = sequence.length();
int utf8Length = utf16Length;
int i = 0;
// This loop optimizes for pure ASCII.
while (i < utf16Length && sequence.charAt(i) < 0x80) {
i++;
}
// This loop optimizes for chars less than 0x800.
for (; i < utf16Length; i++) {
char c = sequence.charAt(i);
if (c < 0x800) {
utf8Length += ((0x7f - c) >>> 31); // branch free!
} else {
utf8Length += encodedLengthGeneral(sequence, i);
break;
}
}
if (utf8Length < utf16Length) {
// Necessary and sufficient condition for overflow because of maximum 3x expansion
throw new IllegalArgumentException(
"UTF-8 length does not fit in int: " + (utf8Length + (1L << 32)));
}
return utf8Length;
}
private static int encodedLengthGeneral(CharSequence sequence, int start) {
int utf16Length = sequence.length();
int utf8Length = 0;
for (int i = start; i < utf16Length; i++) {
char c = sequence.charAt(i);
if (c < 0x800) {
utf8Length += (0x7f - c) >>> 31; // branch free!
} else {
utf8Length += 2;
// jdk7+: if (Character.isSurrogate(c)) {
if (Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) {
// Check that we have a well-formed surrogate pair.
int cp = Character.codePointAt(sequence, i);
if (cp < MIN_SUPPLEMENTARY_CODE_POINT) {
throw new IllegalArgumentException(
"Unpaired surrogate at index " + i + " of " + utf16Length);
}
i++;
}
}
}
return utf8Length;
}
public static int encodeUtf8(CharSequence in, byte[] out, int offset, int length) {
long outIx = offset;
final long outLimit = outIx + length;
final int inLimit = in.length();
if (inLimit > length || out.length - length < offset) {
// Not even enough room for an ASCII-encoded string.
throw new ArrayIndexOutOfBoundsException(
"Failed writing " + in.charAt(inLimit - 1) + " at index "
+ (offset + length));
}
// Designed to take advantage of
// https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination
int inIx = 0;
for (char c; inIx < inLimit && (c = in.charAt(inIx)) < 0x80; ++inIx) {
UnsafeUtil.putByte(out, outIx++, (byte) c);
}
if (inIx == inLimit) {
// We're done, it was ASCII encoded.
return (int) outIx;
}
for (char c; inIx < inLimit; ++inIx) {
c = in.charAt(inIx);
if (c < 0x80 && outIx < outLimit) {
UnsafeUtil.putByte(out, outIx++, (byte) c);
} else if (c < 0x800 && outIx <= outLimit - 2L) { // 11 bits, two UTF-8 bytes
UnsafeUtil.putByte(out, outIx++, (byte) ((0xF << 6) | (c >>> 6)));
UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & c)));
} else if ((c < MIN_SURROGATE || MAX_SURROGATE < c) && outIx <= outLimit - 3L) {
// Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes
UnsafeUtil.putByte(out, outIx++, (byte) ((0xF << 5) | (c >>> 12)));
UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & (c >>> 6))));
UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & c)));
} else if (outIx <= outLimit - 4L) {
// Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four
// UTF-8
// bytes
final char low;
if (inIx + 1 == inLimit || !isSurrogatePair(c, (low = in.charAt(++inIx)))) {
throw new IllegalArgumentException(
"Unpaired surrogate at index " + (inIx - 1) + " of " + inLimit);
}
int codePoint = toCodePoint(c, low);
UnsafeUtil.putByte(out, outIx++, (byte) ((0xF << 4) | (codePoint >>> 18)));
UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 12))));
UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 6))));
UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & codePoint)));
} else {
if ((MIN_SURROGATE <= c && c <= MAX_SURROGATE)
&& (inIx + 1 == inLimit || !isSurrogatePair(c, in.charAt(inIx + 1)))) {
// We are surrogates and we're not a surrogate pair.
throw new IllegalArgumentException(
"Unpaired surrogate at index " + inIx + " of " + inLimit);
}
// Not enough space in the output buffer.
throw new ArrayIndexOutOfBoundsException(
"Failed writing " + c + " at index " + outIx);
}
}
// All bytes have been encoded.
return (int) outIx;
}
public static String decodeUtf8(byte[] bytes, int index, int size) {
if ((index | size | bytes.length - index - size) < 0) {
throw new ArrayIndexOutOfBoundsException(
"buffer length=" + bytes.length + ", index=" + index + ", size="
+ size);
}
int offset = index;
final int limit = offset + size;
// The longest possible resulting String is the same as the number of input bytes, when
// it is
// all ASCII. For other cases, this over-allocates and we will truncate in the end.
char[] resultArr = new char[size];
int resultPos = 0;
// Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this).
// This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII).
while (offset < limit) {
byte b = UnsafeUtil.getByte(bytes, offset);
if (!DecodeUtil.isOneByte(b)) {
break;
}
offset++;
DecodeUtil.handleOneByte(b, resultArr, resultPos++);
}
while (offset < limit) {
byte byte1 = UnsafeUtil.getByte(bytes, offset++);
if (DecodeUtil.isOneByte(byte1)) {
DecodeUtil.handleOneByte(byte1, resultArr, resultPos++);
// It's common for there to be multiple ASCII characters in a run mixed in, so
// add an
// extra optimized loop to take care of these runs.
while (offset < limit) {
byte b = UnsafeUtil.getByte(bytes, offset);
if (!DecodeUtil.isOneByte(b)) {
break;
}
offset++;
DecodeUtil.handleOneByte(b, resultArr, resultPos++);
}
} else if (DecodeUtil.isTwoBytes(byte1)) {
if (offset >= limit) {
throw invalidUtf8();
}
DecodeUtil.handleTwoBytes(byte1, /* byte2 */UnsafeUtil.getByte(bytes, offset++),
resultArr,
resultPos++);
} else if (DecodeUtil.isThreeBytes(byte1)) {
if (offset >= limit - 1) {
throw invalidUtf8();
}
DecodeUtil.handleThreeBytes(byte1,
/* byte2 */UnsafeUtil.getByte(bytes, offset++),
/* byte3 */UnsafeUtil.getByte(bytes, offset++), resultArr, resultPos++);
} else {
if (offset >= limit - 2) {
throw invalidUtf8();
}
DecodeUtil.handleFourBytes(byte1,
/* byte2 */UnsafeUtil.getByte(bytes, offset++),
/* byte3 */UnsafeUtil.getByte(bytes, offset++),
/* byte4 */UnsafeUtil.getByte(bytes, offset++), resultArr, resultPos++);
// 4-byte case requires two chars.
resultPos++;
}
}
if (resultPos < resultArr.length) {
resultArr = Arrays.copyOf(resultArr, resultPos);
}
return UnsafeUtil.moveToString(resultArr);
}
static IllegalStateException invalidUtf8() {
return new IllegalStateException("Message had invalid UTF-8.");
}
private static class DecodeUtil {
/**
* Returns whether this is a single-byte codepoint (i.e., ASCII) with the form '0XXXXXXX'.
*/
private static boolean isOneByte(byte b) {
return b >= 0;
}
/**
* Returns whether this is a two-byte codepoint with the form '10XXXXXX'.
*/
private static boolean isTwoBytes(byte b) {
return b < (byte) 0xE0;
}
/**
* Returns whether this is a three-byte codepoint with the form '110XXXXX'.
*/
private static boolean isThreeBytes(byte b) {
return b < (byte) 0xF0;
}
private static void handleOneByte(byte byte1, char[] resultArr, int resultPos) {
resultArr[resultPos] = (char) byte1;
}
private static void handleTwoBytes(byte byte1, byte byte2, char[] resultArr,
int resultPos) {
// Simultaneously checks for illegal trailing-byte in leading position (<=
// '11000000') and
// overlong 2-byte, '11000001'.
if (byte1 < (byte) 0xC2 || isNotTrailingByte(byte2)) {
throw invalidUtf8();
}
resultArr[resultPos] = (char) (((byte1 & 0x1F) << 6) | trailingByteValue(byte2));
}
private static void handleThreeBytes(byte byte1, byte byte2, byte byte3, char[] resultArr,
int resultPos) {
if (isNotTrailingByte(byte2)
// overlong? 5 most significant bits must not all be zero
|| (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0)
// check for illegal surrogate codepoints
|| (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) || isNotTrailingByte(byte3)) {
throw invalidUtf8();
}
resultArr[resultPos] =
(char) (((byte1 & 0x0F) << 12) | (trailingByteValue(byte2) << 6) |
trailingByteValue(byte3));
}
private static void handleFourBytes(byte byte1, byte byte2, byte byte3, byte byte4,
char[] resultArr,
int resultPos) {
if (isNotTrailingByte(byte2)
// Check that 1 <= plane <= 16. Tricky optimized form of:
// valid 4-byte leading byte?
// if (byte1 > (byte) 0xF4 ||
// overlong? 4 most significant bits must not all be zero
// byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 ||
// codepoint larger than the highest code point (U+10FFFF)?
// byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F)
|| (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 || isNotTrailingByte(byte3)
|| isNotTrailingByte(byte4)) {
throw invalidUtf8();
}
int codePoint = ((byte1 & 0x07) << 18) | (trailingByteValue(byte2) << 12) |
(trailingByteValue(byte3) << 6)
| trailingByteValue(byte4);
resultArr[resultPos] = DecodeUtil.highSurrogate(codePoint);
resultArr[resultPos + 1] = DecodeUtil.lowSurrogate(codePoint);
}
/**
* Returns whether the byte is not a valid continuation of the form '10XXXXXX'.
*/
private static boolean isNotTrailingByte(byte b) {
return b > (byte) 0xBF;
}
/**
* Returns the actual value of the trailing byte (removes the prefix '10') for composition.
*/
private static int trailingByteValue(byte b) {
return b & 0x3F;
}
private static char highSurrogate(int codePoint) {
return (char) ((MIN_HIGH_SURROGATE - (MIN_SUPPLEMENTARY_CODE_POINT >>> 10)) +
(codePoint >>> 10));
}
private static char lowSurrogate(int codePoint) {
return (char) (MIN_LOW_SURROGATE + (codePoint & 0x3ff));
}
}
}

331
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/UnsafeUtil.java Normal file
View File

@ -0,0 +1,331 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import java.lang.reflect.Field;
import lombok.extern.slf4j.Slf4j;
/**
* TODO: refer license later, 76% match, maybe refer to jraft-core (1.2.6)
*/
@Slf4j
public class UnsafeUtil {
private static final Object UNSAFE = getUnsafe0();
private static final UnsafeAccessor UNSAFE_ACCESSOR = getUnsafeAccessor0();
private static final long BYTE_ARRAY_BASE_OFFSET = arrayBaseOffset(byte[].class);
private static final long STRING_VALUE_OFFSET = objectFieldOffset(stringValueField());
public static boolean hasUnsafe() {
return UNSAFE != null;
}
public static Object getUnsafe0() {
Object unsafe;
try {
final Class<?> unsafeClass = Class.forName("sun.misc.Unsafe");
final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe");
unsafeField.setAccessible(true);
unsafe = unsafeField.get(null);
} catch (final Throwable t) {
if (log.isWarnEnabled()) {
log.warn("sun.misc.Unsafe.theUnsafe: unavailable.", t);
}
unsafe = null;
}
return unsafe;
}
public static void putByte(final byte[] target, final long index, final byte value) {
UNSAFE_ACCESSOR.putByte(target, BYTE_ARRAY_BASE_OFFSET + index, value);
}
public static byte getByte(final byte[] target, final long index) {
return UNSAFE_ACCESSOR.getByte(target, BYTE_ARRAY_BASE_OFFSET + index);
}
public static int arrayBaseOffset(final Class<?> clazz) {
return hasUnsafe() ? UNSAFE_ACCESSOR.arrayBaseOffset(clazz) : -1;
}
public static String moveToString(final char[] chars) {
if (STRING_VALUE_OFFSET == -1) {
// In the off-chance that this JDK does not implement String as we'd expect, just do
// a copy.
return new String(chars);
}
final String str;
try {
str = (String) UNSAFE_ACCESSOR.allocateInstance(String.class);
} catch (final InstantiationException e) {
// This should never happen, but return a copy as a fallback just in case.
return new String(chars);
}
UNSAFE_ACCESSOR.putObject(str, STRING_VALUE_OFFSET, chars);
return str;
}
public static long objectFieldOffset(final Field field) {
return field == null || hasUnsafe() ? UNSAFE_ACCESSOR.objectFieldOffset(field) : -1;
}
private static Field stringValueField() {
return field(String.class, "value", char[].class);
}
private static Field field(final Class<?> clazz, final String fieldName,
final Class<?> expectedType) {
Field field;
try {
field = clazz.getDeclaredField(fieldName);
field.setAccessible(true);
if (!field.getType().equals(expectedType)) {
return null;
}
} catch (final Throwable t) {
// Failed to access the fields.
field = null;
}
return field;
}
private static UnsafeAccessor getUnsafeAccessor0() {
return hasUnsafe() ? new UnsafeAccessor(UNSAFE) : null;
}
public static class UnsafeAccessor {
private final sun.misc.Unsafe unsafe;
public UnsafeAccessor(Object unsafe) {
this.unsafe = (sun.misc.Unsafe) unsafe;
}
/**
* Returns the {@link sun.misc.Unsafe}'s instance.
*/
public sun.misc.Unsafe getUnsafe() {
return unsafe;
}
public byte getByte(final Object target, final long offset) {
return this.unsafe.getByte(target, offset);
}
public void putByte(final Object target, final long offset, final byte value) {
this.unsafe.putByte(target, offset, value);
}
public short getShort(final Object target, final long offset) {
return this.unsafe.getShort(target, offset);
}
public void putShort(final Object target, final long offset, final short value) {
this.unsafe.putShort(target, offset, value);
}
public int getInt(final Object target, final long offset) {
return this.unsafe.getInt(target, offset);
}
public void putInt(final Object target, final long offset, final int value) {
this.unsafe.putInt(target, offset, value);
}
public long getLong(final Object target, final long offset) {
return this.unsafe.getLong(target, offset);
}
public void putLong(final Object target, final long offset, final long value) {
this.unsafe.putLong(target, offset, value);
}
public boolean getBoolean(final Object target, final long offset) {
return this.unsafe.getBoolean(target, offset);
}
public void putBoolean(final Object target, final long offset, final boolean value) {
this.unsafe.putBoolean(target, offset, value);
}
public float getFloat(final Object target, final long offset) {
return this.unsafe.getFloat(target, offset);
}
public void putFloat(final Object target, final long offset, final float value) {
this.unsafe.putFloat(target, offset, value);
}
public double getDouble(final Object target, final long offset) {
return this.unsafe.getDouble(target, offset);
}
public void putDouble(final Object target, final long offset, final double value) {
this.unsafe.putDouble(target, offset, value);
}
public Object getObject(final Object target, final long offset) {
return this.unsafe.getObject(target, offset);
}
public void putObject(final Object target, final long offset, final Object value) {
this.unsafe.putObject(target, offset, value);
}
public byte getByte(final long address) {
return this.unsafe.getByte(address);
}
public void putByte(final long address, final byte value) {
this.unsafe.putByte(address, value);
}
public short getShort(final long address) {
return this.unsafe.getShort(address);
}
public void putShort(final long address, final short value) {
this.unsafe.putShort(address, value);
}
public int getInt(final long address) {
return this.unsafe.getInt(address);
}
public void putInt(final long address, final int value) {
this.unsafe.putInt(address, value);
}
public long getLong(final long address) {
return this.unsafe.getLong(address);
}
public void putLong(final long address, final long value) {
this.unsafe.putLong(address, value);
}
public void copyMemory(final Object srcBase, final long srcOffset, final Object dstBase,
final long dstOffset,
final long bytes) {
this.unsafe.copyMemory(srcBase, srcOffset, dstBase, dstOffset, bytes);
}
public void copyMemory(final long srcAddress, final long dstAddress, final long bytes) {
this.unsafe.copyMemory(srcAddress, dstAddress, bytes);
}
public byte getByteVolatile(final Object target, final long offset) {
return this.unsafe.getByteVolatile(target, offset);
}
public void putByteVolatile(final Object target, final long offset, final byte value) {
this.unsafe.putByteVolatile(target, offset, value);
}
public short getShortVolatile(final Object target, final long offset) {
return this.unsafe.getShortVolatile(target, offset);
}
public void putShortVolatile(final Object target, final long offset, final short value) {
this.unsafe.putShortVolatile(target, offset, value);
}
public int getIntVolatile(final Object target, final long offset) {
return this.unsafe.getIntVolatile(target, offset);
}
public void putIntVolatile(final Object target, final long offset, final int value) {
this.unsafe.putIntVolatile(target, offset, value);
}
public long getLongVolatile(final Object target, final long offset) {
return this.unsafe.getLongVolatile(target, offset);
}
public void putLongVolatile(final Object target, final long offset, final long value) {
this.unsafe.putLongVolatile(target, offset, value);
}
public boolean getBooleanVolatile(final Object target, final long offset) {
return this.unsafe.getBooleanVolatile(target, offset);
}
public void putBooleanVolatile(final Object target, final long offset,
final boolean value) {
this.unsafe.putBooleanVolatile(target, offset, value);
}
public float getFloatVolatile(final Object target, final long offset) {
return this.unsafe.getFloatVolatile(target, offset);
}
public void putFloatVolatile(final Object target, final long offset, final float value) {
this.unsafe.putFloatVolatile(target, offset, value);
}
public double getDoubleVolatile(final Object target, final long offset) {
return this.unsafe.getDoubleVolatile(target, offset);
}
public void putDoubleVolatile(final Object target, final long offset, final double value) {
this.unsafe.putDoubleVolatile(target, offset, value);
}
public Object getObjectVolatile(final Object target, final long offset) {
return this.unsafe.getObjectVolatile(target, offset);
}
public void putObjectVolatile(final Object target, final long offset, final Object value) {
this.unsafe.putObjectVolatile(target, offset, value);
}
/**
* Reports the offset of the first element in the storage allocation of a
* given array class.
*/
public int arrayBaseOffset(final Class<?> clazz) {
return this.unsafe != null ? this.unsafe.arrayBaseOffset(clazz) : -1;
}
/**
* Reports the scale factor for addressing elements in the storage
* allocation of a given array class.
*/
public int arrayIndexScale(final Class<?> clazz) {
return this.unsafe != null ? this.unsafe.arrayIndexScale(clazz) : -1;
}
/**
* Returns the offset of the provided field, or {@code -1} if {@code sun.misc.Unsafe} is not
* available.
*/
public long objectFieldOffset(final Field field) {
return field == null || this.unsafe == null ? -1 : this.unsafe.objectFieldOffset(field);
}
public Object allocateInstance(final Class<?> clazz) throws InstantiationException {
return this.unsafe.allocateInstance(clazz);
}
public void throwException(final Throwable t) {
this.unsafe.throwException(t);
}
}
}

51
hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/util/Version.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hugegraph.store.util;
import java.io.InputStream;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Version {
private static String version = "";
/**
* 软件版本号
*/
public static String getVersion() {
if (version.isEmpty()) {
try (InputStream is = Version.class.getResourceAsStream("/version.txt")) {
byte[] buf = new byte[64];
int len = is.read(buf);
version = new String(buf, 0, len);
} catch (Exception e) {
log.error("Version.getVersion exception: ", e);
}
}
return version;
}
/**
* 存储格式版本号
*/
public static int getDataFmtVersion() {
return 1;
}
}

14
hugegraph-store/pom.xml
View File

@ -39,8 +39,8 @@
<module>hg-store-client</module>
<module>hg-store-test</module>
<module>hg-store-rocksdb</module>
<module>hg-store-core</module>
<!-- TODO: uncomment later-->
<!-- <module>hg-store-core</module>-->
<!-- <module>hg-store-node</module>-->
<!-- <module>hg-store-dist</module>-->
<!-- <module>hg-store-cli</module>-->
@ -56,7 +56,6 @@
<dependencyManagement>
<dependencies>
<!-- TODO: uncomment later-->
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-common</artifactId>
@ -77,11 +76,12 @@
<artifactId>hg-store-client</artifactId>
<version>${project.version}</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.apache.hugegraph</groupId>-->
<!-- <artifactId>hg-store-core</artifactId>-->
<!-- <version>${project.version}</version>-->
<!-- </dependency>-->
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-core</artifactId>
<version>${project.version}</version>
</dependency>
<!-- TODO: uncomment later-->
<!-- <dependency>-->
<!-- <groupId>org.apache.hugegraph</groupId>-->
<!-- <artifactId>hg-store-transfer</artifactId>-->