anph
/
hugegraph-sync
forked from hugegraph/hugegraph-sync
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: preliminary support for hstore backend (#2301) 

* chore: move hugegraph-hstore

* chore: simple adapt

* chore: register hstore

* build: intro grpc dep

* chore: migrate org.apache.hugegraph.meta

* chore: migrate org.apache.hugegraph.space

* chore: adapt meta schema

* chore: migrate SchemaTransactionV2

* chore: adapt SchemaTransactionV2

* refact: intro ISchemaTransaction for adaptation

* fix: init meta server

* fix: avoid typecast from UnmodifiableRandomAccessList

class java.util.Collections cannot be cast to class java.util.ArrayList

* refact: query task and server info by graph tx

* fix: register task and server table for rocksdb BE

* chore: add example for hstore BE scanning tables

* fix: register task and server table for other BEs

* build: remove system dependency to hugegraph-core in hg-store-core

* chore: remove author comments

* chore: remove AbstractDistributedLock class and lock method

* chore: replace HashMap with ImmutableMap

* chore: code review

* chore: load driver version in graph store for hstore

* fix: index serialization and deserialization logic

* fix: convertTaskOrServerToVertex for other BEs

* chore: init meta server in ctor of StandardHugeGraph

* chore: add comments for lock module

---------

Co-authored-by: heiyan <heiyan2020@gmail.com>
This commit is contained in:
V_Galaxy 2023-09-18 17:17:37 +08:00 committed by GitHub
parent 0903e900e4
commit b670c01002
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
116 changed files with 12583 additions and 561 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java
View File

@ -54,8 +54,6 @@ import lombok.extern.slf4j.Slf4j;
/**
* PD客户端实现类
*
* @author yanjinbing
*/
@Slf4j
public class PDClient {

5
hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/API.java
View File

@ -136,11 +136,6 @@ public class API {
return builder;
}
/**
* @param object
* @return
* @author tianxiaohui
*/
public String toJSON(Object object) {
ObjectMapper mapper = new ObjectMapper();
try {

2
hugegraph-server/hugegraph-cassandra/src/main/java/org/apache/hugegraph/backend/store/cassandra/CassandraStore.java
View File

@ -594,7 +594,7 @@ public abstract class CassandraStore extends AbstractBackendStore<CassandraSessi
@Override
protected final CassandraTable table(HugeType type) {
return this.table(type.string());
return this.table(convertTaskOrServerToVertex(type).string());
}
protected final CassandraTable table(String name) {

26
hugegraph-server/hugegraph-core/pom.xml
View File

@ -236,6 +236,32 @@
<version>${jjwt.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.9</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-pd-client</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-common</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.etcd</groupId>
<artifactId>jetcd-core</artifactId>
<version>0.5.9</version>
<exclusions>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-core</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>

3
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraphParams.java
View File

@ -22,6 +22,7 @@ import org.apache.hugegraph.backend.store.BackendFeatures;
import org.apache.hugegraph.backend.store.BackendStore;
import org.apache.hugegraph.backend.store.ram.RamTable;
import org.apache.hugegraph.backend.tx.GraphTransaction;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.job.EphemeralJob;
import org.apache.hugegraph.task.ServerInfoManager;
@ -46,7 +47,7 @@ public interface HugeGraphParams {
GraphReadMode readMode();
SchemaTransaction schemaTransaction();
ISchemaTransaction schemaTransaction();
GraphTransaction systemTransaction();

45
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java
View File

@ -18,6 +18,7 @@
package org.apache.hugegraph;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
@ -37,6 +38,7 @@ import org.apache.hugegraph.backend.cache.CacheNotifier.GraphCacheNotifier;
import org.apache.hugegraph.backend.cache.CacheNotifier.SchemaCacheNotifier;
import org.apache.hugegraph.backend.cache.CachedGraphTransaction;
import org.apache.hugegraph.backend.cache.CachedSchemaTransaction;
import org.apache.hugegraph.backend.cache.CachedSchemaTransactionV2;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.backend.id.SnowflakeIdGenerator;
@ -51,9 +53,8 @@ import org.apache.hugegraph.backend.store.BackendStoreProvider;
import org.apache.hugegraph.backend.store.raft.RaftBackendStoreProvider;
import org.apache.hugegraph.backend.store.raft.RaftGroupManager;
import org.apache.hugegraph.backend.store.ram.RamTable;
import org.apache.hugegraph.task.EphemeralJobQueue;
import org.apache.hugegraph.backend.tx.GraphTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.config.CoreOptions;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.config.TypedOption;
@ -69,6 +70,7 @@ import org.apache.hugegraph.masterelection.RoleElectionOptions;
import org.apache.hugegraph.masterelection.RoleElectionStateMachine;
import org.apache.hugegraph.masterelection.StandardClusterRoleStore;
import org.apache.hugegraph.masterelection.StandardRoleElectionStateMachine;
import org.apache.hugegraph.meta.MetaManager;
import org.apache.hugegraph.perf.PerfUtil.Watched;
import org.apache.hugegraph.rpc.RpcServiceConfig4Client;
import org.apache.hugegraph.rpc.RpcServiceConfig4Server;
@ -84,6 +86,7 @@ import org.apache.hugegraph.structure.HugeEdgeProperty;
import org.apache.hugegraph.structure.HugeFeatures;
import org.apache.hugegraph.structure.HugeVertex;
import org.apache.hugegraph.structure.HugeVertexProperty;
import org.apache.hugegraph.task.EphemeralJobQueue;
import org.apache.hugegraph.task.ServerInfoManager;
import org.apache.hugegraph.task.TaskManager;
import org.apache.hugegraph.task.TaskScheduler;
@ -176,6 +179,8 @@ public class StandardHugeGraph implements HugeGraph {
private final RamTable ramtable;
private final MetaManager metaManager = MetaManager.instance();
public StandardHugeGraph(HugeConfig config) {
this.params = new StandardHugeGraphParams();
this.configuration = config;
@ -221,6 +226,10 @@ public class StandardHugeGraph implements HugeGraph {
throw new HugeException(message, e);
}
if (isHstore()) {
initMetaManager();
}
try {
this.tx = new TinkerPopTransaction(this);
boolean supportsPersistence = this.backendStoreFeatures().supportsPersistence();
@ -453,9 +462,24 @@ public class StandardHugeGraph implements HugeGraph {
}
}
private SchemaTransaction openSchemaTransaction() throws HugeException {
private boolean isHstore() {
return this.storeProvider.isHstore();
}
private void initMetaManager() {
this.metaManager.connect("hg", MetaManager.MetaDriverType.PD,
"ca", "ca", "ca",
Collections.singletonList("127.0.0.1:8686"));
}
private ISchemaTransaction openSchemaTransaction() throws HugeException {
this.checkGraphNotClosed();
try {
if (isHstore()) {
return new CachedSchemaTransactionV2(
MetaManager.instance().metaDriver(),
MetaManager.instance().cluster(), this.params);
}
return new CachedSchemaTransaction(this.params, loadSchemaStore());
} catch (BackendException e) {
String message = "Failed to open schema transaction";
@ -500,11 +524,14 @@ public class StandardHugeGraph implements HugeGraph {
}
private BackendStore loadSystemStore() {
if (isHstore()) {
return this.storeProvider.loadGraphStore(this.configuration);
}
return this.storeProvider.loadSystemStore(this.configuration);
}
@Watched
private SchemaTransaction schemaTransaction() {
private ISchemaTransaction schemaTransaction() {
this.checkGraphNotClosed();
/*
* NOTE: each schema operation will be auto committed,
@ -1192,7 +1219,7 @@ public class StandardHugeGraph implements HugeGraph {
}
@Override
public SchemaTransaction schemaTransaction() {
public ISchemaTransaction schemaTransaction() {
return StandardHugeGraph.this.schemaTransaction();
}
@ -1443,7 +1470,7 @@ public class StandardHugeGraph implements HugeGraph {
}
}
private SchemaTransaction schemaTransaction() {
private ISchemaTransaction schemaTransaction() {
return this.getOrNewTransaction().schemaTx;
}
@ -1464,7 +1491,7 @@ public class StandardHugeGraph implements HugeGraph {
Txs txs = this.transactions.get();
if (txs == null) {
SchemaTransaction schemaTransaction = null;
ISchemaTransaction schemaTransaction = null;
SysTransaction sysTransaction = null;
GraphTransaction graphTransaction = null;
try {
@ -1507,12 +1534,12 @@ public class StandardHugeGraph implements HugeGraph {
private static final class Txs {
private final SchemaTransaction schemaTx;
private final ISchemaTransaction schemaTx;
private final SysTransaction systemTx;
private final GraphTransaction graphTx;
private long openedTime;
public Txs(SchemaTransaction schemaTx, SysTransaction systemTx,
public Txs(ISchemaTransaction schemaTx, SysTransaction systemTx,
GraphTransaction graphTx) {
assert schemaTx != null && systemTx != null && graphTx != null;
this.schemaTx = schemaTx;

5
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeAccess.java
View File

@ -229,4 +229,9 @@ public class HugeAccess extends Relationship {
return super.initProperties(props);
}
}
public static HugeAccess fromMap(Map<String, Object> map) {
HugeAccess access = new HugeAccess(null, null, null);
return fromMap(map, access);
}
}

14
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeBelong.java
View File

@ -33,10 +33,15 @@ import org.apache.hugegraph.auth.SchemaDefine.Relationship;
public class HugeBelong extends Relationship {
public static final String UG = "ug";
public static final String UR = "ur";
public static final String GR = "gr";
public static final String ALL = "*";
private static final long serialVersionUID = -7242751631755533423L;
private final Id user;
private final Id group;
private String link;
private String description;
public HugeBelong(Id user, Id group) {
@ -75,6 +80,10 @@ public class HugeBelong extends Relationship {
return this.group;
}
public String link() {
return this.link;
}
public String description() {
return this.description;
}
@ -196,4 +205,9 @@ public class HugeBelong extends Relationship {
return super.initProperties(props);
}
}
public static HugeBelong fromMap(Map<String, Object> map) {
HugeBelong belong = new HugeBelong(null, null);
return fromMap(map, belong);
}
}

14
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeGroup.java
View File

@ -37,6 +37,7 @@ public class HugeGroup extends Entity {
private static final long serialVersionUID = 2330399818352242686L;
private String name;
private String nickname;
private String description;
public HugeGroup(String name) {
@ -68,6 +69,14 @@ public class HugeGroup extends Entity {
return this.name;
}
public String nickname() {
return this.nickname;
}
public void nickname(String nickname) {
this.nickname = nickname;
}
public String description() {
return this.description;
}
@ -195,4 +204,9 @@ public class HugeGroup extends Entity {
return super.initProperties(props);
}
}
public static HugeGroup fromMap(Map<String, Object> map) {
HugeGroup group = new HugeGroup("");
return fromMap(map, group);
}
}

240
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeRole.java Normal file
View File

@ -0,0 +1,240 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.auth;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.hugegraph.HugeGraphParams;
import org.apache.hugegraph.auth.SchemaDefine.Entity;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.schema.VertexLabel;
import org.apache.hugegraph.util.E;
import org.apache.tinkerpop.gremlin.structure.Graph.Hidden;
import org.apache.tinkerpop.gremlin.structure.T;
public class HugeRole extends Entity {
private static final long serialVersionUID = 2330399818352242686L;
private String name;
private String nickname;
private String graphSpace;
private String description;
public HugeRole(Id id, String name, String graphSpace) {
this.id = id;
this.name = name;
this.graphSpace = graphSpace;
this.description = null;
}
public HugeRole(String name, String graphSpace) {
this(StringUtils.isNotEmpty(name) ? IdGenerator.of(name) : null,
name, graphSpace);
}
public HugeRole(Id id, String graphSpace) {
this(id, id.asString(), graphSpace);
}
public static HugeRole fromMap(Map<String, Object> map) {
HugeRole role = new HugeRole("", "");
return fromMap(map, role);
}
public static Schema schema(HugeGraphParams graph) {
return new Schema(graph);
}
@Override
public ResourceType type() {
return ResourceType.GRANT;
}
@Override
public String label() {
return P.ROLE;
}
@Override
public String name() {
return this.name;
}
public void name(String name) {
this.name = name;
}
public String nickname() {
return this.nickname;
}
public void nickname(String nickname) {
this.nickname = nickname;
}
public String graphSpace() {
return this.graphSpace;
}
public String description() {
return this.description;
}
public void description(String description) {
this.description = description;
}
@Override
public String toString() {
return String.format("HugeGroup(%s)", this.id);
}
@Override
protected boolean property(String key, Object value) {
if (super.property(key, value)) {
return true;
}
switch (key) {
case P.GRAPHSPACE:
this.graphSpace = (String) value;
break;
case P.NAME:
this.name = (String) value;
break;
case P.NICKNAME:
this.nickname = (String) value;
break;
case P.DESCRIPTION:
this.description = (String) value;
break;
default:
throw new AssertionError("Unsupported key: " + key);
}
return true;
}
@Override
protected Object[] asArray() {
E.checkState(this.name != null, "Group name can't be null");
List<Object> list = new ArrayList<>(12);
list.add(T.label);
list.add(P.ROLE);
list.add(P.GRAPHSPACE);
list.add(this.graphSpace);
list.add(P.NAME);
list.add(this.name);
if (this.nickname != null) {
list.add(P.NICKNAME);
list.add(this.nickname);
}
if (this.description != null) {
list.add(P.DESCRIPTION);
list.add(this.description);
}
return super.asArray(list);
}
@Override
public Map<String, Object> asMap() {
E.checkState(this.name != null, "Group name can't be null");
Map<String, Object> map = new HashMap<>();
map.put(Hidden.unHide(P.NAME), this.name);
map.put(Hidden.unHide(P.GRAPHSPACE), this.graphSpace);
if (this.description != null) {
map.put(Hidden.unHide(P.DESCRIPTION), this.description);
}
if (this.nickname != null) {
map.put(Hidden.unHide(P.NICKNAME), this.nickname);
}
return super.asMap(map);
}
public static final class P {
public static final String ROLE = Hidden.hide("role");
public static final String ID = T.id.getAccessor();
public static final String LABEL = T.label.getAccessor();
public static final String NAME = "~role_name";
public static final String NICKNAME = "~role_nickname";
public static final String GRAPHSPACE = "~graphspace";
public static final String DESCRIPTION = "~role_description";
public static String unhide(String key) {
final String prefix = Hidden.hide("role_");
if (key.startsWith(prefix)) {
return key.substring(prefix.length());
}
return key;
}
}
public static final class Schema extends SchemaDefine {
public Schema(HugeGraphParams graph) {
super(graph, P.ROLE);
}
@Override
public void initSchemaIfNeeded() {
if (this.existVertexLabel(this.label)) {
return;
}
String[] properties = this.initProperties();
// Create vertex label
VertexLabel label = this.schema().vertexLabel(this.label)
.properties(properties)
.usePrimaryKeyId()
.primaryKeys(P.NAME)
.nullableKeys(P.DESCRIPTION, P.NICKNAME)
.enableLabelIndex(true)
.build();
this.graph.schemaTransaction().addVertexLabel(label);
}
protected String[] initProperties() {
List<String> props = new ArrayList<>();
props.add(createPropertyKey(P.NAME));
props.add(createPropertyKey(P.DESCRIPTION));
props.add(createPropertyKey(P.NICKNAME));
return super.initProperties(props);
}
}
}

18
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeTarget.java
View File

@ -42,6 +42,7 @@ public class HugeTarget extends Entity {
private String name;
private String graph;
private String description;
private String url;
private List<HugeResource> resources;
@ -92,6 +93,18 @@ public class HugeTarget extends Entity {
return this.graph;
}
public void graph(String graph) {
this.graph = graph;
}
public String description() {
return this.description;
}
public void description(String description) {
this.description = description;
}
public String url() {
return this.url;
}
@ -258,4 +271,9 @@ public class HugeTarget extends Entity {
return super.initProperties(props);
}
}
public static HugeTarget fromMap(Map<String, Object> map) {
HugeTarget target = new HugeTarget(null);
return fromMap(map, target);
}
}

14
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/HugeUser.java
View File

@ -37,6 +37,7 @@ public class HugeUser extends Entity {
private static final long serialVersionUID = -8951193710873772717L;
private String name;
private String nickname;
private String password;
private String phone;
private String email;
@ -74,6 +75,14 @@ public class HugeUser extends Entity {
return this.name;
}
public String nickname() {
return nickname;
}
public void nickname(String nickname) {
this.nickname = nickname;
}
public String password() {
return this.password;
}
@ -281,4 +290,9 @@ public class HugeUser extends Entity {
return super.initProperties(props);
}
}
public static HugeUser fromMap(Map<String, Object> map) {
HugeUser user = new HugeUser("");
return fromMap(map, user);
}
}

22
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/auth/SchemaDefine.java
View File

@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.hugegraph.auth.HugeTarget.P;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.schema.IndexLabel;
import org.apache.hugegraph.schema.PropertyKey;
import org.apache.hugegraph.schema.SchemaManager;
@ -245,6 +246,14 @@ public abstract class SchemaDefine {
private static final long serialVersionUID = 4113319546914811762L;
public static <T extends Entity> T fromMap(Map<String, Object> map, T entity) {
for (Map.Entry<String, Object> item : map.entrySet()) {
entity.property(Hidden.hide(item.getKey()), item.getValue());
}
entity.id(IdGenerator.of(entity.name()));
return entity;
}
public static <T extends Entity> T fromVertex(Vertex vertex, T entity) {
E.checkArgument(vertex.label().equals(entity.label()),
"Illegal vertex label '%s' for entity '%s'",
@ -281,6 +290,19 @@ public abstract class SchemaDefine {
public abstract Id target();
public void setId() {
this.id(IdGenerator.of(this.source().asString() + "->" +
this.target().asString()));
}
public static <T extends Relationship> T fromMap(Map<String, Object> map, T entity) {
for (Map.Entry<String, Object> item : map.entrySet()) {
entity.property(Hidden.hide(item.getKey()), item.getValue());
}
entity.setId();
return entity;
}
public static <T extends Relationship> T fromEdge(Edge edge,
T relationship) {
E.checkArgument(edge.label().equals(relationship.label()),

469
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java vendored Normal file
View File

@ -0,0 +1,469 @@
package org.apache.hugegraph.backend.cache;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
import org.apache.hugegraph.HugeGraphParams;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.backend.store.ram.IntObjectMap;
import org.apache.hugegraph.backend.tx.SchemaTransactionV2;
import org.apache.hugegraph.config.CoreOptions;
import org.apache.hugegraph.event.EventHub;
import org.apache.hugegraph.event.EventListener;
import org.apache.hugegraph.meta.MetaDriver;
import org.apache.hugegraph.meta.MetaManager;
import org.apache.hugegraph.perf.PerfUtil;
import org.apache.hugegraph.schema.SchemaElement;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.Events;
import com.google.common.collect.ImmutableSet;
public class CachedSchemaTransactionV2 extends SchemaTransactionV2 {
private final Cache<Id, Object> idCache;
private final Cache<Id, Object> nameCache;
private final SchemaCaches<SchemaElement> arrayCaches;
private EventListener storeEventListener;
private EventListener cacheEventListener;
public CachedSchemaTransactionV2(MetaDriver metaDriver,
String cluster,
HugeGraphParams graphParams) {
super(metaDriver, cluster, graphParams);
final long capacity = graphParams.configuration()
.get(CoreOptions.SCHEMA_CACHE_CAPACITY);
this.idCache = this.cache("schema-id", capacity);
this.nameCache = this.cache("schema-name", capacity);
SchemaCaches<SchemaElement> attachment = this.idCache.attachment();
if (attachment == null) {
int acSize = (int) (capacity >> 3);
attachment = this.idCache.attachment(new SchemaCaches<>(acSize));
}
this.arrayCaches = attachment;
this.listenChanges();
}
private static Id generateId(HugeType type, Id id) {
// NOTE: it's slower performance to use:
// String.format("%x-%s", type.code(), name)
return IdGenerator.of(type.string() + "-" + id.asString());
}
private static Id generateId(HugeType type, String name) {
return IdGenerator.of(type.string() + "-" + name);
}
public void close() {
this.clearCache(false);
this.unlistenChanges();
}
private Cache<Id, Object> cache(String prefix, long capacity) {
// TODO: uncomment later - graph space
//final String name = prefix + "-" + this.graph().spaceGraphName();
final String name = prefix + "-" + "";
// NOTE: must disable schema cache-expire due to getAllSchema()
return CacheManager.instance().cache(name, capacity);
}
private void listenChanges() {
// Listen store event: "store.init", "store.clear", ...
Set<String> storeEvents = ImmutableSet.of(Events.STORE_INIT,
Events.STORE_CLEAR,
Events.STORE_TRUNCATE);
this.storeEventListener = event -> {
if (storeEvents.contains(event.name())) {
LOG.debug("Graph {} clear schema cache on event '{}'",
this.graph(), event.name());
this.clearCache(true);
return true;
}
return false;
};
this.graphParams().loadGraphStore().provider().listen(this.storeEventListener);
// Listen cache event: "cache"(invalid cache item)
this.cacheEventListener = event -> {
LOG.debug("Graph {} received schema cache event: {}",
this.graph(), event);
Object[] args = event.args();
E.checkArgument(args.length > 0 && args[0] instanceof String,
"Expect event action argument");
if (Cache.ACTION_INVALID.equals(args[0])) {
event.checkArgs(String.class, HugeType.class, Id.class);
HugeType type = (HugeType) args[1];
Id id = (Id) args[2];
this.arrayCaches.remove(type, id);
id = generateId(type, id);
Object value = this.idCache.get(id);
if (value != null) {
// Invalidate id cache
this.idCache.invalidate(id);
// Invalidate name cache
SchemaElement schema = (SchemaElement) value;
Id prefixedName = generateId(schema.type(),
schema.name());
this.nameCache.invalidate(prefixedName);
}
this.resetCachedAll(type);
return true;
} else if (Cache.ACTION_CLEAR.equals(args[0])) {
event.checkArgs(String.class, HugeType.class);
this.clearCache(false);
return true;
}
return false;
};
EventHub schemaEventHub = this.graphParams().schemaEventHub();
if (!schemaEventHub.containsListener(Events.CACHE)) {
schemaEventHub.listen(Events.CACHE, this.cacheEventListener);
}
}
public void clearCache(boolean notify) {
this.idCache.clear();
this.nameCache.clear();
this.arrayCaches.clear();
}
private void resetCachedAllIfReachedCapacity() {
if (this.idCache.size() >= this.idCache.capacity()) {
LOG.warn("Schema cache reached capacity({}): {}",
this.idCache.capacity(), this.idCache.size());
this.cachedTypes().clear();
}
}
private void unlistenChanges() {
// Unlisten store event
this.graphParams().loadGraphStore().provider()
.unlisten(this.storeEventListener);
// Unlisten cache event
EventHub schemaEventHub = this.graphParams().schemaEventHub();
schemaEventHub.unlisten(Events.CACHE, this.cacheEventListener);
}
private CachedTypes cachedTypes() {
return this.arrayCaches.cachedTypes();
}
private void resetCachedAll(HugeType type) {
// Set the cache all flag of the schema type to false
this.cachedTypes().put(type, false);
}
private void invalidateCache(HugeType type, Id id) {
// remove from id cache and name cache
Id prefixedId = generateId(type, id);
Object value = this.idCache.get(prefixedId);
if (value != null) {
this.idCache.invalidate(prefixedId);
SchemaElement schema = (SchemaElement) value;
Id prefixedName = generateId(schema.type(), schema.name());
this.nameCache.invalidate(prefixedName);
}
// remove from optimized array cache
this.arrayCaches.remove(type, id);
}
@Override
protected void updateSchema(SchemaElement schema,
Consumer<SchemaElement> updateCallback) {
super.updateSchema(schema, updateCallback);
this.updateCache(schema);
}
@Override
protected void addSchema(SchemaElement schema) {
super.addSchema(schema);
this.updateCache(schema);
if (!this.graph().option(CoreOptions.TASK_SYNC_DELETION)) {
MetaManager.instance()
// TODO: uncomment later - graph space
//.notifySchemaCacheClear(this.graph().graphSpace(),
// this.graph().name());
.notifySchemaCacheClear("",
this.graph().name());
}
}
private void updateCache(SchemaElement schema) {
this.resetCachedAllIfReachedCapacity();
// update id cache
Id prefixedId = generateId(schema.type(), schema.id());
this.idCache.update(prefixedId, schema);
// update name cache
Id prefixedName = generateId(schema.type(), schema.name());
this.nameCache.update(prefixedName, schema);
// update optimized array cache
this.arrayCaches.updateIfNeeded(schema);
}
@Override
protected void removeSchema(SchemaElement schema) {
super.removeSchema(schema);
this.invalidateCache(schema.type(), schema.id());
if (!this.graph().option(CoreOptions.TASK_SYNC_DELETION)) {
MetaManager.instance()
// TODO: uncomment later - graph space
//.notifySchemaCacheClear(this.graph().graphSpace(),
// this.graph().name());
.notifySchemaCacheClear("",
this.graph().name());
}
}
@Override
@SuppressWarnings("unchecked")
protected <T extends SchemaElement> T getSchema(HugeType type, Id id) {
// try get from optimized array cache
if (id.number() && id.asLong() > 0L) {
SchemaElement value = this.arrayCaches.get(type, id);
if (value != null) {
return (T) value;
}
}
Id prefixedId = generateId(type, id);
Object value = this.idCache.get(prefixedId);
if (value == null) {
value = super.getSchema(type, id);
if (value != null) {
this.resetCachedAllIfReachedCapacity();
this.idCache.update(prefixedId, value);
SchemaElement schema = (SchemaElement) value;
Id prefixedName = generateId(schema.type(), schema.name());
this.nameCache.update(prefixedName, schema);
}
}
// update optimized array cache
this.arrayCaches.updateIfNeeded((SchemaElement) value);
return (T) value;
}
@Override
@SuppressWarnings("unchecked")
protected <T extends SchemaElement> T getSchema(HugeType type,
String name) {
Id prefixedName = generateId(type, name);
Object value = this.nameCache.get(prefixedName);
if (value == null) {
value = super.getSchema(type, name);
if (value != null) {
this.resetCachedAllIfReachedCapacity();
this.nameCache.update(prefixedName, value);
SchemaElement schema = (SchemaElement) value;
Id prefixedId = generateId(schema.type(), schema.id());
this.idCache.update(prefixedId, schema);
}
}
return (T) value;
}
@Override
protected <T extends SchemaElement> List<T> getAllSchema(HugeType type) {
Boolean cachedAll = this.cachedTypes().getOrDefault(type, false);
List<T> results;
if (cachedAll) {
results = new ArrayList<>();
// Get from cache
this.idCache.traverse(value -> {
@SuppressWarnings("unchecked")
T schema = (T) value;
if (schema.type() == type) {
results.add(schema);
}
});
return results;
} else {
results = super.getAllSchema(type);
long free = this.idCache.capacity() - this.idCache.size();
if (results.size() <= free) {
// Update cache
for (T schema : results) {
Id prefixedId = generateId(schema.type(), schema.id());
this.idCache.update(prefixedId, schema);
Id prefixedName = generateId(schema.type(), schema.name());
this.nameCache.update(prefixedName, schema);
}
this.cachedTypes().putIfAbsent(type, true);
}
return results;
}
}
@Override
public void clear() {
// Clear schema info firstly
super.clear();
this.clearCache(false);
}
private static final class SchemaCaches<V extends SchemaElement> {
private final int size;
private final IntObjectMap<V> pks;
private final IntObjectMap<V> vls;
private final IntObjectMap<V> els;
private final IntObjectMap<V> ils;
private final CachedTypes cachedTypes;
public SchemaCaches(int size) {
// TODO: improve size of each type for optimized array cache
this.size = size;
this.pks = new IntObjectMap<>(size);
this.vls = new IntObjectMap<>(size);
this.els = new IntObjectMap<>(size);
this.ils = new IntObjectMap<>(size);
this.cachedTypes = new CachedTypes();
}
public void updateIfNeeded(V schema) {
if (schema == null) {
return;
}
Id id = schema.id();
if (id.number() && id.asLong() > 0L) {
this.set(schema.type(), id, schema);
}
}
@PerfUtil.Watched
public V get(HugeType type, Id id) {
assert id.number();
long longId = id.asLong();
if (longId <= 0L) {
assert false : id;
return null;
}
int key = (int) longId;
if (key >= this.size) {
return null;
}
switch (type) {
case PROPERTY_KEY:
return this.pks.get(key);
case VERTEX_LABEL:
return this.vls.get(key);
case EDGE_LABEL:
return this.els.get(key);
case INDEX_LABEL:
return this.ils.get(key);
default:
return null;
}
}
public void set(HugeType type, Id id, V value) {
assert id.number();
long longId = id.asLong();
if (longId <= 0L) {
assert false : id;
return;
}
int key = (int) longId;
if (key >= this.size) {
return;
}
switch (type) {
case PROPERTY_KEY:
this.pks.set(key, value);
break;
case VERTEX_LABEL:
this.vls.set(key, value);
break;
case EDGE_LABEL:
this.els.set(key, value);
break;
case INDEX_LABEL:
this.ils.set(key, value);
break;
default:
// pass
break;
}
}
public void remove(HugeType type, Id id) {
assert id.number();
long longId = id.asLong();
if (longId <= 0L) {
return;
}
int key = (int) longId;
V value = null;
if (key >= this.size) {
return;
}
switch (type) {
case PROPERTY_KEY:
this.pks.set(key, value);
break;
case VERTEX_LABEL:
this.vls.set(key, value);
break;
case EDGE_LABEL:
this.els.set(key, value);
break;
case INDEX_LABEL:
this.ils.set(key, value);
break;
default:
// pass
break;
}
}
public void clear() {
this.pks.clear();
this.vls.clear();
this.els.clear();
this.ils.clear();
this.cachedTypes.clear();
}
public CachedTypes cachedTypes() {
return this.cachedTypes;
}
}
private static class CachedTypes
extends ConcurrentHashMap<HugeType, Boolean> {
private static final long serialVersionUID = -2215549791679355996L;
}
}

2
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/IdGenerator.java
View File

@ -379,7 +379,7 @@ public abstract class IdGenerator {
/**
* This class is just used by backend store for wrapper object as Id
*/
private static final class ObjectId implements Id {
public static final class ObjectId implements Id {
private final Object object;

88
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/Condition.java
View File

@ -17,6 +17,7 @@
package org.apache.hugegraph.backend.query;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
@ -47,7 +48,8 @@ public abstract class Condition {
NONE,
RELATION,
AND,
OR;
OR,
NOT;
}
public enum RelationType implements BiPredicate<Object, Object> {
@ -300,7 +302,8 @@ public abstract class Condition {
public boolean isLogic() {
return this.type() == ConditionType.AND ||
this.type() == ConditionType.OR;
this.type() == ConditionType.OR ||
this.type() == ConditionType.NOT;
}
public boolean isFlattened() {
@ -315,6 +318,10 @@ public abstract class Condition {
return new Or(left, right);
}
public static Condition not(Condition condition) {
return new Not(condition);
}
public static Relation eq(HugeKeys key, Object value) {
return new SyspropRelation(key, RelationType.EQ, value);
}
@ -536,6 +543,79 @@ public abstract class Condition {
}
}
public static class Not extends Condition implements Serializable {
Condition condition;
public Not(Condition condition) {
super();
this.condition = condition;
}
public Condition condition() {
return condition;
}
@Override
public ConditionType type() {
return ConditionType.NOT;
}
@Override
public boolean test(Object value) {
return !this.condition.test(value);
}
@Override
public boolean test(HugeElement element) {
return !this.condition.test(element);
}
@Override
public Condition copy() {
return new Not(this.condition.copy());
}
@Override
public boolean isSysprop() {
return this.condition.isSysprop();
}
@Override
public List<? extends Relation> relations() {
return new ArrayList<Relation>(this.condition.relations());
}
@Override
public Condition replace(Relation from, Relation to) {
this.condition = this.condition.replace(from, to);
return this;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder(64);
sb.append(this.type().name()).append(' ');
sb.append(this.condition);
return sb.toString();
}
@Override
public boolean equals(Object object) {
if (!(object instanceof Not)) {
return false;
}
Not other = (Not) object;
return this.type().equals(other.type()) &&
this.condition.equals(other.condition());
}
@Override
public int hashCode() {
return this.type().hashCode() ^
this.condition.hashCode();
}
}
public abstract static class Relation extends Condition {
// Relational operator (like: =, >, <, in, ...)
@ -565,6 +645,10 @@ public abstract class Condition {
return this.value;
}
public void value(Object value) {
this.value = value;
}
public void serialKey(Object key) {
this.serialKey = key;
}

29
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/ConditionQuery.java
View File

@ -18,6 +18,7 @@
package org.apache.hugegraph.backend.query;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -34,6 +35,8 @@ import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.SplicingIdGenerator;
import org.apache.hugegraph.backend.query.Condition.Relation;
import org.apache.hugegraph.backend.query.Condition.RelationType;
import org.apache.hugegraph.backend.query.serializer.QueryAdapter;
import org.apache.hugegraph.backend.query.serializer.QueryIdAdapter;
import org.apache.hugegraph.perf.PerfUtil.Watched;
import org.apache.hugegraph.structure.HugeElement;
import org.apache.hugegraph.structure.HugeProperty;
@ -44,9 +47,12 @@ import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.InsertionOrderUtil;
import org.apache.hugegraph.util.LongEncoding;
import org.apache.hugegraph.util.NumericUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
public class ConditionQuery extends IdQuery {
@ -71,6 +77,12 @@ public class ConditionQuery extends IdQuery {
private static final List<Condition> EMPTY_CONDITIONS = ImmutableList.of();
private static final Gson gson = new GsonBuilder()
.registerTypeAdapter(Condition.class, new QueryAdapter())
.registerTypeAdapter(Id.class, new QueryIdAdapter())
.setDateFormat("yyyy-MM-dd HH:mm:ss.SSS")
.create();
// Conditions will be contacted with `and` by default
private List<Condition> conditions = EMPTY_CONDITIONS;
@ -681,6 +693,18 @@ public class ConditionQuery extends IdQuery {
}
}
public static ConditionQuery fromBytes(byte[] bytes) {
Gson gson = new GsonBuilder()
.registerTypeAdapter(Condition.class, new QueryAdapter())
.registerTypeAdapter(Id.class, new QueryIdAdapter())
.setDateFormat("yyyy-MM-dd HH:mm:ss.SSS")
.create();
String cqs = new String(bytes, StandardCharsets.UTF_8);
ConditionQuery conditionQuery = gson.fromJson(cqs, ConditionQuery.class);
return conditionQuery;
}
private static boolean needConvertNumber(Object value) {
// Numeric or date values should be converted to number from string
return NumericUtil.isNumber(value) || value instanceof Date;
@ -870,4 +894,9 @@ public class ConditionQuery extends IdQuery {
boolean test(HugeElement element);
}
public byte[] bytes() {
String cqs = gson.toJson(this);
return cqs.getBytes(StandardCharsets.UTF_8);
}
}

66
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/serializer/AbstractSerializerAdapter.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.query.serializer;
import java.lang.reflect.Type;
import java.util.Map;
import org.apache.hugegraph.backend.BackendException;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
// TODO: optimize by binary protocol
public abstract class AbstractSerializerAdapter<T> implements JsonSerializer<T>,
JsonDeserializer<T> {
//Note: By overriding the method to get the mapping
public abstract Map<String, Type> validType();
@Override
public T deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws
JsonParseException {
JsonObject object = json.getAsJsonObject();
String type = object.get("cls").getAsString();
JsonElement element = object.get("el");
try {
return context.deserialize(element, validType().get(type));
} catch (Exception e) {
throw new BackendException("Unknown element type: " + type, e);
}
}
/*
* Note: Currently, only the first character of the class name is taken as the key
* to reduce serialization results
* */
@Override
public JsonElement serialize(T src, Type typeOfSrc, JsonSerializationContext context) {
JsonObject result = new JsonObject();
Class clazz = src.getClass();
result.add("cls", new JsonPrimitive(clazz.getSimpleName().substring(0, 1).toUpperCase()));
result.add("el", context.serialize(src, clazz));
return result;
}
}

148
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/serializer/QueryAdapter.java Normal file
View File

@ -0,0 +1,148 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.query.serializer;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import org.apache.hugegraph.backend.query.Condition;
import org.apache.hugegraph.type.define.Directions;
import com.google.common.collect.ImmutableMap;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;
import com.google.gson.reflect.TypeToken;
public class QueryAdapter extends AbstractSerializerAdapter<Condition> {
static ImmutableMap<String, Type> cls =
ImmutableMap.<String, Type>builder()
// TODO: uncomment later
.put("N", Condition.Not.class)
.put("A", Condition.And.class)
.put("O", Condition.Or.class)
.put("S", Condition.SyspropRelation.class)
.put("U", Condition.UserpropRelation.class)
.build();
static boolean isPrimitive(Class clz) {
try {
return (clz == Date.class) || ((Class) clz.getField("TYPE").get(null)).isPrimitive();
} catch (Exception e) {
return false;
}
}
@Override
public Map<String, Type> validType() {
return cls;
}
@Override
public Condition deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
throws JsonParseException {
Condition condition = super.deserialize(json, typeOfT, context);
if (condition instanceof Condition.Relation) {
JsonObject object = json.getAsJsonObject();
if (object.has("el")) {
JsonElement elElement = object.get("el");
JsonElement valueElement = elElement.getAsJsonObject().get("value");
if (valueElement.isJsonObject()) {
String cls = valueElement.getAsJsonObject().get("cls").getAsString();
try {
Class actualClass = Class.forName(cls);
Object obj = context.deserialize(valueElement, actualClass);
((Condition.Relation) condition).value(obj);
} catch (ClassNotFoundException e) {
throw new JsonParseException(e.getMessage());
}
} else if (elElement.getAsJsonObject().has("valuecls")) {
if (valueElement.isJsonArray()) {
String cls = elElement.getAsJsonObject().get("valuecls").getAsString();
try {
Class actualClass = Class.forName(cls);
Type type = TypeToken.getParameterized(ArrayList.class, actualClass)
.getType();
Object value = context.deserialize(valueElement, type);
((Condition.Relation) condition).value(value);
} catch (ClassNotFoundException e) {
throw new JsonParseException(e.getMessage());
}
} else {
String cls = elElement.getAsJsonObject().get("valuecls").getAsString();
try {
Class actualClass = Class.forName(cls);
Object obj = context.deserialize(valueElement, actualClass);
((Condition.Relation) condition).value(obj);
} catch (ClassNotFoundException e) {
throw new JsonParseException(e.getMessage());
}
}
} else if (valueElement.isJsonPrimitive() &&
valueElement.getAsJsonPrimitive().isString()) {
switch ((String) ((Condition.Relation) condition).value()) {
case "OUT":
((Condition.Relation) condition).value(Directions.OUT);
break;
case "IN":
((Condition.Relation) condition).value(Directions.IN);
break;
default:
break;
}
}
}
}
return condition;
}
@Override
public JsonElement serialize(Condition src, Type typeOfSrc, JsonSerializationContext context) {
JsonElement result = super.serialize(src, typeOfSrc, context);
if (src instanceof Condition.Relation) {
JsonObject object = result.getAsJsonObject();
JsonElement valueElement = object.get("el").getAsJsonObject().get("value");
if (valueElement.isJsonObject()) {
valueElement.getAsJsonObject()
.add("cls",
new JsonPrimitive(
((Condition.Relation) src).value().getClass().getName()));
} else if (isPrimitive(((Condition.Relation) src).value().getClass())) {
object.get("el").getAsJsonObject()
.add("valuecls",
new JsonPrimitive(
((Condition.Relation) src).value().getClass().getName()));
} else if (valueElement.isJsonArray()) {
if (((Condition.Relation) src).value() instanceof List) {
String valueCls =
((List) ((Condition.Relation) src).value()).get(0).getClass().getName();
object.get("el").getAsJsonObject().add("valuecls", new JsonPrimitive(valueCls));
}
}
}
return result;
}
}

46
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/serializer/QueryIdAdapter.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.query.serializer;
import java.lang.reflect.Type;
import java.util.Map;
import org.apache.hugegraph.backend.id.EdgeId;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.backend.serializer.BinaryBackendEntry;
import com.google.common.collect.ImmutableMap;
public class QueryIdAdapter extends AbstractSerializerAdapter<Id> {
static ImmutableMap<String, Type> cls =
ImmutableMap.<String, Type>builder()
.put("E", EdgeId.class)
.put("S", IdGenerator.StringId.class)
.put("L", IdGenerator.LongId.class)
.put("U", IdGenerator.UuidId.class)
.put("O", IdGenerator.ObjectId.class)
.put("B", BinaryBackendEntry.BinaryId.class)
.build();
@Override
public Map<String, Type> validType() {
return cls;
}
}

9
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/AbstractSerializer.java
View File

@ -17,14 +17,19 @@
package org.apache.hugegraph.backend.serializer;
import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.backend.BackendException;
import org.apache.hugegraph.backend.id.EdgeId;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.query.ConditionQuery;
import org.apache.hugegraph.backend.query.IdQuery;
import org.apache.hugegraph.backend.query.Query;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.iterator.CIter;
import org.apache.hugegraph.structure.HugeVertex;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.tinkerpop.gremlin.structure.Edge;
public abstract class AbstractSerializer
implements GraphSerializer, SchemaSerializer {
@ -89,4 +94,8 @@ public abstract class AbstractSerializer
return query;
}
public CIter<Edge> readEdges(HugeGraph graph, BackendEntry bytesEntry) {
throw new RuntimeException("Method not implemented error.");
}
}

2
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinaryBackendEntry.java
View File

@ -199,7 +199,7 @@ public class BinaryBackendEntry implements BackendEntry {
return this.id().hashCode() ^ this.columns.size();
}
protected static final class BinaryId implements Id {
public static final class BinaryId implements Id {
private final byte[] bytes;
private final Id id;

40
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java
View File

@ -17,6 +17,8 @@
package org.apache.hugegraph.backend.serializer;
import static org.apache.hugegraph.schema.SchemaElement.UNDEF;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
@ -33,7 +35,10 @@ import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.backend.page.PageState;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.backend.store.BackendEntry.BackendColumn;
import org.apache.hugegraph.iterator.CIter;
import org.apache.hugegraph.iterator.MapperIterator;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.type.define.EdgeLabelType;
import org.apache.hugegraph.util.*;
import org.apache.hugegraph.backend.query.Condition;
import org.apache.hugegraph.backend.query.Condition.RangeConditions;
@ -68,6 +73,7 @@ import org.apache.hugegraph.type.define.SerialEnum;
import org.apache.hugegraph.type.define.WriteType;
import org.apache.hugegraph.util.JsonUtil;
import org.apache.hugegraph.util.StringEncoding;
import org.apache.tinkerpop.gremlin.structure.Edge;
public class BinarySerializer extends AbstractSerializer {
@ -523,6 +529,40 @@ public class BinarySerializer extends AbstractSerializer {
return edges.iterator().next();
}
@Override
public CIter<Edge> readEdges(HugeGraph graph, BackendEntry bytesEntry) {
BinaryBackendEntry entry = this.convertEntry(bytesEntry);
// Parse id
Id id = entry.id().origin();
Id vid = id.edge() ? ((EdgeId) id).ownerVertexId() : id;
HugeVertex vertex = new HugeVertex(graph, vid, VertexLabel.NONE);
// Parse all properties and edges of a Vertex
Iterator<BackendColumn> iterator = entry.columns().iterator();
for (int index = 0; iterator.hasNext(); index++) {
BackendColumn col = iterator.next();
if (entry.type().isEdge()) {
// NOTE: the entry id type is vertex even if entry type is edge
// Parse vertex edges
this.parseColumn(col, vertex);
} else {
assert entry.type().isVertex();
// Parse vertex properties
assert entry.columnsSize() >= 1 : entry.columnsSize();
if (index == 0) {
this.parseVertex(col.value, vertex);
} else {
this.parseVertexOlap(col.value, vertex);
}
}
}
// convert to CIter
return new MapperIterator<>(vertex.getEdges().iterator(),
(edge) -> edge);
}
@Override
public BackendEntry writeIndex(HugeIndex index) {
BinaryBackendEntry entry;

12
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java
View File

@ -597,6 +597,10 @@ public final class BytesBuffer extends OutputStream {
}
}
public static byte getType(int value) {
return (byte) (value & 0x3f);
}
public Object readProperty(DataType dataType) {
switch (dataType) {
case BOOLEAN:
@ -752,11 +756,11 @@ public final class BytesBuffer extends OutputStream {
public BinaryId readIndexId(HugeType type) {
byte[] id;
if (type.isRange4Index()) {
// IndexLabel 4 bytes + fieldValue 4 bytes
id = this.read(8);
// HugeTypeCode 1 byte + IndexLabel 4 bytes + fieldValue 4 bytes
id = this.read(9);
} else if (type.isRange8Index()) {
// IndexLabel 4 bytes + fieldValue 8 bytes
id = this.read(12);
// HugeTypeCode 1 byte + IndexLabel 4 bytes + fieldValue 8 bytes
id = this.read(13);
} else {
assert type.isStringIndex();
id = this.readBytesWithEnding();

3
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/GraphSerializer.java
View File

@ -22,12 +22,14 @@ import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.query.ConditionQuery;
import org.apache.hugegraph.backend.query.Query;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.iterator.CIter;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.structure.HugeEdge;
import org.apache.hugegraph.structure.HugeEdgeProperty;
import org.apache.hugegraph.structure.HugeIndex;
import org.apache.hugegraph.structure.HugeVertex;
import org.apache.hugegraph.structure.HugeVertexProperty;
import org.apache.tinkerpop.gremlin.structure.Edge;
public interface GraphSerializer {
@ -44,6 +46,7 @@ public interface GraphSerializer {
BackendEntry writeEdgeProperty(HugeEdgeProperty<?> prop);
HugeEdge readEdge(HugeGraph graph, BackendEntry entry);
CIter<Edge> readEdges(HugeGraph graph, BackendEntry bytesEntry);
BackendEntry writeIndex(HugeIndex index);

10
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/TextSerializer.java
View File

@ -30,6 +30,7 @@ import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hugegraph.iterator.CIter;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.util.JsonUtil;
import org.apache.hugegraph.backend.id.EdgeId;
@ -66,6 +67,8 @@ import org.apache.hugegraph.type.define.IndexType;
import org.apache.hugegraph.type.define.SchemaStatus;
import org.apache.hugegraph.type.define.WriteType;
import org.apache.hugegraph.util.E;
import org.apache.tinkerpop.gremlin.structure.Edge;
import com.google.common.collect.ImmutableMap;
public class TextSerializer extends AbstractSerializer {
@ -352,6 +355,13 @@ public class TextSerializer extends AbstractSerializer {
throw new NotImplementedException("Unsupported readEdge()");
}
@Override
public CIter<Edge> readEdges(HugeGraph graph, BackendEntry bytesEntry) {
E.checkNotNull(graph, "serializer graph");
// TODO: implement
throw new NotImplementedException("Unsupported readEdges()");
}
@Override
public BackendEntry writeIndex(HugeIndex index) {
TextBackendEntry entry = newBackendEntry(index.type(), index.id());

7
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/AbstractBackendStore.java
View File

@ -80,6 +80,13 @@ public abstract class AbstractBackendStore<Session extends BackendSession>
protected abstract BackendTable<Session, ?> table(HugeType type);
protected static HugeType convertTaskOrServerToVertex(HugeType type) {
if (HugeType.TASK.equals(type) || HugeType.SERVER.equals(type)) {
return HugeType.VERTEX;
}
return type;
}
// NOTE: Need to support passing null
protected abstract Session session(HugeType type);
}

2
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendFeatures.java
View File

@ -31,6 +31,8 @@ public interface BackendFeatures {
return false;
}
default boolean supportsTaskAndServerVertex() { return false; }
boolean supportsScanToken();
boolean supportsScanKeyPrefix();

4
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendMutation.java
View File

@ -341,4 +341,8 @@ public class BackendMutation {
this.mutations.clear();
}
}
public Map<HugeType, Map<Id, List<BackendAction>>> mutations() {
return this.updates.mutations;
}
}

9
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendStoreInfo.java
View File

@ -41,9 +41,14 @@ public class BackendStoreInfo {
}
public boolean checkVersion() {
BackendStore store;
if (this.storeProvider.isHstore()) {
store = this.storeProvider.loadGraphStore(this.config);
} else {
store = this.storeProvider.loadSystemStore(this.config);
}
String driverVersion = this.storeProvider.driverVersion();
String storedVersion = this.storeProvider.loadSystemStore(this.config)
.storedVersion();
String storedVersion = store.storedVersion();
if (!driverVersion.equals(storedVersion)) {
LOG.error("The backend driver version '{}' is inconsistent with " +
"the data version '{}' of backend store for graph '{}'",

4
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendStoreProvider.java
View File

@ -73,4 +73,8 @@ public interface BackendStoreProvider {
void onCloneConfig(HugeConfig config, String newGraph);
void onDeleteConfig(HugeConfig config);
default boolean isHstore() {
return "hstore".equals(type());
}
}

8
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/memory/InMemoryDBStore.java
View File

@ -27,10 +27,6 @@ import org.apache.hugegraph.backend.LocalCounter;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.query.Query;
import org.apache.hugegraph.backend.serializer.TextBackendEntry;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.type.define.Action;
import org.slf4j.Logger;
import org.apache.hugegraph.backend.store.AbstractBackendStore;
import org.apache.hugegraph.backend.store.BackendAction;
import org.apache.hugegraph.backend.store.BackendEntry;
@ -39,7 +35,9 @@ import org.apache.hugegraph.backend.store.BackendMutation;
import org.apache.hugegraph.backend.store.BackendSession;
import org.apache.hugegraph.backend.store.BackendStoreProvider;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
/**
* NOTE:
@ -95,7 +93,7 @@ public abstract class InMemoryDBStore
@Override
protected final InMemoryDBTable table(HugeType type) {
assert type != null;
InMemoryDBTable table = this.tables.get(type);
InMemoryDBTable table = this.tables.get(convertTaskOrServerToVertex(type));
if (table == null) {
throw new BackendException("Unsupported table type: %s", type);
}

10
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java
View File

@ -191,7 +191,7 @@ public class GraphIndexTransaction extends AbstractTransaction {
* @param removed remove or add index
*/
protected void updateIndex(Id ilId, HugeElement element, boolean removed) {
SchemaTransaction schema = this.params().schemaTransaction();
ISchemaTransaction schema = this.params().schemaTransaction();
IndexLabel indexLabel = schema.getIndexLabel(ilId);
E.checkArgument(indexLabel != null,
"Not exist index label with id '%s'", ilId);
@ -730,7 +730,7 @@ public class GraphIndexTransaction extends AbstractTransaction {
@Watched(prefix = "index")
private Set<MatchedIndex> collectMatchedIndexes(ConditionQuery query) {
SchemaTransaction schema = this.params().schemaTransaction();
ISchemaTransaction schema = this.params().schemaTransaction();
Id label = query.condition(HugeKeys.LABEL);
List<? extends SchemaLabel> schemaLabels;
@ -780,7 +780,7 @@ public class GraphIndexTransaction extends AbstractTransaction {
@Watched(prefix = "index")
private MatchedIndex collectMatchedIndex(SchemaLabel schemaLabel,
ConditionQuery query) {
SchemaTransaction schema = this.params().schemaTransaction();
ISchemaTransaction schema = this.params().schemaTransaction();
Set<IndexLabel> ils = InsertionOrderUtil.newSet();
for (Id il : schemaLabel.indexLabels()) {
IndexLabel indexLabel = schema.getIndexLabel(il);
@ -1748,7 +1748,9 @@ public class GraphIndexTransaction extends AbstractTransaction {
HugeElement element) {
if (element.type() != HugeType.VERTEX &&
element.type() != HugeType.EDGE_OUT &&
element.type() != HugeType.EDGE_IN) {
element.type() != HugeType.EDGE_IN &&
element.type() != HugeType.TASK &&
element.type() != HugeType.SERVER) {
throw new HugeException("Only accept element of type VERTEX " +
"and EDGE to remove left index, " +
"but got: '%s'", element.type());

38
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java
View File

@ -734,16 +734,50 @@ public class GraphTransaction extends IndexableTransaction {
return vertex;
}
public Iterator<Vertex> queryTaskInfos(Query query) {
return this.queryVertices(query);
}
public Iterator<Vertex> queryTaskInfos(Object... vertexIds) {
if (this.graph().backendStoreFeatures().supportsTaskAndServerVertex()) {
return this.queryVerticesByIds(vertexIds, false, false,
HugeType.TASK);
}
return this.queryVerticesByIds(vertexIds, false, false,
HugeType.VERTEX);
}
public Iterator<Vertex> queryServerInfos(Query query) {
return this.queryVertices(query);
}
public Iterator<Vertex> queryServerInfos(Object... vertexIds) {
if (this.graph().backendStoreFeatures().supportsTaskAndServerVertex()) {
return this.queryVerticesByIds(vertexIds, false, false,
HugeType.SERVER);
}
return this.queryVerticesByIds(vertexIds, false, false,
HugeType.VERTEX);
}
protected Iterator<Vertex> queryVerticesByIds(Object[] vertexIds,
boolean adjacentVertex,
boolean checkMustExist) {
return this.queryVerticesByIds(vertexIds, adjacentVertex, checkMustExist,
HugeType.VERTEX);
}
protected Iterator<Vertex> queryVerticesByIds(Object[] vertexIds,
boolean adjacentVertex,
boolean checkMustExist) {
boolean checkMustExist,
HugeType type) {
Query.checkForceCapacity(vertexIds.length);
// NOTE: allowed duplicated vertices if query by duplicated ids
List<Id> ids = InsertionOrderUtil.newList();
Map<Id, HugeVertex> vertices = new HashMap<>(vertexIds.length);
IdQuery query = new IdQuery(HugeType.VERTEX);
IdQuery query = new IdQuery(type);
for (Object vertexId : vertexIds) {
HugeVertex vertex;
Id id = HugeVertex.getIdValue(vertexId);

92
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/ISchemaTransaction.java Normal file
View File

@ -0,0 +1,92 @@
package org.apache.hugegraph.backend.tx;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.schema.EdgeLabel;
import org.apache.hugegraph.schema.IndexLabel;
import org.apache.hugegraph.schema.PropertyKey;
import org.apache.hugegraph.schema.SchemaElement;
import org.apache.hugegraph.schema.SchemaLabel;
import org.apache.hugegraph.schema.VertexLabel;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.type.define.GraphMode;
import org.apache.hugegraph.type.define.SchemaStatus;
public interface ISchemaTransaction {
List<PropertyKey> getPropertyKeys();
Id removePropertyKey(Id pkey);
PropertyKey getPropertyKey(Id id);
PropertyKey getPropertyKey(String name);
Id clearOlapPk(PropertyKey propertyKey);
void addVertexLabel(VertexLabel label);
void updateVertexLabel(VertexLabel label);
Id removeVertexLabel(Id label);
List<VertexLabel> getVertexLabels();
VertexLabel getVertexLabel(Id id);
VertexLabel getVertexLabel(String name);
List<EdgeLabel> getEdgeLabels();
Id addPropertyKey(PropertyKey pkey);
void updatePropertyKey(PropertyKey pkey);
void updateEdgeLabel(EdgeLabel label);
void addEdgeLabel(EdgeLabel label);
Id removeEdgeLabel(Id id);
EdgeLabel getEdgeLabel(Id id);
EdgeLabel getEdgeLabel(String name);
void addIndexLabel(SchemaLabel schemaLabel, IndexLabel indexLabel);
void updateIndexLabel(IndexLabel label);
Id removeIndexLabel(Id id);
Id rebuildIndex(SchemaElement schema);
Id rebuildIndex(SchemaElement schema, Set<Id> dependencies);
List<IndexLabel> getIndexLabels();
IndexLabel getIndexLabel(Id id);
IndexLabel getIndexLabel(String name);
void close();
Id getNextId(HugeType type);
Id validOrGenerateId(HugeType type, Id id, String name);
void checkSchemaName(String name);
String graphName();
void updateSchemaStatus(SchemaElement element, SchemaStatus status);
GraphMode graphMode();
boolean existsSchemaId(HugeType type, Id id);
void removeIndexLabelFromBaseLabel(IndexLabel indexLabel);
void createIndexLabelForOlapPk(PropertyKey propertyKey);
}

142
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/IdCounter.java Normal file
View File

@ -0,0 +1,142 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.tx;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hugegraph.backend.BackendException;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.grpc.Pdpb;
import org.apache.hugegraph.store.term.HgPair;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.util.E;
public class IdCounter {
private static final int TIMES = 10000;
private static final int DELTA = 10000;
private static final String DELIMITER = "/";
private static final Map<String, HgPair<AtomicLong, AtomicLong>> ids =
new ConcurrentHashMap<>();
private final PDClient pdClient;
private final String graphName;
public IdCounter(PDClient pdClient, String graphName) {
this.graphName = graphName;
this.pdClient = pdClient;
}
public Id nextId(HugeType type) {
long counter = this.getCounter(type);
E.checkState(counter != 0L, "Please check whether '%s' is OK",
this.pdClient.toString());
return IdGenerator.of(counter);
}
public void setCounterLowest(HugeType type, long lowest) {
long current = this.getCounter(type);
if (current >= lowest) {
return;
}
long increment = lowest - current;
this.increaseCounter(type, increment);
}
public long getCounter(HugeType type) {
return this.getCounterFromPd(type);
}
public synchronized void increaseCounter(HugeType type, long lowest) {
String key = toKey(this.graphName, type);
getCounterFromPd(type);
HgPair<AtomicLong, AtomicLong> idPair = ids.get(key);
AtomicLong currentId = idPair.getKey();
AtomicLong maxId = idPair.getValue();
if (currentId.longValue() >= lowest) {
return;
}
if (maxId.longValue() >= lowest) {
currentId.set(lowest);
return;
}
synchronized (ids) {
try {
this.pdClient.getIdByKey(key, (int) (lowest - maxId.longValue()));
ids.remove(key);
} catch (Exception e) {
throw new BackendException(e);
}
}
}
protected String toKey(String graphName, HugeType type) {
return new StringBuilder().append(graphName)
.append(DELIMITER)
.append(type.code()).toString();
}
public long getCounterFromPd(HugeType type) {
AtomicLong currentId;
AtomicLong maxId;
HgPair<AtomicLong, AtomicLong> idPair;
String key = toKey(this.graphName, type);
if ((idPair = ids.get(key)) == null) {
synchronized (ids) {
if ((idPair = ids.get(key)) == null) {
try {
currentId = new AtomicLong(0);
maxId = new AtomicLong(0);
idPair = new HgPair<>(currentId, maxId);
ids.put(key, idPair);
} catch (Exception e) {
throw new BackendException(String.format(
"Failed to get the ID from pd,%s", e));
}
}
}
}
currentId = idPair.getKey();
maxId = idPair.getValue();
for (int i = 0; i < TIMES; i++) {
synchronized (currentId) {
if ((currentId.incrementAndGet()) <= maxId.longValue()) {
return currentId.longValue();
}
if (currentId.longValue() > maxId.longValue()) {
try {
Pdpb.GetIdResponse idByKey = pdClient.getIdByKey(key, DELTA);
idPair.getValue().getAndSet(idByKey.getId() +
idByKey.getDelta());
idPair.getKey().getAndSet(idByKey.getId());
} catch (Exception e) {
throw new BackendException(String.format(
"Failed to get the ID from pd,%s", e));
}
}
}
}
E.checkArgument(false,
"Having made too many attempts to get the" +
" ID for type '%s'", type.name());
return 0L;
}
}

2
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/SchemaTransaction.java
View File

@ -68,7 +68,7 @@ import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.LockUtil;
import com.google.common.collect.ImmutableSet;
public class SchemaTransaction extends IndexableTransaction {
public class SchemaTransaction extends IndexableTransaction implements ISchemaTransaction {
private final SchemaIndexTransaction indexTx;
private final SystemSchemaStore systemSchemaStore;

735
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/SchemaTransactionV2.java Normal file
View File

@ -0,0 +1,735 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.tx;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.HugeGraphParams;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.config.CoreOptions;
import org.apache.hugegraph.exception.NotAllowException;
import org.apache.hugegraph.job.JobBuilder;
import org.apache.hugegraph.job.schema.EdgeLabelRemoveJob;
import org.apache.hugegraph.job.schema.IndexLabelRebuildJob;
import org.apache.hugegraph.job.schema.IndexLabelRemoveJob;
import org.apache.hugegraph.job.schema.OlapPropertyKeyClearJob;
import org.apache.hugegraph.job.schema.OlapPropertyKeyCreateJob;
import org.apache.hugegraph.job.schema.OlapPropertyKeyRemoveJob;
import org.apache.hugegraph.job.schema.SchemaJob;
import org.apache.hugegraph.job.schema.VertexLabelRemoveJob;
import org.apache.hugegraph.meta.MetaDriver;
import org.apache.hugegraph.meta.MetaManager;
import org.apache.hugegraph.meta.PdMetaDriver;
import org.apache.hugegraph.meta.managers.SchemaMetaManager;
import org.apache.hugegraph.perf.PerfUtil.Watched;
import org.apache.hugegraph.schema.EdgeLabel;
import org.apache.hugegraph.schema.IndexLabel;
import org.apache.hugegraph.schema.PropertyKey;
import org.apache.hugegraph.schema.SchemaElement;
import org.apache.hugegraph.schema.SchemaLabel;
import org.apache.hugegraph.schema.Userdata;
import org.apache.hugegraph.schema.VertexLabel;
import org.apache.hugegraph.task.HugeTask;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.type.define.GraphMode;
import org.apache.hugegraph.type.define.SchemaStatus;
import org.apache.hugegraph.type.define.WriteType;
import org.apache.hugegraph.util.DateUtil;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.LockUtil;
import org.apache.hugegraph.util.Log;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.slf4j.Logger;
import com.google.common.collect.ImmutableSet;
public class SchemaTransactionV2 implements ISchemaTransaction {
protected static final Logger LOG = Log.logger(SchemaTransaction.class);
private final String graphSpace;
private final String graph;
private final HugeGraphParams graphParams;
private final IdCounter idCounter;
private final SchemaMetaManager schemaMetaManager;
public SchemaTransactionV2(MetaDriver metaDriver,
String cluster,
HugeGraphParams graphParams) {
E.checkNotNull(graphParams, "graphParams");
this.graphParams = graphParams;
// TODO: uncomment later - graph space
//this.graphSpace = graphParams.graph().graphSpace();
this.graphSpace = "";
this.graph = graphParams.name();
this.schemaMetaManager =
new SchemaMetaManager(metaDriver, cluster, this.graph());
this.idCounter = new IdCounter(((PdMetaDriver) metaDriver).pdClient(),
idKeyName(this.graphSpace, this.graph));
}
private static void setCreateTimeIfNeeded(SchemaElement schema) {
if (!schema.userdata().containsKey(Userdata.CREATE_TIME)) {
schema.userdata(Userdata.CREATE_TIME, DateUtil.now());
}
}
/**
* 异步任务系列
*/
private static Id asyncRun(HugeGraph graph, SchemaElement schema,
SchemaJob job) {
return asyncRun(graph, schema, job, ImmutableSet.of());
}
@Watched(prefix = "schema")
private static Id asyncRun(HugeGraph graph, SchemaElement schema,
SchemaJob job, Set<Id> dependencies) {
E.checkArgument(schema != null, "Schema can't be null");
String name = SchemaJob.formatTaskName(schema.type(),
schema.id(),
schema.name());
JobBuilder<Object> builder = JobBuilder.of(graph).name(name)
.job(job)
.dependencies(dependencies);
HugeTask<?> task = builder.schedule();
// If TASK_SYNC_DELETION is true, wait async thread done before
// continue. This is used when running tests.
if (graph.option(CoreOptions.TASK_SYNC_DELETION)) {
task.syncWait();
}
return task.id();
}
public String idKeyName(String graphSpace, String graph) {
// {graphSpace}/{graph}/m "m" means "schema"
return String.join("/", graphSpace, graph, "m");
}
@Watched(prefix = "schema")
public List<PropertyKey> getPropertyKeys(boolean cache) {
return this.getAllSchema(HugeType.PROPERTY_KEY);
}
@Watched(prefix = "schema")
public List<PropertyKey> getPropertyKeys() {
return this.getAllSchema(HugeType.PROPERTY_KEY);
}
@Watched(prefix = "schema")
public List<VertexLabel> getVertexLabels() {
return this.getAllSchema(HugeType.VERTEX_LABEL);
}
@Watched(prefix = "schema")
public List<EdgeLabel> getEdgeLabels() {
return this.getAllSchema(HugeType.EDGE_LABEL);
}
@Watched(prefix = "schema")
public List<IndexLabel> getIndexLabels() {
return this.getAllSchema(HugeType.INDEX_LABEL);
}
@Watched(prefix = "schema")
public Id addPropertyKey(PropertyKey propertyKey) {
this.addSchema(propertyKey);
if (!propertyKey.olap()) {
return IdGenerator.ZERO;
}
return this.createOlapPk(propertyKey);
}
@Watched(prefix = "schema")
public void updatePropertyKey(PropertyKey propertyKey) {
this.updateSchema(propertyKey, null);
}
public void updatePropertyKey(PropertyKey old, PropertyKey update) {
this.removePropertyKey(old.id());
this.addPropertyKey(update);
}
@Watched(prefix = "schema")
public PropertyKey getPropertyKey(Id id) {
E.checkArgumentNotNull(id, "Property key id can't be null");
return this.getSchema(HugeType.PROPERTY_KEY, id);
}
@Watched(prefix = "schema")
public PropertyKey getPropertyKey(String name) {
E.checkArgumentNotNull(name, "Property key name can't be null");
E.checkArgument(!name.isEmpty(), "Property key name can't be empty");
return this.getSchema(HugeType.PROPERTY_KEY, name);
}
@Watched(prefix = "schema")
public Id removePropertyKey(Id id) {
LOG.debug("SchemaTransaction remove property key '{}'", id);
PropertyKey propertyKey = this.getPropertyKey(id);
// If the property key does not exist, return directly
if (propertyKey == null) {
return null;
}
List<VertexLabel> vertexLabels = this.getVertexLabels();
for (VertexLabel vertexLabel : vertexLabels) {
if (vertexLabel.properties().contains(id)) {
throw new NotAllowException(
"Not allowed to remove property key: '%s' " +
"because the vertex label '%s' is still using it.",
propertyKey, vertexLabel.name());
}
}
List<EdgeLabel> edgeLabels = this.getEdgeLabels();
for (EdgeLabel edgeLabel : edgeLabels) {
if (edgeLabel.properties().contains(id)) {
throw new NotAllowException(
"Not allowed to remove property key: '%s' " +
"because the edge label '%s' is still using it.",
propertyKey, edgeLabel.name());
}
}
if (propertyKey.oltp()) {
this.removeSchema(propertyKey);
return IdGenerator.ZERO;
} else {
return this.removeOlapPk(propertyKey);
}
}
@Watched(prefix = "schema")
public void addVertexLabel(VertexLabel vertexLabel) {
this.addSchema(vertexLabel);
}
@Watched(prefix = "schema")
public void updateVertexLabel(VertexLabel vertexLabel) {
this.updateSchema(vertexLabel, null);
}
@Watched(prefix = "schema")
public VertexLabel getVertexLabel(Id id) {
E.checkArgumentNotNull(id, "Vertex label id can't be null");
if (SchemaElement.OLAP_ID.equals(id)) {
return VertexLabel.OLAP_VL;
}
return this.getSchema(HugeType.VERTEX_LABEL, id);
}
@Watched(prefix = "schema")
public VertexLabel getVertexLabel(String name) {
E.checkArgumentNotNull(name, "Vertex label name can't be null");
E.checkArgument(!name.isEmpty(), "Vertex label name can't be empty");
if (SchemaElement.OLAP.equals(name)) {
return VertexLabel.OLAP_VL;
}
return this.getSchema(HugeType.VERTEX_LABEL, name);
}
@Watched(prefix = "schema")
public Id removeVertexLabel(Id id) {
LOG.debug("SchemaTransaction remove vertex label '{}'", id);
SchemaJob job = new VertexLabelRemoveJob();
VertexLabel schema = this.getVertexLabel(id);
return asyncRun(this.graph(), schema, job);
}
@Watched(prefix = "schema")
public void addEdgeLabel(EdgeLabel edgeLabel) {
this.addSchema(edgeLabel);
}
@Watched(prefix = "schema")
public void updateEdgeLabel(EdgeLabel edgeLabel) {
this.updateSchema(edgeLabel, null);
}
@Watched(prefix = "schema")
public EdgeLabel getEdgeLabel(Id id) {
E.checkArgumentNotNull(id, "Edge label id can't be null");
return this.getSchema(HugeType.EDGE_LABEL, id);
}
@Watched(prefix = "schema")
public EdgeLabel getEdgeLabel(String name) {
E.checkArgumentNotNull(name, "Edge label name can't be null");
E.checkArgument(!name.isEmpty(), "Edge label name can't be empty");
return this.getSchema(HugeType.EDGE_LABEL, name);
}
@Watched(prefix = "schema")
public Id removeEdgeLabel(Id id) {
/*
* Call an asynchronous task and call back the corresponding
* removeSchema() method after the task ends to complete the delete
* schema operation
*/
LOG.debug("SchemaTransaction remove edge label '{}'", id);
EdgeLabel schema = this.getEdgeLabel(id);
// TODO: uncomment later - el
//if (schema.edgeLabelType().parent()) {
// List<EdgeLabel> edgeLabels = this.getEdgeLabels();
// for (EdgeLabel edgeLabel : edgeLabels) {
// if (edgeLabel.edgeLabelType().sub() &&
// edgeLabel.fatherId() == id) {
// throw new NotAllowException(
// "Not allowed to remove a parent edge label: '%s' " +
// "because the sub edge label '%s' is still existing",
// schema.name(), edgeLabel.name());
// }
// }
//}
SchemaJob job = new EdgeLabelRemoveJob();
return asyncRun(this.graph(), schema, job);
}
@Watched(prefix = "schema")
public void addIndexLabel(SchemaLabel baseLabel, IndexLabel indexLabel) {
/*
* Create index and update index name in base-label(VL/EL)
* TODO: should wrap update base-label and create index in one tx.
*/
this.addSchema(indexLabel);
if (baseLabel.equals(VertexLabel.OLAP_VL)) {
return;
}
this.updateSchema(baseLabel, schema -> {
// NOTE: Do schema update in the lock block
baseLabel.addIndexLabel(indexLabel.id());
});
}
@Watched(prefix = "schema")
public void updateIndexLabel(IndexLabel indexLabel) {
this.updateSchema(indexLabel, null);
}
@Watched(prefix = "schema")
public IndexLabel getIndexLabel(Id id) {
E.checkArgumentNotNull(id, "Index label id can't be null");
return this.getSchema(HugeType.INDEX_LABEL, id);
}
@Watched(prefix = "schema")
public IndexLabel getIndexLabel(String name) {
E.checkArgumentNotNull(name, "Index label name can't be null");
E.checkArgument(!name.isEmpty(), "Index label name can't be empty");
return this.getSchema(HugeType.INDEX_LABEL, name);
}
@Override
public void close() {
}
@Watched(prefix = "schema")
public Id removeIndexLabel(Id id) {
LOG.debug("SchemaTransaction remove index label '{}'", id);
SchemaJob job = new IndexLabelRemoveJob();
IndexLabel schema = this.getIndexLabel(id);
return asyncRun(this.graph(), schema, job);
}
// 通用性 的schema处理函数
@Watched(prefix = "schema")
public void updateSchemaStatus(SchemaElement schema, SchemaStatus status) {
if (!this.existsSchemaId(schema.type(), schema.id())) {
LOG.warn("Can't update schema '{}', it may be deleted", schema);
return;
}
this.updateSchema(schema, schemaToUpdate -> {
// NOTE: Do schema update in the lock block
schema.status(status);
});
}
@Watched(prefix = "schema")
public boolean existsSchemaId(HugeType type, Id id) {
return this.getSchema(type, id) != null;
}
@Override
public void removeIndexLabelFromBaseLabel(IndexLabel indexLabel) {
}
protected void updateSchema(SchemaElement schema,
Consumer<SchemaElement> updateCallback) {
LOG.debug("SchemaTransaction update {} with id '{}'",
schema.type(), schema.id());
this.saveSchema(schema, true, updateCallback);
}
protected void addSchema(SchemaElement schema) {
LOG.debug("SchemaTransaction add {} with id '{}'",
schema.type(), schema.id());
setCreateTimeIfNeeded(schema);
this.saveSchema(schema, false, null);
}
@SuppressWarnings("unchecked")
private void saveSchema(SchemaElement schema, boolean update,
Consumer<SchemaElement> updateCallback) {
// Lock for schema update
// TODO: uncomment later - graph space
//String spaceGraph = this.graphParams()
// .graph().spaceGraphName();
LockUtil.Locks locks = new LockUtil.Locks(graph);
try {
locks.lockWrites(LockUtil.hugeType2Group(schema.type()), schema.id());
if (updateCallback != null) {
// NOTE: Do schema update in the lock block
updateCallback.accept(schema);
}
// 调对应的方法
switch (schema.type()) {
case PROPERTY_KEY:
this.schemaMetaManager.addPropertyKey(this.graphSpace,
this.graph,
(PropertyKey) schema);
break;
case VERTEX_LABEL:
this.schemaMetaManager.addVertexLabel(this.graphSpace,
this.graph,
(VertexLabel) schema);
// 点的label发生变化, 清空对应图的点缓存信息
MetaManager.instance().notifyGraphVertexCacheClear(this.graphSpace, this.graph);
break;
case EDGE_LABEL:
this.schemaMetaManager.addEdgeLabel(this.graphSpace,
this.graph,
(EdgeLabel) schema);
// 边的label发生变化, 清空对应图的边缓存信息
MetaManager.instance().notifyGraphEdgeCacheClear(this.graphSpace, this.graph);
break;
case INDEX_LABEL:
this.schemaMetaManager.addIndexLabel(this.graphSpace,
this.graph,
(IndexLabel) schema);
break;
default:
throw new AssertionError(String.format(
"Invalid key '%s' for saveSchema", schema.type()));
}
} finally {
locks.unlock();
}
}
@SuppressWarnings("unchecked")
protected <T extends SchemaElement> T getSchema(HugeType type, Id id) {
LOG.debug("SchemaTransaction get {} by id '{}'",
type.readableName(), id);
switch (type) {
case PROPERTY_KEY:
return (T) this.schemaMetaManager.getPropertyKey(this.graphSpace,
this.graph, id);
case VERTEX_LABEL:
return (T) this.schemaMetaManager.getVertexLabel(this.graphSpace,
this.graph, id);
case EDGE_LABEL:
return (T) this.schemaMetaManager.getEdgeLabel(this.graphSpace,
this.graph, id);
case INDEX_LABEL:
return (T) this.schemaMetaManager.getIndexLabel(this.graphSpace,
this.graph, id);
default:
throw new AssertionError(String.format(
"Invalid type '%s' for getSchema", type));
}
}
/**
* Currently doesn't allow to exist schema with the same name
*
* @param type the query schema type
* @param name the query schema name
* @param <T> SubClass of SchemaElement
* @return the queried schema object
*/
@SuppressWarnings("unchecked")
protected <T extends SchemaElement> T getSchema(HugeType type, String name) {
LOG.debug("SchemaTransaction get {} by name '{}'",
type.readableName(), name);
switch (type) {
case PROPERTY_KEY:
return (T) this.schemaMetaManager.getPropertyKey(this.graphSpace,
this.graph, name);
case VERTEX_LABEL:
return (T) this.schemaMetaManager.getVertexLabel(this.graphSpace,
this.graph, name);
case EDGE_LABEL:
return (T) this.schemaMetaManager.getEdgeLabel(this.graphSpace,
this.graph, name);
case INDEX_LABEL:
return (T) this.schemaMetaManager.getIndexLabel(this.graphSpace,
this.graph, name);
default:
throw new AssertionError(String.format(
"Invalid type '%s' for getSchema", type));
}
}
@SuppressWarnings("unchecked")
protected <T extends SchemaElement> List<T> getAllSchema(HugeType type) {
LOG.debug("SchemaTransaction getAllSchema {}", type.readableName());
switch (type) {
case PROPERTY_KEY:
return (List<T>) this.schemaMetaManager.getPropertyKeys(this.graphSpace,
this.graph);
case VERTEX_LABEL:
return (List<T>) this.schemaMetaManager.getVertexLabels(this.graphSpace,
this.graph);
case EDGE_LABEL:
return (List<T>) this.schemaMetaManager.getEdgeLabels(this.graphSpace, this.graph);
case INDEX_LABEL:
return (List<T>) this.schemaMetaManager.getIndexLabels(this.graphSpace, this.graph);
default:
throw new AssertionError(String.format(
"Invalid type '%s' for getSchema", type));
}
}
protected void removeSchema(SchemaElement schema) {
LOG.debug("SchemaTransaction remove {} by id '{}'",
schema.type(), schema.id());
// TODO: uncomment later - graph space
//String spaceGraph = this.graphParams()
// .graph().spaceGraphName();
LockUtil.Locks locks = new LockUtil.Locks(graph);
try {
locks.lockWrites(LockUtil.hugeType2Group(schema.type()),
schema.id());
switch (schema.type()) {
case PROPERTY_KEY:
this.schemaMetaManager.removePropertyKey(this.graphSpace, this.graph,
schema.id());
break;
case VERTEX_LABEL:
this.schemaMetaManager.removeVertexLabel(this.graphSpace, this.graph,
schema.id());
break;
case EDGE_LABEL:
this.schemaMetaManager.removeEdgeLabel(this.graphSpace, this.graph,
schema.id());
break;
case INDEX_LABEL:
this.schemaMetaManager.removeIndexLabel(this.graphSpace, this.graph,
schema.id());
break;
default:
throw new AssertionError(String.format(
"Invalid key '%s' for saveSchema", schema.type()));
}
} finally {
locks.unlock();
}
}
// olap 相关的方法
public void createIndexLabelForOlapPk(PropertyKey propertyKey) {
WriteType writeType = propertyKey.writeType();
if (writeType == WriteType.OLTP ||
writeType == WriteType.OLAP_COMMON) {
return;
}
String indexName = SchemaElement.OLAP + "_by_" + propertyKey.name();
IndexLabel.Builder builder = this.graph().schema()
.indexLabel(indexName)
.onV(SchemaElement.OLAP)
.by(propertyKey.name());
if (propertyKey.writeType() == WriteType.OLAP_SECONDARY) {
builder.secondary();
} else {
assert propertyKey.writeType() == WriteType.OLAP_RANGE;
builder.range();
}
builder.build();
this.graph().addIndexLabel(VertexLabel.OLAP_VL, builder.build());
}
public Id removeOlapPk(PropertyKey propertyKey) {
LOG.debug("SchemaTransaction remove olap property key {} with id '{}'",
propertyKey.name(), propertyKey.id());
SchemaJob job = new OlapPropertyKeyRemoveJob();
return asyncRun(this.graph(), propertyKey, job);
}
public void removeOlapPk(Id id) {
this.graphParams().loadGraphStore().removeOlapTable(id);
}
public Id clearOlapPk(PropertyKey propertyKey) {
LOG.debug("SchemaTransaction clear olap property key {} with id '{}'",
propertyKey.name(), propertyKey.id());
SchemaJob job = new OlapPropertyKeyClearJob();
return asyncRun(this.graph(), propertyKey, job);
}
public void clearOlapPk(Id id) {
this.graphParams().loadGraphStore().clearOlapTable(id);
}
public Id createOlapPk(PropertyKey propertyKey) {
LOG.debug("SchemaTransaction create olap property key {} with id '{}'",
propertyKey.name(), propertyKey.id());
SchemaJob job = new OlapPropertyKeyCreateJob();
return asyncRun(this.graph(), propertyKey, job);
}
// -- store 相关的方法，分为两类：1、olaptable相关 2、id生成策略
// - 1、olaptable相关
public void createOlapPk(Id id) {
this.graphParams().loadGraphStore().createOlapTable(id);
}
// TODO: uncomment later - olap
//public boolean existOlapTable(Id id) {
// return this.graphParams().loadGraphStore().existOlapTable(id);
//}
public void initAndRegisterOlapTables() {
for (PropertyKey pk : this.getPropertyKeys()) {
if (pk.olap()) {
this.graphParams().loadGraphStore().checkAndRegisterOlapTable(pk.id());
}
}
}
// - 2、id生成策略
@Watched(prefix = "schema")
public Id getNextId(HugeType type) {
LOG.debug("SchemaTransaction get next id for {}", type);
return this.idCounter.nextId(type);
}
@Watched(prefix = "schema")
public void setNextIdLowest(HugeType type, long lowest) {
LOG.debug("SchemaTransaction set next id to {} for {}", lowest, type);
this.idCounter.setCounterLowest(type, lowest);
}
@Watched(prefix = "schema")
public Id getNextSystemId() {
LOG.debug("SchemaTransaction get next system id");
Id id = this.idCounter.nextId(HugeType.SYS_SCHEMA);
return IdGenerator.of(-id.asLong());
}
@Watched(prefix = "schema")
public Id validOrGenerateId(HugeType type, Id id, String name) {
boolean forSystem = Graph.Hidden.isHidden(name);
if (id != null) {
this.checkIdAndUpdateNextId(type, id, name, forSystem);
} else {
if (forSystem) {
id = this.getNextSystemId();
} else {
id = this.getNextId(type);
}
}
return id;
}
private void checkIdAndUpdateNextId(HugeType type, Id id,
String name, boolean forSystem) {
if (forSystem) {
if (id.number() && id.asLong() < 0) {
return;
}
throw new IllegalStateException(String.format(
"Invalid system id '%s'", id));
}
E.checkState(id.number() && id.asLong() > 0L,
"Schema id must be number and >0, but got '%s'", id);
GraphMode mode = this.graphMode();
E.checkState(mode == GraphMode.RESTORING,
"Can't build schema with provided id '%s' " +
"when graph '%s' in mode '%s'", id, this.graph, mode);
this.setNextIdLowest(type, id.asLong());
}
// 功能型函数
public void checkSchemaName(String name) {
String illegalReg = this.graphParams().configuration()
.get(CoreOptions.SCHEMA_ILLEGAL_NAME_REGEX);
E.checkNotNull(name, "name");
E.checkArgument(!name.isEmpty(), "The name can't be empty.");
E.checkArgument(name.length() < 256,
"The length of name must less than 256 bytes.");
E.checkArgument(!name.matches(illegalReg),
"Illegal schema name '%s'", name);
final char[] filters = {'#', '>', ':', '!'};
for (char c : filters) {
E.checkArgument(name.indexOf(c) == -1,
"The name can't contain character '%s'.", c);
}
}
@Override
public String graphName() {
return this.graph;
}
protected HugeGraphParams graphParams() {
return this.graphParams;
}
public GraphMode graphMode() {
return this.graphParams().mode();
}
// 获取字段的方法
public HugeGraph graph() {
return this.graphParams.graph();
}
// 重建索引
@Watched(prefix = "schema")
public Id rebuildIndex(SchemaElement schema) {
return this.rebuildIndex(schema, ImmutableSet.of());
}
@Watched(prefix = "schema")
public Id rebuildIndex(SchemaElement schema, Set<Id> dependencies) {
LOG.debug("SchemaTransaction rebuild index for {} with id '{}'",
schema.type(), schema.id());
SchemaJob job = new IndexLabelRebuildJob();
return asyncRun(this.graph(), schema, job, dependencies);
}
/**
* 清除所有的schema信息
*/
public void clear() {
this.schemaMetaManager.clearAllSchema(this.graphSpace, graph);
}
}

3
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/job/schema/EdgeLabelRemoveJob.java
View File

@ -21,6 +21,7 @@ import java.util.Set;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.tx.GraphTransaction;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.schema.EdgeLabel;
import org.apache.hugegraph.type.define.SchemaStatus;
@ -43,7 +44,7 @@ public class EdgeLabelRemoveJob extends SchemaJob {
private static void removeEdgeLabel(HugeGraphParams graph, Id id) {
GraphTransaction graphTx = graph.graphTransaction();
SchemaTransaction schemaTx = graph.schemaTransaction();
ISchemaTransaction schemaTx = graph.schemaTransaction();
EdgeLabel edgeLabel = schemaTx.getEdgeLabel(id);
// If the edge label does not exist, return directly
if (edgeLabel == null) {

5
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/job/schema/IndexLabelRebuildJob.java
View File

@ -24,6 +24,7 @@ import java.util.stream.Collectors;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.tx.GraphTransaction;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.type.define.SchemaStatus;
@ -83,7 +84,7 @@ public class IndexLabelRebuildJob extends SchemaJob {
}
private void rebuildIndex(SchemaLabel label, Collection<Id> indexLabelIds) {
SchemaTransaction schemaTx = this.params().schemaTransaction();
ISchemaTransaction schemaTx = this.params().schemaTransaction();
GraphTransaction graphTx = this.params().graphTransaction();
Consumer<?> indexUpdater = (elem) -> {
@ -148,7 +149,7 @@ public class IndexLabelRebuildJob extends SchemaJob {
}
private void removeIndex(Collection<Id> indexLabelIds) {
SchemaTransaction schemaTx = this.params().schemaTransaction();
ISchemaTransaction schemaTx = this.params().schemaTransaction();
GraphTransaction graphTx = this.params().graphTransaction();
for (Id id : indexLabelIds) {

3
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/job/schema/IndexLabelRemoveJob.java
View File

@ -19,6 +19,7 @@ package org.apache.hugegraph.job.schema;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.tx.GraphTransaction;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.schema.IndexLabel;
import org.apache.hugegraph.type.define.SchemaStatus;
@ -40,7 +41,7 @@ public class IndexLabelRemoveJob extends SchemaJob {
protected static void removeIndexLabel(HugeGraphParams graph, Id id) {
GraphTransaction graphTx = graph.graphTransaction();
SchemaTransaction schemaTx = graph.schemaTransaction();
ISchemaTransaction schemaTx = graph.schemaTransaction();
IndexLabel indexLabel = schemaTx.getIndexLabel(id);
// If the index label does not exist, return directly
if (indexLabel == null) {

5
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/job/schema/OlapPropertyKeyClearJob.java
View File

@ -19,6 +19,7 @@ package org.apache.hugegraph.job.schema;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.tx.GraphTransaction;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.schema.IndexLabel;
import org.apache.hugegraph.type.define.SchemaStatus;
@ -50,7 +51,7 @@ public class OlapPropertyKeyClearJob extends IndexLabelRemoveJob {
return;
}
GraphTransaction graphTx = graph.graphTransaction();
SchemaTransaction schemaTx = graph.schemaTransaction();
ISchemaTransaction schemaTx = graph.schemaTransaction();
IndexLabel indexLabel = schemaTx.getIndexLabel(olapIndexLabel);
// If the index label does not exist, return directly
if (indexLabel == null) {
@ -80,7 +81,7 @@ public class OlapPropertyKeyClearJob extends IndexLabelRemoveJob {
}
protected static Id findOlapIndexLabel(HugeGraphParams graph, Id olap) {
SchemaTransaction schemaTx = graph.schemaTransaction();
ISchemaTransaction schemaTx = graph.schemaTransaction();
for (IndexLabel indexLabel : schemaTx.getIndexLabels()) {
if (indexLabel.indexFields().contains(olap)) {
return indexLabel.id();

3
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/job/schema/OlapPropertyKeyCreateJob.java
View File

@ -17,6 +17,7 @@
package org.apache.hugegraph.job.schema;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.schema.PropertyKey;
@ -29,7 +30,7 @@ public class OlapPropertyKeyCreateJob extends SchemaJob {
@Override
public Object execute() {
SchemaTransaction schemaTx = this.params().schemaTransaction();
ISchemaTransaction schemaTx = this.params().schemaTransaction();
PropertyKey propertyKey = schemaTx.getPropertyKey(this.schemaId());
// Create olap index label schema
schemaTx.createIndexLabelForOlapPk(propertyKey);

3
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/job/schema/OlapPropertyKeyRemoveJob.java
View File

@ -18,6 +18,7 @@
package org.apache.hugegraph.job.schema;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.schema.PropertyKey;
@ -42,7 +43,7 @@ public class OlapPropertyKeyRemoveJob extends OlapPropertyKeyClearJob {
}
// Remove olap property key
SchemaTransaction schemaTx = this.params().schemaTransaction();
ISchemaTransaction schemaTx = this.params().schemaTransaction();
PropertyKey propertyKey = schemaTx.getPropertyKey(olap);
removeSchema(schemaTx, propertyKey);
return null;

3
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/job/schema/SchemaJob.java
View File

@ -22,6 +22,7 @@ import java.lang.reflect.Method;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.job.SysJob;
import org.apache.hugegraph.schema.SchemaElement;
@ -85,7 +86,7 @@ public abstract class SchemaJob extends SysJob<Object> {
* @param tx The remove operation actual executer
* @param schema the schema to be removed
*/
protected static void removeSchema(SchemaTransaction tx,
protected static void removeSchema(ISchemaTransaction tx,
SchemaElement schema) {
try {
Method method = SchemaTransaction.class

3
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/job/schema/VertexLabelRemoveJob.java
View File

@ -22,6 +22,7 @@ import java.util.Set;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.tx.GraphTransaction;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.schema.EdgeLabel;
import org.apache.hugegraph.schema.VertexLabel;
@ -46,7 +47,7 @@ public class VertexLabelRemoveJob extends SchemaJob {
private static void removeVertexLabel(HugeGraphParams graph, Id id) {
GraphTransaction graphTx = graph.graphTransaction();
SchemaTransaction schemaTx = graph.schemaTransaction();
ISchemaTransaction schemaTx = graph.schemaTransaction();
VertexLabel vertexLabel = schemaTx.getVertexLabel(id);
// If the vertex label does not exist, return directly
if (vertexLabel == null) {

322
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/EtcdMetaDriver.java Normal file
View File

@ -0,0 +1,322 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta;
import java.io.File;
import java.net.URI;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.function.Consumer;
import org.apache.commons.io.FileUtils;
import org.apache.hugegraph.HugeException;
import org.apache.hugegraph.meta.lock.EtcdDistributedLock;
import org.apache.hugegraph.meta.lock.LockResult;
import org.apache.hugegraph.type.define.CollectionType;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.collection.CollectionFactory;
import com.google.common.base.Strings;
import io.etcd.jetcd.ByteSequence;
import io.etcd.jetcd.Client;
import io.etcd.jetcd.ClientBuilder;
import io.etcd.jetcd.KV;
import io.etcd.jetcd.KeyValue;
import io.etcd.jetcd.kv.GetResponse;
import io.etcd.jetcd.lease.LeaseKeepAliveResponse;
import io.etcd.jetcd.options.DeleteOption;
import io.etcd.jetcd.options.GetOption;
import io.etcd.jetcd.options.WatchOption;
import io.etcd.jetcd.watch.WatchEvent;
import io.etcd.jetcd.watch.WatchResponse;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.ApplicationProtocolNames;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
public class EtcdMetaDriver implements MetaDriver {
private final Client client;
private final EtcdDistributedLock lock;
public EtcdMetaDriver(String trustFile, String clientCertFile,
String clientKeyFile, Object... endpoints) {
ClientBuilder builder = this.etcdMetaDriverBuilder(endpoints);
SslContext sslContext = openSslContext(trustFile, clientCertFile,
clientKeyFile);
this.client = builder.sslContext(sslContext).build();
this.lock = EtcdDistributedLock.getInstance(this.client);
}
public EtcdMetaDriver(Object... endpoints) {
ClientBuilder builder = this.etcdMetaDriverBuilder(endpoints);
this.client = builder.build();
this.lock = EtcdDistributedLock.getInstance(this.client);
}
private static ByteSequence toByteSequence(String content) {
return ByteSequence.from(content.getBytes());
}
private static boolean isEtcdPut(WatchEvent event) {
return event.getEventType() == WatchEvent.EventType.PUT;
}
public static SslContext openSslContext(String trustFile,
String clientCertFile,
String clientKeyFile) {
SslContext ssl;
try {
File trustManagerFile = FileUtils.getFile(trustFile);
File keyCertChainFile = FileUtils.getFile(clientCertFile);
File KeyFile = FileUtils.getFile(clientKeyFile);
ApplicationProtocolConfig alpn = new ApplicationProtocolConfig(
ApplicationProtocolConfig.Protocol.ALPN,
ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE,
ApplicationProtocolConfig.SelectedListenerFailureBehavior
.ACCEPT,
ApplicationProtocolNames.HTTP_2);
ssl = SslContextBuilder.forClient()
.applicationProtocolConfig(alpn)
.sslProvider(SslProvider.OPENSSL)
.trustManager(trustManagerFile)
.keyManager(keyCertChainFile, KeyFile)
.build();
} catch (Exception e) {
throw new HugeException("Failed to open ssl context", e);
}
return ssl;
}
public ClientBuilder etcdMetaDriverBuilder(Object... endpoints) {
int length = endpoints.length;
ClientBuilder builder = null;
if (endpoints[0] instanceof List && endpoints.length == 1) {
builder = Client.builder()
.endpoints(((List<String>) endpoints[0])
.toArray(new String[0]));
} else if (endpoints[0] instanceof String) {
for (int i = 1; i < length; i++) {
E.checkArgument(endpoints[i] instanceof String,
"Inconsistent endpoint %s(%s) with %s(%s)",
endpoints[i], endpoints[i].getClass(),
endpoints[0], endpoints[0].getClass());
}
builder = Client.builder().endpoints((String[]) endpoints);
} else if (endpoints[0] instanceof URI) {
for (int i = 1; i < length; i++) {
E.checkArgument(endpoints[i] instanceof String,
"Invalid endpoint %s(%s)",
endpoints[i], endpoints[i].getClass(),
endpoints[0], endpoints[0].getClass());
}
builder = Client.builder().endpoints((URI[]) endpoints);
} else {
E.checkArgument(false, "Invalid endpoint %s(%s)",
endpoints[0], endpoints[0].getClass());
}
return builder;
}
@Override
public long keepAlive(String key, long leaseId) {
try {
LeaseKeepAliveResponse response =
this.client.getLeaseClient().keepAliveOnce(leaseId).get();
return response.getID();
} catch (InterruptedException | ExecutionException e) {
// keepAlive once Failed
return 0;
}
}
@Override
public String get(String key) {
List<KeyValue> keyValues;
KV kvClient = this.client.getKVClient();
try {
keyValues = kvClient.get(toByteSequence(key))
.get().getKvs();
} catch (InterruptedException | ExecutionException e) {
throw new HugeException("Failed to get key '%s' from etcd", e, key);
}
if (!keyValues.isEmpty()) {
return keyValues.get(0).getValue().toString(Charset.defaultCharset());
}
return null;
}
@Override
public void put(String key, String value) {
KV kvClient = this.client.getKVClient();
try {
kvClient.put(toByteSequence(key), toByteSequence(value)).get();
} catch (InterruptedException | ExecutionException e) {
try {
kvClient.delete(toByteSequence(key)).get();
} catch (Throwable t) {
throw new HugeException("Failed to put '%s:%s' to etcd",
e, key, value);
}
}
}
@Override
public void delete(String key) {
KV kvClient = this.client.getKVClient();
try {
kvClient.delete(toByteSequence(key)).get();
} catch (InterruptedException | ExecutionException e) {
throw new HugeException(
"Failed to delete key '%s' from etcd", e, key);
}
}
@Override
public void deleteWithPrefix(String prefix) {
KV kvClient = this.client.getKVClient();
try {
DeleteOption option = DeleteOption.newBuilder()
.isPrefix(true)
.build();
kvClient.delete(toByteSequence(prefix), option);
} catch (Throwable e) {
throw new HugeException(
"Failed to delete prefix '%s' from etcd", e, prefix);
}
}
@Override
public Map<String, String> scanWithPrefix(String prefix) {
GetOption getOption = GetOption.newBuilder()
.isPrefix(true)
.build();
GetResponse response;
try {
response = this.client.getKVClient().get(toByteSequence(prefix),
getOption).get();
} catch (InterruptedException | ExecutionException e) {
throw new HugeException("Failed to scan etcd with prefix '%s'",
e, prefix);
}
int size = (int) response.getCount();
Map<String, String> keyValues = CollectionFactory.newMap(
CollectionType.JCF, size);
for (KeyValue kv : response.getKvs()) {
String key = kv.getKey().toString(Charset.defaultCharset());
String value = kv.getValue().isEmpty() ? "" :
kv.getValue().toString(Charset.defaultCharset());
keyValues.put(key, value);
}
return keyValues;
}
@Override
public <T> List<String> extractValuesFromResponse(T response) {
List<String> values = new ArrayList<>();
E.checkArgument(response instanceof WatchResponse,
"Invalid response type %s", response.getClass());
for (WatchEvent event : ((WatchResponse) response).getEvents()) {
// Skip if not etcd PUT event
if (!isEtcdPut(event)) {
return null;
}
String value = event.getKeyValue().getValue()
.toString(Charset.defaultCharset());
values.add(value);
}
return values;
}
@Override
public <T> Map<String, String> extractKVFromResponse(T response) {
E.checkArgument(response instanceof WatchResponse,
"Invalid response type %s", response.getClass());
Map<String, String> resultMap = new HashMap<>();
for (WatchEvent event : ((WatchResponse) response).getEvents()) {
// Skip if not etcd PUT event
if (!isEtcdPut(event)) {
continue;
}
String key = event.getKeyValue().getKey().toString(Charset.defaultCharset());
String value = event.getKeyValue().getValue()
.toString(Charset.defaultCharset());
if (Strings.isNullOrEmpty(key)) {
continue;
}
resultMap.put(key, value);
}
return resultMap;
}
@Override
public LockResult tryLock(String key, long ttl, long timeout) {
return this.lock.tryLock(key, ttl, timeout);
}
@Override
public boolean isLocked(String key) {
try {
long size = this.client.getKVClient().get(toByteSequence(key))
.get().getCount();
return size > 0;
} catch (InterruptedException | ExecutionException e) {
throw new HugeException("Failed to check is locked '%s'", e, key);
}
}
@Override
public void unlock(String key, LockResult lockResult) {
this.lock.unLock(key, lockResult);
}
@SuppressWarnings("unchecked")
@Override
public <T> void listen(String key, Consumer<T> consumer) {
this.client.getWatchClient().watch(toByteSequence(key),
(Consumer<WatchResponse>) consumer);
}
/**
* Listen etcd key with prefix
*/
@SuppressWarnings("unchecked")
@Override
public <T> void listenPrefix(String prefix, Consumer<T> consumer) {
ByteSequence sequence = toByteSequence(prefix);
WatchOption option = WatchOption.newBuilder().isPrefix(true).build();
this.client.getWatchClient().watch(sequence, option, (Consumer<WatchResponse>) consumer);
}
}

73
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaDriver.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.apache.hugegraph.meta.lock.LockResult;
public interface MetaDriver {
public void put(String key, String value);
public String get(String key);
public void delete(String key);
public void deleteWithPrefix(String prefix);
public Map<String, String> scanWithPrefix(String prefix);
public <T> void listen(String key, Consumer<T> consumer);
public <T> void listenPrefix(String prefix, Consumer<T> consumer);
public <T> List<String> extractValuesFromResponse(T response);
/**
* Extract K-V pairs of response
*
* @param <T>
* @param response
* @return
*/
public <T> Map<String, String> extractKVFromResponse(T response);
public LockResult tryLock(String key, long ttl, long timeout);
/**
* return if the key is Locked.
*
* @param key
* @return bool
*/
public boolean isLocked(String key);
public void unlock(String key, LockResult lockResult);
/**
* keepAlive of current lease
*
* @param key
* @param lease
* @return next leaseId
*/
public long keepAlive(String key, long lease);
}

1296
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaManager.java Normal file
View File

File diff suppressed because it is too large Load Diff

212
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/PdMetaDriver.java Normal file
View File

@ -0,0 +1,212 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.apache.hugegraph.HugeException;
import org.apache.hugegraph.meta.lock.LockResult;
import org.apache.hugegraph.meta.lock.PdDistributedLock;
import org.apache.hugegraph.pd.client.KvClient;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.client.PDConfig;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.kv.KResponse;
import org.apache.hugegraph.pd.grpc.kv.LockResponse;
import org.apache.hugegraph.pd.grpc.kv.ScanPrefixResponse;
import org.apache.hugegraph.pd.grpc.kv.TTLResponse;
import org.apache.hugegraph.pd.grpc.kv.WatchEvent;
import org.apache.hugegraph.pd.grpc.kv.WatchResponse;
import org.apache.hugegraph.pd.grpc.kv.WatchType;
import com.google.common.base.Strings;
public class PdMetaDriver implements MetaDriver {
private final KvClient<WatchResponse> client;
private final PDClient pdClient;
private final PdDistributedLock lock;
public PdMetaDriver(String pdPeer) {
PDConfig pdConfig = PDConfig.of(pdPeer);
this.client = new KvClient<>(pdConfig);
this.pdClient = PDClient.create(pdConfig);
this.lock = new PdDistributedLock(this.client);
}
public PDClient pdClient() {
return this.pdClient;
}
@Override
public void put(String key, String value) {
try {
this.client.put(key, value);
} catch (PDException e) {
throw new HugeException("Failed to put '%s:%s' to pd", e, key, value);
}
}
@Override
public String get(String key) {
try {
KResponse response = this.client.get(key);
return response.getValue();
} catch (PDException e) {
throw new HugeException("Failed to get '%s' from pd", e, key);
}
}
@Override
public void delete(String key) {
try {
this.client.delete(key);
} catch (PDException e) {
throw new HugeException("Failed to delete '%s' from pd", e, key);
}
}
@Override
public void deleteWithPrefix(String prefix) {
try {
this.client.deletePrefix(prefix);
} catch (PDException e) {
throw new HugeException("Failed to deleteWithPrefix '%s' from pd", e, prefix);
}
}
@Override
public Map<String, String> scanWithPrefix(String prefix) {
try {
ScanPrefixResponse response = this.client.scanPrefix(prefix);
return response.getKvsMap();
} catch (PDException e) {
throw new HugeException("Failed to scanWithPrefix '%s' from pd", e, prefix);
}
}
@Override
public <T> void listen(String key, Consumer<T> consumer) {
try {
this.client.listen(key, (Consumer<WatchResponse>) consumer);
} catch (PDException e) {
throw new HugeException("Failed to listen '%s' to pd", e, key);
}
}
@Override
public <T> void listenPrefix(String prefix, Consumer<T> consumer) {
try {
this.client.listenPrefix(prefix, (Consumer<WatchResponse>) consumer);
} catch (PDException e) {
throw new HugeException("Failed to listenPrefix '%s' to pd", e, prefix);
}
}
@Override
public <T> List<String> extractValuesFromResponse(T response) {
List<String> values = new ArrayList<>();
WatchResponse res = (WatchResponse) response;
for (WatchEvent event : res.getEventsList()) {
// Skip if not PUT event
if (!event.getType().equals(WatchType.Put)) {
return null;
}
String value = event.getCurrent().getValue();
values.add(value);
}
return values;
}
@Override
public <T> Map<String, String> extractKVFromResponse(T response) {
Map<String, String> resultMap = new HashMap<>();
WatchResponse res = (WatchResponse) response;
for (WatchEvent event : res.getEventsList()) {
// Skip if not etcd PUT event
if (!event.getType().equals(WatchType.Put)) {
continue;
}
String key = event.getCurrent().getKey();
String value = event.getCurrent().getValue();
if (Strings.isNullOrEmpty(key)) {
continue;
}
resultMap.put(key, value);
}
return resultMap;
}
@Override
public LockResult tryLock(String key, long ttl, long timeout) {
return this.lock.lock(key, ttl);
}
@Override
public boolean isLocked(String key) {
LockResponse locked;
try {
locked = this.client.isLocked(key);
} catch (PDException e) {
throw new HugeException("Failed to get isLocked '%s' from pd", key);
}
return locked.getSucceed();
}
@Override
public void unlock(String key, LockResult lockResult) {
this.lock.unLock(key, lockResult);
}
@Override
public long keepAlive(String key, long lease) {
try {
LockResponse lockResponse = this.client.keepAlive(key);
boolean succeed = lockResponse.getSucceed();
if (!succeed) {
throw new HugeException("Failed to keepAlive '%s' to pd", key);
}
return lockResponse.getClientId();
} catch (PDException e) {
throw new HugeException("Failed to keepAlive '%s' to pd", e, key);
}
}
public boolean keepTTLAlive(String key) {
try {
TTLResponse response = this.client.keepTTLAlive(key);
return response.getSucceed();
} catch (PDException e) {
throw new HugeException("Failed to keepTTLAlive '%s' to pd", e, key);
}
}
public boolean putTTL(String key, String value, long ttl) {
try {
TTLResponse response = this.client.putTTL(key, value, ttl);
return response.getSucceed();
} catch (PDException e) {
throw new HugeException("Failed to keepTTLAlive '%s' to pd", e, key);
}
}
}

167
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/lock/EtcdDistributedLock.java Normal file
View File

@ -0,0 +1,167 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.lock;
import java.nio.charset.Charset;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
import io.etcd.jetcd.ByteSequence;
import io.etcd.jetcd.Client;
import io.etcd.jetcd.KV;
import io.etcd.jetcd.Lease;
import io.etcd.jetcd.Lock;
public class EtcdDistributedLock {
protected static final Logger LOG = Log.logger(EtcdDistributedLock.class);
private static final long UNLIMITED_TIMEOUT = -1L;
private final static Object mutex = new Object();
private static EtcdDistributedLock lockProvider = null;
private final KV kvClient;
private final Lock lockClient;
private final Lease leaseClient;
private static final int poolSize = 8;
private final ScheduledExecutorService service = new ScheduledThreadPoolExecutor(poolSize, r -> {
Thread t = new Thread(r, "keepalive");
t.setDaemon(true);
return t;
});
private EtcdDistributedLock(Client client) {
this.kvClient = client.getKVClient();
this.lockClient = client.getLockClient();
this.leaseClient = client.getLeaseClient();
}
public static EtcdDistributedLock getInstance(Client client) {
synchronized (mutex) {
if (null == lockProvider) {
lockProvider = new EtcdDistributedLock(client);
}
}
return lockProvider;
}
private static ByteSequence toByteSequence(String content) {
return ByteSequence.from(content, Charset.defaultCharset());
}
public LockResult tryLock(String lockName, long ttl, long timeout) {
LockResult lockResult = new LockResult();
lockResult.lockSuccess(false);
lockResult.setService(service);
long leaseId;
try {
leaseId = this.leaseClient.grant(ttl).get().getID();
} catch (InterruptedException | ExecutionException e) {
LOG.warn(String.format("Thread {} failed to create lease for {} " +
"with ttl {}", Thread.currentThread().getName(),
lockName, ttl),
e);
return lockResult;
}
lockResult.setLeaseId(leaseId);
long period = ttl - ttl / 5;
service.scheduleAtFixedRate(new KeepAliveTask(this.leaseClient, leaseId),
period, period, TimeUnit.SECONDS);
try {
if (timeout == UNLIMITED_TIMEOUT) {
this.lockClient.lock(toByteSequence(lockName), leaseId).get();
} else {
this.lockClient.lock(toByteSequence(lockName), leaseId)
.get(1, TimeUnit.SECONDS);
}
} catch (InterruptedException | ExecutionException e) {
LOG.warn(String.format("Thread {} failed to lock {}",
Thread.currentThread().getName(), lockName),
e);
service.shutdown();
this.revokeLease(leaseId);
return lockResult;
} catch (TimeoutException e) {
// 获取锁超时
LOG.warn("Thread {} timeout to lock {}",
Thread.currentThread().getName(), lockName);
service.shutdown();
this.revokeLease(leaseId);
return lockResult;
}
lockResult.lockSuccess(true);
return lockResult;
}
public LockResult lock(String lockName, long ttl) {
return tryLock(lockName, ttl, UNLIMITED_TIMEOUT);
}
public void unLock(String lockName, LockResult lockResult) {
LOG.debug("Thread {} start to unlock {}",
Thread.currentThread().getName(), lockName);
lockResult.getService().shutdown();
if (lockResult.getLeaseId() != 0L) {
this.revokeLease(lockResult.getLeaseId());
}
LOG.debug("Thread {} unlock {} successfully",
Thread.currentThread().getName(), lockName);
}
private void revokeLease(long leaseId) {
try {
this.leaseClient.revoke(leaseId).get();
} catch (InterruptedException | ExecutionException e) {
LOG.warn(String.format("Thread %s failed to revoke release %s",
Thread.currentThread().getName(), leaseId), e);
}
}
public static class KeepAliveTask implements Runnable {
private final Lease leaseClient;
private final long leaseId;
KeepAliveTask(Lease leaseClient, long leaseId) {
this.leaseClient = leaseClient;
this.leaseId = leaseId;
}
@Override
public void run() {
// TODO: calculate the time interval between the calls
this.leaseClient.keepAliveOnce(this.leaseId);
}
}
}

61
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/lock/LockResult.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.lock;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
public class LockResult {
private boolean lockSuccess;
private long leaseId;
private ScheduledExecutorService service;
private ScheduledFuture<?> future;
public void lockSuccess(boolean isLockSuccess) {
this.lockSuccess = isLockSuccess;
}
public boolean lockSuccess() {
return this.lockSuccess;
}
public long getLeaseId() {
return this.leaseId;
}
public void setLeaseId(long leaseId) {
this.leaseId = leaseId;
}
public ScheduledExecutorService getService() {
return this.service;
}
public void setService(ScheduledExecutorService service) {
this.service = service;
}
public ScheduledFuture<?> getFuture() {
return future;
}
public void setFuture(ScheduledFuture<?> future) {
this.future = future;
}
}

94
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/lock/PdDistributedLock.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.lock;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.hugegraph.HugeException;
import org.apache.hugegraph.pd.client.KvClient;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.kv.LockResponse;
public class PdDistributedLock {
private static final int poolSize = 8;
private final KvClient<?> client;
private final ScheduledExecutorService service = new ScheduledThreadPoolExecutor(poolSize, r -> {
Thread t = new Thread(r, "keepalive");
t.setDaemon(true);
return t;
});
public PdDistributedLock(KvClient<?> client) {
this.client = client;
}
public LockResult lock(String key, long second) {
long ttl = second * 1000L;
try {
LockResponse response = this.client.lock(key, ttl);
boolean succeed = response.getSucceed();
LockResult result = new LockResult();
if (succeed) {
result.setLeaseId(response.getClientId());
result.lockSuccess(true);
long period = ttl - ttl / 4;
ScheduledFuture<?> future = service.scheduleAtFixedRate(() -> {
// TODO: why synchronized?
synchronized (result) {
keepAlive(key);
}
}, 10, period, TimeUnit.MILLISECONDS);
result.setFuture(future);
}
return result;
} catch (PDException e) {
throw new HugeException("Failed to lock '%s' to pd", e, key);
}
}
public void unLock(String key, LockResult lockResult) {
try {
LockResponse response = this.client.unlock(key);
boolean succeed = response.getSucceed();
if (!succeed) {
throw new HugeException("Failed to unlock '%s' to pd", key);
}
if (lockResult.getFuture() != null) {
// TODO: why synchronized?
synchronized (lockResult) {
lockResult.getFuture().cancel(true);
}
}
} catch (PDException e) {
throw new HugeException("Failed to unlock '%s' to pd", e, key);
}
}
public boolean keepAlive(String key) {
try {
LockResponse alive = this.client.keepAlive(key);
return alive.getSucceed();
} catch (PDException e) {
throw new HugeException("Failed to keepAlive '%s' to pd", key);
}
}
}

94
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/AbstractMetaManager.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.managers;
import static org.apache.hugegraph.meta.MetaManager.LOCK_DEFAULT_LEASE;
import static org.apache.hugegraph.meta.MetaManager.LOCK_DEFAULT_TIMEOUT;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_DELIMITER;
import java.util.Map;
import java.util.Optional;
import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
import org.apache.hugegraph.auth.SchemaDefine;
import org.apache.hugegraph.meta.MetaDriver;
import org.apache.hugegraph.meta.lock.LockResult;
import org.apache.hugegraph.schema.SchemaElement;
import org.apache.hugegraph.util.JsonUtil;
public class AbstractMetaManager {
protected final MetaDriver metaDriver;
protected final String cluster;
public AbstractMetaManager(MetaDriver metaDriver, String cluster) {
this.metaDriver = metaDriver;
this.cluster = cluster;
}
protected static String serialize(SchemaDefine.AuthElement element) {
Map<String, Object> objectMap = element.asMap();
return JsonUtil.toJson(objectMap);
}
protected static String serialize(SchemaElement element) {
Map<String, Object> objectMap = element.asMap();
return JsonUtil.toJson(objectMap);
}
@SuppressWarnings("unchecked")
protected static Map<String, Object> configMap(String config) {
return JsonUtil.fromJson(config, Map.class);
}
protected <T> void listen(String key, Consumer<T> consumer) {
this.metaDriver.listen(key, consumer);
}
protected <T> void listenPrefix(String prefix, Consumer<T> consumer) {
this.metaDriver.listenPrefix(prefix, consumer);
}
public String getRaw(String key) {
String result = this.metaDriver.get(key);
return Optional.ofNullable(result).orElse("");
}
public void putOrDeleteRaw(String key, String val) {
if (StringUtils.isEmpty(val)) {
this.metaDriver.delete(key);
} else {
this.metaDriver.put(key, val);
}
}
public LockResult tryLock(String key) {
return this.metaDriver.tryLock(key, LOCK_DEFAULT_LEASE,
LOCK_DEFAULT_TIMEOUT);
}
public void unlock(LockResult lockResult, String... keys) {
String key = String.join(META_PATH_DELIMITER, keys);
this.unlock(key, lockResult);
}
public void unlock(String key, LockResult lockResult) {
this.metaDriver.unlock(key, lockResult);
}
}

1035
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/AuthMetaManager.java Normal file
View File

File diff suppressed because it is too large Load Diff

149
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/ConfigMetaManager.java Normal file
View File

@ -0,0 +1,149 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.managers;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_DELIMITER;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_GRAPHSPACE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_GREMLIN_YAML;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_HUGEGRAPH;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_REST_PROPERTIES;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_SERVICE;
import java.util.Map;
import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
import org.apache.hugegraph.meta.MetaDriver;
import org.apache.hugegraph.util.JsonUtil;
public class ConfigMetaManager extends AbstractMetaManager {
public ConfigMetaManager(MetaDriver metaDriver, String cluster) {
super(metaDriver, cluster);
}
@SuppressWarnings("unchecked")
public Map<String, Object> restProperties(String graphSpace,
String serviceId) {
Map<String, Object> map = null;
String result = this.metaDriver.get(restPropertiesKey(graphSpace,
serviceId));
if (StringUtils.isNotEmpty(result)) {
map = JsonUtil.fromJson(result, Map.class);
}
return map;
}
@SuppressWarnings("unchecked")
public Map<String, Object> restProperties(String graphSpace,
String serviceId,
Map<String, Object> properties) {
Map<String, Object> map;
String result = this.metaDriver.get(restPropertiesKey(graphSpace,
serviceId));
if (StringUtils.isNotEmpty(result)) {
map = JsonUtil.fromJson(result, Map.class);
for (Map.Entry<String, Object> item : properties.entrySet()) {
map.put(item.getKey(), item.getValue());
}
} else {
map = properties;
}
this.metaDriver.put(restPropertiesKey(graphSpace, serviceId),
JsonUtil.toJson(map));
return map;
}
@SuppressWarnings("unchecked")
public Map<String, Object> deleteRestProperties(String graphSpace,
String serviceId,
String key) {
Map<String, Object> map = null;
String result = this.metaDriver.get(restPropertiesKey(graphSpace,
serviceId));
if (StringUtils.isNotEmpty(result)) {
map = JsonUtil.fromJson(result, Map.class);
map.remove(key);
this.metaDriver.put(restPropertiesKey(graphSpace, serviceId),
JsonUtil.toJson(map));
}
return map;
}
@SuppressWarnings("unchecked")
public Map<String, Object> clearRestProperties(String graphSpace,
String serviceId) {
Map<String, Object> map = null;
String key = restPropertiesKey(graphSpace, serviceId);
String result = this.metaDriver.get(key);
if (StringUtils.isNotEmpty(result)) {
map = JsonUtil.fromJson(result, Map.class);
this.metaDriver.delete(key);
}
return map;
}
public String gremlinYaml(String graphSpace, String serviceId) {
return this.metaDriver.get(gremlinYamlKey(graphSpace, serviceId));
}
public String gremlinYaml(String graphSpace, String serviceId,
String yaml) {
this.metaDriver.put(gremlinYamlKey(graphSpace, serviceId), yaml);
return yaml;
}
public <T> void listenRestPropertiesUpdate(String graphSpace,
String serviceId,
Consumer<T> consumer) {
this.listen(this.restPropertiesKey(graphSpace, serviceId), consumer);
}
public <T> void listenGremlinYamlUpdate(String graphSpace,
String serviceId,
Consumer<T> consumer) {
this.listen(this.gremlinYamlKey(graphSpace, serviceId), consumer);
}
private String restPropertiesKey(String graphSpace, String serviceId) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/SERVICE/
// {serviceId}/REST_PROPERTIES
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
META_PATH_SERVICE,
serviceId,
META_PATH_REST_PROPERTIES);
}
private String gremlinYamlKey(String graphSpace, String serviceId) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/SERVICE/
// {serviceId}/GREMLIN_YAML
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
META_PATH_SERVICE,
serviceId,
META_PATH_GREMLIN_YAML);
}
}

284
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/GraphMetaManager.java Normal file
View File

@ -0,0 +1,284 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.managers;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_ADD;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_CLEAR;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_DELIMITER;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_EDGE_LABEL;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_EVENT;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_GRAPH;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_GRAPHSPACE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_GRAPH_CONF;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_HUGEGRAPH;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_JOIN;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_REMOVE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_SCHEMA;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_UPDATE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_VERTEX_LABEL;
import java.util.Map;
import java.util.function.Consumer;
import org.apache.hugegraph.meta.MetaDriver;
import org.apache.hugegraph.type.define.CollectionType;
import org.apache.hugegraph.util.JsonUtil;
import org.apache.hugegraph.util.collection.CollectionFactory;
import org.apache.logging.log4j.util.Strings;
public class GraphMetaManager extends AbstractMetaManager {
public GraphMetaManager(MetaDriver metaDriver, String cluster) {
super(metaDriver, cluster);
}
private static String graphName(String graphSpace, String name) {
return String.join(META_PATH_JOIN, graphSpace, name);
}
public Map<String, Map<String, Object>> graphConfigs(String graphSpace) {
Map<String, Map<String, Object>> configs =
CollectionFactory.newMap(CollectionType.EC);
Map<String, String> keyValues = this.metaDriver.scanWithPrefix(
this.graphConfPrefix(graphSpace));
for (Map.Entry<String, String> entry : keyValues.entrySet()) {
String key = entry.getKey();
String[] parts = key.split(META_PATH_DELIMITER);
String name = parts[parts.length - 1];
String graphName = String.join("-", graphSpace, name);
configs.put(graphName, configMap(entry.getValue()));
}
return configs;
}
public void removeGraphConfig(String graphSpace, String graph) {
this.metaDriver.delete(this.graphConfKey(graphSpace, graph));
}
public void notifyGraphAdd(String graphSpace, String graph) {
this.metaDriver.put(this.graphAddKey(),
graphName(graphSpace, graph));
}
public void notifyGraphRemove(String graphSpace, String graph) {
this.metaDriver.put(this.graphRemoveKey(),
graphName(graphSpace, graph));
}
public void notifyGraphUpdate(String graphSpace, String graph) {
this.metaDriver.put(this.graphUpdateKey(),
graphName(graphSpace, graph));
}
public void notifyGraphClear(String graphSpace, String graph) {
this.metaDriver.put(this.graphClearKey(),
graphName(graphSpace, graph));
}
public void notifySchemaCacheClear(String graphSpace, String graph) {
this.metaDriver.put(this.schemaCacheClearKey(),
graphName(graphSpace, graph));
}
public void notifyGraphCacheClear(String graphSpace, String graph) {
this.metaDriver.put(this.graphCacheClearKey(),
graphName(graphSpace, graph));
}
/**
* 通知 点信息 cache clear
*
* @param graphSpace
* @param graph
*/
public void notifyGraphVertexCacheClear(String graphSpace, String graph) {
this.metaDriver.put(this.graphVertexCacheClearKey(),
graphName(graphSpace, graph));
}
/**
* 通知 边信息 cache clear
*
* @param graphSpace
* @param graph
*/
public void notifyGraphEdgeCacheClear(String graphSpace, String graph) {
this.metaDriver.put(this.graphEdgeCacheClearKey(),
graphName(graphSpace, graph));
}
public Map<String, Object> getGraphConfig(String graphSpace, String graph) {
return configMap(this.metaDriver.get(this.graphConfKey(graphSpace,
graph)));
}
public void addGraphConfig(String graphSpace, String graph,
Map<String, Object> configs) {
this.metaDriver.put(this.graphConfKey(graphSpace, graph),
JsonUtil.toJson(configs));
}
public void updateGraphConfig(String graphSpace, String graph,
Map<String, Object> configs) {
this.metaDriver.put(this.graphConfKey(graphSpace, graph),
JsonUtil.toJson(configs));
}
public <T> void listenGraphAdd(Consumer<T> consumer) {
this.listen(this.graphAddKey(), consumer);
}
public <T> void listenGraphUpdate(Consumer<T> consumer) {
this.listen(this.graphUpdateKey(), consumer);
}
public <T> void listenGraphRemove(Consumer<T> consumer) {
this.listen(this.graphRemoveKey(), consumer);
}
public <T> void listenGraphClear(Consumer<T> consumer) {
this.listen(this.graphClearKey(), consumer);
}
public <T> void listenSchemaCacheClear(Consumer<T> consumer) {
this.listen(this.schemaCacheClearKey(), consumer);
}
public <T> void listenGraphCacheClear(Consumer<T> consumer) {
this.listen(this.graphCacheClearKey(), consumer);
}
public <T> void listenGraphVertexCacheClear(Consumer<T> consumer) {
this.listen(this.graphVertexCacheClearKey(), consumer);
}
public <T> void listenGraphEdgeCacheClear(Consumer<T> consumer) {
this.listen(this.graphEdgeCacheClearKey(), consumer);
}
private String graphConfPrefix(String graphSpace) {
return this.graphConfKey(graphSpace, Strings.EMPTY);
}
private String graphConfKey(String graphSpace, String graph) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH_CONF/{graph}
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
META_PATH_GRAPH_CONF,
graph);
}
private String graphAddKey() {
// HUGEGRAPH/{cluster}/EVENT/GRAPH/ADD
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_GRAPH,
META_PATH_ADD);
}
private String graphRemoveKey() {
// HUGEGRAPH/{cluster}/EVENT/GRAPH/REMOVE
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_GRAPH,
META_PATH_REMOVE);
}
private String graphUpdateKey() {
// HUGEGRAPH/{cluster}/EVENT/GRAPH/UPDATE
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_GRAPH,
META_PATH_UPDATE);
}
private String graphClearKey() {
// HUGEGRAPH/{cluster}/EVENT/GRAPH/CLEAR
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_GRAPH,
META_PATH_CLEAR);
}
private String schemaCacheClearKey() {
// HUGEGRAPH/{cluster}/EVENT/GRAPH/SCHEMA/CLEAR
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_GRAPH,
META_PATH_SCHEMA,
META_PATH_CLEAR);
}
private String graphCacheClearKey() {
// HUGEGRAPH/{cluster}/EVENT/GRAPH/GRAPH/CLEAR
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_GRAPH,
META_PATH_GRAPH,
META_PATH_CLEAR);
}
/**
* pd监听 vertex label更新的key
*
* @return
*/
private String graphVertexCacheClearKey() {
// HUGEGRAPH/{cluster}/EVENT/GRAPH/GRAPH/META_PATH_VERTEX_LABEL/CLEAR
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_GRAPH,
META_PATH_GRAPH,
META_PATH_VERTEX_LABEL,
META_PATH_CLEAR);
}
/**
* pd监听 edge label更新的key
*
* @return
*/
private String graphEdgeCacheClearKey() {
// HUGEGRAPH/{cluster}/EVENT/GRAPH/GRAPH/META_PATH_EDGE_LABEL/CLEAR
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_GRAPH,
META_PATH_GRAPH,
META_PATH_EDGE_LABEL,
META_PATH_CLEAR);
}
}

46
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/KafkaMetaManager.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.managers;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_DELIMITER;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_HUGEGRAPH;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_KAFKA;
import java.util.function.Consumer;
import org.apache.hugegraph.meta.MetaDriver;
public class KafkaMetaManager extends AbstractMetaManager {
public KafkaMetaManager(MetaDriver metaDriver, String cluster) {
super(metaDriver, cluster);
}
public <T> void listenKafkaConfig(Consumer<T> consumer) {
String prefix = this.kafkaPrefixKey();
this.listenPrefix(prefix, consumer);
}
private String kafkaPrefixKey() {
// HUGEGRAPH/{cluster}/KAFKA
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_KAFKA);
}
}

27
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/LockMetaManager.java Normal file
View File

@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.managers;
import org.apache.hugegraph.meta.MetaDriver;
public class LockMetaManager extends AbstractMetaManager {
public LockMetaManager(MetaDriver metaDriver, String cluster) {
super(metaDriver, cluster);
}
}

517
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/SchemaMetaManager.java Normal file
View File

@ -0,0 +1,517 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.managers;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_DELIMITER;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_EDGE_LABEL;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_GRAPHSPACE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_HUGEGRAPH;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_ID;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_INDEX_LABEL;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_NAME;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_PROPERTY_KEY;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_SCHEMA;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_VERTEX_LABEL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.meta.MetaDriver;
import org.apache.hugegraph.meta.PdMetaDriver;
import org.apache.hugegraph.schema.EdgeLabel;
import org.apache.hugegraph.schema.IndexLabel;
import org.apache.hugegraph.schema.PropertyKey;
import org.apache.hugegraph.schema.VertexLabel;
import org.apache.hugegraph.util.JsonUtil;
public class SchemaMetaManager extends AbstractMetaManager {
private final HugeGraph graph;
public SchemaMetaManager(MetaDriver metaDriver, String cluster, HugeGraph graph) {
super(metaDriver, cluster);
this.graph = graph;
}
public static void main(String[] args) {
MetaDriver metaDriver = new PdMetaDriver("127.0.0.1:8686");
SchemaMetaManager schemaMetaManager = new SchemaMetaManager(metaDriver, "hg", null);
PropertyKey propertyKey = new PropertyKey(null, IdGenerator.of(5), "test");
propertyKey.userdata("key1", "value1");
propertyKey.userdata("key2", 23);
schemaMetaManager.addPropertyKey("DEFAULT1", "hugegraph", propertyKey);
// PropertyKey propertyKey1 = schemaMetaManager.getPropertyKey("DEFAULT1", "hugegraph",
// IdGenerator.of(1));
schemaMetaManager.removePropertyKey("DEFAULT", "hugegraph", IdGenerator.of(1));
// propertyKey1 = schemaMetaManager.getPropertyKey("DEFAULT1", "hugegraph", "test");
// System.out.println(propertyKey1 );
//
// propertyKey1 = schemaMetaManager.getPropertyKey("DEFAULT1", "hugegraph", "5");
// System.out.println(propertyKey1 );
}
public void addPropertyKey(String graphSpace, String graph,
PropertyKey propertyKey) {
String content = serialize(propertyKey);
this.metaDriver.put(propertyKeyIdKey(graphSpace, graph,
propertyKey.id()), content);
this.metaDriver.put(propertyKeyNameKey(graphSpace, graph,
propertyKey.name()), content);
}
public void updatePropertyKey(String graphSpace, String graph,
PropertyKey pkey) {
this.addPropertyKey(graphSpace, graph, pkey);
}
@SuppressWarnings("unchecked")
public PropertyKey getPropertyKey(String graphSpace, String graph,
Id propertyKey) {
String content = this.metaDriver.get(propertyKeyIdKey(graphSpace, graph,
propertyKey));
if (content == null || content.length() == 0) {
return null;
} else {
return PropertyKey.fromMap(JsonUtil.fromJson(content, Map.class), this.graph);
}
}
@SuppressWarnings("unchecked")
public PropertyKey getPropertyKey(String graphSpace, String graph,
String propertyKey) {
String content = this.metaDriver.get(propertyKeyNameKey(graphSpace,
graph,
propertyKey));
if (content == null || content.length() == 0) {
return null;
} else {
return PropertyKey.fromMap(JsonUtil.fromJson(content, Map.class), this.graph);
}
}
@SuppressWarnings("unchecked")
public List<PropertyKey> getPropertyKeys(String graphSpace, String graph) {
Map<String, String> propertyKeysKvs = this.metaDriver.scanWithPrefix(
propertyKeyPrefix(graphSpace, graph));
List<PropertyKey> propertyKeys =
new ArrayList<>(propertyKeysKvs.size());
for (String value : propertyKeysKvs.values()) {
propertyKeys.add(PropertyKey.fromMap(JsonUtil.fromJson(value, Map.class), this.graph));
}
return propertyKeys;
}
public Id removePropertyKey(String graphSpace, String graph,
Id propertyKey) {
PropertyKey p = this.getPropertyKey(graphSpace, graph, propertyKey);
this.metaDriver.delete(propertyKeyNameKey(graphSpace, graph,
p.name()));
this.metaDriver.delete(propertyKeyIdKey(graphSpace, graph,
propertyKey));
return IdGenerator.ZERO;
}
public void addVertexLabel(String graphSpace, String graph,
VertexLabel vertexLabel) {
String content = serialize(vertexLabel);
this.metaDriver.put(vertexLabelIdKey(graphSpace, graph,
vertexLabel.id()), content);
this.metaDriver.put(vertexLabelNameKey(graphSpace, graph,
vertexLabel.name()), content);
}
public void updateVertexLabel(String graphSpace, String graph,
VertexLabel vertexLabel) {
this.addVertexLabel(graphSpace, graph, vertexLabel);
}
@SuppressWarnings("unchecked")
public VertexLabel getVertexLabel(String graphSpace, String graph,
Id vertexLabel) {
String content = this.metaDriver.get(vertexLabelIdKey(graphSpace, graph,
vertexLabel));
if (content == null || content.length() == 0) {
return null;
} else {
return VertexLabel.fromMap(JsonUtil.fromJson(content, Map.class), this.graph);
}
}
@SuppressWarnings("unchecked")
public VertexLabel getVertexLabel(String graphSpace, String graph,
String vertexLabel) {
String content = this.metaDriver.get(vertexLabelNameKey(graphSpace,
graph,
vertexLabel));
if (content == null || content.length() == 0) {
return null;
} else {
return VertexLabel.fromMap(JsonUtil.fromJson(content, Map.class), this.graph);
}
}
@SuppressWarnings("unchecked")
public List<VertexLabel> getVertexLabels(String graphSpace, String graph) {
Map<String, String> vertexLabelKvs = this.metaDriver.scanWithPrefix(
vertexLabelPrefix(graphSpace, graph));
List<VertexLabel> vertexLabels =
new ArrayList<>(vertexLabelKvs.size());
for (String value : vertexLabelKvs.values()) {
vertexLabels.add(VertexLabel.fromMap(
JsonUtil.fromJson(value, Map.class), this.graph));
}
return vertexLabels;
}
public Id removeVertexLabel(String graphSpace, String graph,
Id vertexLabel) {
VertexLabel v = this.getVertexLabel(graphSpace, graph,
vertexLabel);
this.metaDriver.delete(vertexLabelNameKey(graphSpace, graph,
v.name()));
this.metaDriver.delete(vertexLabelIdKey(graphSpace, graph,
vertexLabel));
return IdGenerator.ZERO;
}
public void addEdgeLabel(String graphSpace, String graph,
EdgeLabel edgeLabel) {
String content = serialize(edgeLabel);
this.metaDriver.put(edgeLabelIdKey(graphSpace, graph,
edgeLabel.id()), content);
this.metaDriver.put(edgeLabelNameKey(graphSpace, graph,
edgeLabel.name()), content);
}
public void updateEdgeLabel(String graphSpace, String graph,
EdgeLabel edgeLabel) {
this.addEdgeLabel(graphSpace, graph, edgeLabel);
}
@SuppressWarnings("unchecked")
public EdgeLabel getEdgeLabel(String graphSpace, String graph,
Id edgeLabel) {
String content = this.metaDriver.get(edgeLabelIdKey(graphSpace, graph,
edgeLabel));
if (content == null || content.length() == 0) {
return null;
} else {
return EdgeLabel.fromMap(JsonUtil.fromJson(content, Map.class), this.graph);
}
}
@SuppressWarnings("unchecked")
public EdgeLabel getEdgeLabel(String graphSpace, String graph,
String edgeLabel) {
String content = this.metaDriver.get(edgeLabelNameKey(graphSpace,
graph,
edgeLabel));
if (content == null || content.length() == 0) {
return null;
} else {
return EdgeLabel.fromMap(JsonUtil.fromJson(content, Map.class), this.graph);
}
}
@SuppressWarnings("unchecked")
public List<EdgeLabel> getEdgeLabels(String graphSpace, String graph) {
Map<String, String> edgeLabelKvs = this.metaDriver.scanWithPrefix(
edgeLabelPrefix(graphSpace, graph));
List<EdgeLabel> edgeLabels =
new ArrayList<>(edgeLabelKvs.size());
for (String value : edgeLabelKvs.values()) {
edgeLabels.add(EdgeLabel.fromMap(
JsonUtil.fromJson(value, Map.class), this.graph));
}
return edgeLabels;
}
public Id removeEdgeLabel(String graphSpace, String graph,
Id edgeLabel) {
EdgeLabel e = this.getEdgeLabel(graphSpace, graph,
edgeLabel);
this.metaDriver.delete(edgeLabelNameKey(graphSpace, graph,
e.name()));
this.metaDriver.delete(edgeLabelIdKey(graphSpace, graph,
edgeLabel));
return IdGenerator.ZERO;
}
public void addIndexLabel(String graphSpace, String graph,
IndexLabel indexLabel) {
String content = serialize(indexLabel);
this.metaDriver.put(indexLabelIdKey(graphSpace, graph,
indexLabel.id()), content);
this.metaDriver.put(indexLabelNameKey(graphSpace, graph,
indexLabel.name()), content);
}
public void updateIndexLabel(String graphSpace, String graph,
IndexLabel indexLabel) {
this.addIndexLabel(graphSpace, graph, indexLabel);
}
@SuppressWarnings("unchecked")
public IndexLabel getIndexLabel(String graphSpace, String graph,
Id indexLabel) {
String content = this.metaDriver.get(indexLabelIdKey(graphSpace, graph,
indexLabel));
if (content == null || content.length() == 0) {
return null;
} else {
return IndexLabel.fromMap(JsonUtil.fromJson(content, Map.class), this.graph);
}
}
@SuppressWarnings("unchecked")
public IndexLabel getIndexLabel(String graphSpace, String graph,
String edgeLabel) {
String content = this.metaDriver.get(indexLabelNameKey(graphSpace,
graph,
edgeLabel));
if (content == null || content.length() == 0) {
return null;
} else {
return IndexLabel.fromMap(JsonUtil.fromJson(content, Map.class), this.graph);
}
}
@SuppressWarnings("unchecked")
public List<IndexLabel> getIndexLabels(String graphSpace, String graph) {
Map<String, String> indexLabelKvs = this.metaDriver.scanWithPrefix(
indexLabelPrefix(graphSpace, graph));
List<IndexLabel> indexLabels =
new ArrayList<>(indexLabelKvs.size());
for (String value : indexLabelKvs.values()) {
indexLabels.add(IndexLabel.fromMap(
JsonUtil.fromJson(value, Map.class), this.graph));
}
return indexLabels;
}
public Id removeIndexLabel(String graphSpace, String graph, Id indexLabel) {
IndexLabel i = this.getIndexLabel(graphSpace, graph,
indexLabel);
this.metaDriver.delete(indexLabelNameKey(graphSpace, graph,
i.name()));
this.metaDriver.delete(indexLabelIdKey(graphSpace, graph,
indexLabel));
return IdGenerator.ZERO;
}
private String propertyKeyPrefix(String graphSpace, String graph) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/PROPERTY_KEY/NAME
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_PROPERTY_KEY,
META_PATH_NAME);
}
private String propertyKeyIdKey(String graphSpace, String graph, Id id) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/PROPERTY_KEY/ID/{id}
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_PROPERTY_KEY,
META_PATH_ID,
id.asString());
}
private String propertyKeyNameKey(String graphSpace, String graph,
String name) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/PROPERTY_KEY/NAME/{name}
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_PROPERTY_KEY,
META_PATH_NAME,
name);
}
private String vertexLabelPrefix(String graphSpace, String graph) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/VERTEX_LABEL/NAME
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_VERTEX_LABEL,
META_PATH_NAME);
}
private String vertexLabelIdKey(String graphSpace, String graph, Id id) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/VERTEX_LABEL/ID/{id}
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_VERTEX_LABEL,
META_PATH_ID,
id.asString());
}
private String vertexLabelNameKey(String graphSpace, String graph,
String name) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/VERTEX_LABEL/NAME/{name}
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_VERTEX_LABEL,
META_PATH_NAME,
name);
}
private String edgeLabelPrefix(String graphSpace, String graph) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/PROPERTYKEY/NAME
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_EDGE_LABEL,
META_PATH_NAME);
}
private String edgeLabelIdKey(String graphSpace, String graph, Id id) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/PROPERTYKEY/ID/{id}
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_EDGE_LABEL,
META_PATH_ID,
id.asString());
}
private String edgeLabelNameKey(String graphSpace, String graph,
String name) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/EDGE_LABEL/NAME/{name}
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_EDGE_LABEL,
META_PATH_NAME,
name);
}
private String indexLabelPrefix(String graphSpace, String graph) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/INDEX_LABEL/NAME
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_INDEX_LABEL,
META_PATH_NAME);
}
private String indexLabelIdKey(String graphSpace, String graph, Id id) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/INDEX_LABEL/ID/{id}
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_INDEX_LABEL,
META_PATH_ID,
id.asString());
}
private String indexLabelNameKey(String graphSpace, String graph,
String name) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph
// }/SCHEMA/INDEX_LABEL/NAME/{name}
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA,
META_PATH_INDEX_LABEL,
META_PATH_NAME,
name);
}
private String graphNameKey(String graphSpace, String graph) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph}/SCHEMA
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graph,
META_PATH_SCHEMA);
}
public void clearAllSchema(String graphSpace, String graph) {
this.metaDriver.deleteWithPrefix(graphNameKey(graphSpace, graph));
}
}

107
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/SchemaTemplateMetaManager.java Normal file
View File

@ -0,0 +1,107 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.managers;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_DELIMITER;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_GRAPHSPACE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_HUGEGRAPH;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_SCHEMA_TEMPLATE;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.hugegraph.HugeException;
import org.apache.hugegraph.meta.MetaDriver;
import org.apache.hugegraph.space.SchemaTemplate;
import org.apache.hugegraph.util.JsonUtil;
import org.apache.logging.log4j.util.Strings;
public class SchemaTemplateMetaManager extends AbstractMetaManager {
public SchemaTemplateMetaManager(MetaDriver metaDriver, String cluster) {
super(metaDriver, cluster);
}
public Set<String> schemaTemplates(String graphSpace) {
Set<String> result = new HashSet<>();
Map<String, String> keyValues = this.metaDriver.scanWithPrefix(
this.schemaTemplatePrefix(graphSpace));
for (String key : keyValues.keySet()) {
String[] parts = key.split(META_PATH_DELIMITER);
result.add(parts[parts.length - 1]);
}
return result;
}
@SuppressWarnings("unchecked")
public SchemaTemplate schemaTemplate(String graphSpace,
String schemaTemplate) {
String s = this.metaDriver.get(this.schemaTemplateKey(graphSpace,
schemaTemplate));
if (StringUtils.isEmpty(s)) {
return null;
}
return SchemaTemplate.fromMap(JsonUtil.fromJson(s, Map.class));
}
public void addSchemaTemplate(String graphSpace, SchemaTemplate template) {
String key = this.schemaTemplateKey(graphSpace, template.name());
String data = this.metaDriver.get(key);
if (StringUtils.isNotEmpty(data)) {
throw new HugeException("Cannot create schema template " +
"since it has been created");
}
this.metaDriver.put(this.schemaTemplateKey(graphSpace, template.name()),
JsonUtil.toJson(template.asMap()));
}
public void updateSchemaTemplate(String graphSpace,
SchemaTemplate template) {
this.metaDriver.put(this.schemaTemplateKey(graphSpace, template.name()),
JsonUtil.toJson(template.asMap()));
}
public void removeSchemaTemplate(String graphSpace, String name) {
this.metaDriver.delete(this.schemaTemplateKey(graphSpace, name));
}
public void clearSchemaTemplate(String graphSpace) {
String prefix = this.schemaTemplatePrefix(graphSpace);
this.metaDriver.deleteWithPrefix(prefix);
}
private String schemaTemplatePrefix(String graphSpace) {
return this.schemaTemplateKey(graphSpace, Strings.EMPTY);
}
private String schemaTemplateKey(String graphSpace, String schemaTemplate) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/SCHEMA_TEMPLATE
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
META_PATH_SCHEMA_TEMPLATE,
schemaTemplate);
}
}

170
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/ServiceMetaManager.java Normal file
View File

@ -0,0 +1,170 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.managers;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_ADD;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_DELIMITER;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_EVENT;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_GRAPHSPACE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_HUGEGRAPH;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_JOIN;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_REMOVE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_SERVICE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_SERVICE_CONF;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_UPDATE;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
import org.apache.hugegraph.meta.MetaDriver;
import org.apache.hugegraph.space.Service;
import org.apache.hugegraph.util.JsonUtil;
import org.apache.logging.log4j.util.Strings;
public class ServiceMetaManager extends AbstractMetaManager {
public ServiceMetaManager(MetaDriver metaDriver, String cluster) {
super(metaDriver, cluster);
}
private static String serviceName(String graphSpace, String name) {
return String.join(META_PATH_JOIN, graphSpace, name);
}
public Map<String, Service> serviceConfigs(String graphSpace) {
Map<String, Service> serviceMap = new HashMap<>();
Map<String, String> keyValues = this.metaDriver.scanWithPrefix(
this.serviceConfPrefix(graphSpace));
for (Map.Entry<String, String> entry : keyValues.entrySet()) {
String key = entry.getKey();
String[] parts = key.split(META_PATH_DELIMITER);
serviceMap.put(parts[parts.length - 1],
JsonUtil.fromJson(entry.getValue(), Service.class));
}
return serviceMap;
}
public String getServiceRawConfig(String graphSpace, String service) {
return this.metaDriver.get(this.serviceConfKey(graphSpace, service));
}
public Service getServiceConfig(String graphSpace, String service) {
String s = this.getServiceRawConfig(graphSpace, service);
return this.parseServiceRawConfig(s);
}
public Service parseServiceRawConfig(String serviceRawConf) {
return JsonUtil.fromJson(serviceRawConf, Service.class);
}
public void notifyServiceAdd(String graphSpace, String name) {
this.metaDriver.put(this.serviceAddKey(),
serviceName(graphSpace, name));
}
public void notifyServiceRemove(String graphSpace, String name) {
this.metaDriver.put(this.serviceRemoveKey(),
serviceName(graphSpace, name));
}
public void notifyServiceUpdate(String graphSpace, String name) {
this.metaDriver.put(this.serviceUpdateKey(),
serviceName(graphSpace, name));
}
public Service service(String graphSpace, String name) {
String service = this.metaDriver.get(this.serviceConfKey(graphSpace,
name));
if (StringUtils.isEmpty(service)) {
return null;
}
return JsonUtil.fromJson(service, Service.class);
}
public void addServiceConfig(String graphSpace, Service service) {
this.metaDriver.put(this.serviceConfKey(graphSpace, service.name()),
JsonUtil.toJson(service));
}
public void removeServiceConfig(String graphSpace, String service) {
this.metaDriver.delete(this.serviceConfKey(graphSpace, service));
}
public void updateServiceConfig(String graphSpace, Service service) {
this.addServiceConfig(graphSpace, service);
}
public <T> void listenServiceAdd(Consumer<T> consumer) {
this.listen(this.serviceAddKey(), consumer);
}
public <T> void listenServiceRemove(Consumer<T> consumer) {
this.listen(this.serviceRemoveKey(), consumer);
}
public <T> void listenServiceUpdate(Consumer<T> consumer) {
this.listen(this.serviceUpdateKey(), consumer);
}
private String serviceConfPrefix(String graphSpace) {
return this.serviceConfKey(graphSpace, Strings.EMPTY);
}
private String serviceAddKey() {
// HUGEGRAPH/{cluster}/EVENT/SERVICE/ADD
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_SERVICE,
META_PATH_ADD);
}
private String serviceRemoveKey() {
// HUGEGRAPH/{cluster}/EVENT/SERVICE/REMOVE
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_SERVICE,
META_PATH_REMOVE);
}
private String serviceUpdateKey() {
// HUGEGRAPH/{cluster}/EVENT/SERVICE/UPDATE
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_SERVICE,
META_PATH_UPDATE);
}
private String serviceConfKey(String graphSpace, String name) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/SERVICE_CONF
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
META_PATH_SERVICE_CONF,
name);
}
}

202
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/SpaceMetaManager.java Normal file
View File

@ -0,0 +1,202 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.managers;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_ADD;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_CONF;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_DELIMITER;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_EVENT;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_GRAPHSPACE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_GRAPHSPACE_LIST;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_HUGEGRAPH;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_REMOVE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_UPDATE;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
import org.apache.hugegraph.meta.MetaDriver;
import org.apache.hugegraph.space.GraphSpace;
import org.apache.hugegraph.type.define.CollectionType;
import org.apache.hugegraph.util.JsonUtil;
import org.apache.hugegraph.util.collection.CollectionFactory;
public class SpaceMetaManager extends AbstractMetaManager {
public SpaceMetaManager(MetaDriver metaDriver, String cluster) {
super(metaDriver, cluster);
}
public List<String> listGraphSpace() {
List<String> result = new ArrayList<>();
Map<String, String> graphSpaceMap = this.metaDriver.scanWithPrefix(
graphSpaceListKey());
for (Map.Entry<String, String> item : graphSpaceMap.entrySet()) {
result.add(item.getValue());
}
return result;
}
public Map<String, GraphSpace> graphSpaceConfigs() {
Map<String, String> keyValues = this.metaDriver.scanWithPrefix(
this.graphSpaceConfPrefix());
Map<String, GraphSpace> configs =
CollectionFactory.newMap(CollectionType.EC);
for (Map.Entry<String, String> entry : keyValues.entrySet()) {
String key = entry.getKey();
String[] parts = key.split(META_PATH_DELIMITER);
configs.put(parts[parts.length - 1],
JsonUtil.fromJson(entry.getValue(), GraphSpace.class));
}
return configs;
}
public GraphSpace graphSpace(String name) {
String space = this.metaDriver.get(this.graphSpaceConfKey(name));
if (StringUtils.isEmpty(space)) {
return null;
}
return JsonUtil.fromJson(space, GraphSpace.class);
}
public GraphSpace getGraphSpaceConfig(String graphSpace) {
String gs = this.metaDriver.get(this.graphSpaceConfKey(graphSpace));
if (StringUtils.isEmpty(gs)) {
return null;
}
return JsonUtil.fromJson(gs, GraphSpace.class);
}
public void addGraphSpaceConfig(String name, GraphSpace space) {
this.metaDriver.put(this.graphSpaceConfKey(name),
JsonUtil.toJson(space));
}
public void removeGraphSpaceConfig(String name) {
this.metaDriver.delete(this.graphSpaceConfKey(name));
}
public void updateGraphSpaceConfig(String name, GraphSpace space) {
this.metaDriver.put(this.graphSpaceConfKey(name),
JsonUtil.toJson(space));
}
public void appendGraphSpaceList(String name) {
String key = this.graphSpaceListKey(name);
this.metaDriver.put(key, name);
}
public void clearGraphSpaceList(String name) {
String key = this.graphSpaceListKey(name);
this.metaDriver.delete(key);
}
public <T> void listenGraphSpaceAdd(Consumer<T> consumer) {
this.listen(this.graphSpaceAddKey(), consumer);
}
public <T> void listenGraphSpaceRemove(Consumer<T> consumer) {
this.listen(this.graphSpaceRemoveKey(), consumer);
}
public <T> void listenGraphSpaceUpdate(Consumer<T> consumer) {
this.listen(this.graphSpaceUpdateKey(), consumer);
}
public void notifyGraphSpaceAdd(String graphSpace) {
this.metaDriver.put(this.graphSpaceAddKey(), graphSpace);
}
public void notifyGraphSpaceRemove(String graphSpace) {
this.metaDriver.put(this.graphSpaceRemoveKey(), graphSpace);
}
public void notifyGraphSpaceUpdate(String graphSpace) {
this.metaDriver.put(this.graphSpaceUpdateKey(), graphSpace);
}
private String graphSpaceConfPrefix() {
// HUGEGRAPH/{cluster}/GRAPHSPACE/CONF
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
META_PATH_CONF);
}
private String graphSpaceAddKey() {
// HUGEGRAPH/{cluster}/EVENT/GRAPHSPACE/ADD
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_GRAPHSPACE,
META_PATH_ADD);
}
private String graphSpaceRemoveKey() {
// HUGEGRAPH/{cluster}/EVENT/GRAPHSPACE/REMOVE
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_GRAPHSPACE,
META_PATH_REMOVE);
}
private String graphSpaceUpdateKey() {
// HUGEGRAPH/{cluster}/EVENT/GRAPHSPACE/UPDATE
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_EVENT,
META_PATH_GRAPHSPACE,
META_PATH_UPDATE);
}
private String graphSpaceConfKey(String name) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/CONF/{graphspace}
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
META_PATH_CONF,
name);
}
private String graphSpaceListKey(String name) {
// HUGEGRAPH/{cluster}/GRAPHSPACE_LIST/{graphspace}
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE_LIST,
name);
}
private String graphSpaceListKey() {
// HUGEGRAPH/{cluster}/GRAPHSPACE_LIST
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE_LIST);
}
}

76
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/TaskMetaManager.java Normal file
View File

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.meta.managers;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_DELIMITER;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_GRAPHSPACE;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_HUGEGRAPH;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_TASK;
import static org.apache.hugegraph.meta.MetaManager.META_PATH_TASK_LOCK;
import org.apache.hugegraph.meta.MetaDriver;
import org.apache.hugegraph.meta.lock.LockResult;
public class TaskMetaManager extends AbstractMetaManager {
private static final String TASK_STATUS_POSTFIX = "Status";
private static final String TASK_PROGRESS_POSTFIX = "Progress";
private static final String TASK_CONTEXT_POSTFIX = "Context";
private static final String TASK_RETRY_POSTFIX = "Retry";
public TaskMetaManager(MetaDriver metaDriver, String cluster) {
super(metaDriver, cluster);
}
public LockResult tryLockTask(String graphSpace, String graphName,
String taskId) {
String key = taskLockKey(graphSpace, graphName, taskId);
return this.tryLock(key);
}
public boolean isLockedTask(String graphSpace, String graphName,
String taskId) {
String key = taskLockKey(graphSpace, graphName, taskId);
// 判断当前任务是否锁定
return metaDriver.isLocked(key);
}
public void unlockTask(String graphSpace, String graphName,
String taskId, LockResult lockResult) {
String key = taskLockKey(graphSpace, graphName, taskId);
this.unlock(key, lockResult);
}
private String taskLockKey(String graphSpace,
String graphName,
String taskId) {
// HUGEGRAPH/{cluster}/GRAPHSPACE/{graphSpace}/{graphName}/TASK/{id}/TASK_LOCK
return String.join(META_PATH_DELIMITER,
META_PATH_HUGEGRAPH,
this.cluster,
META_PATH_GRAPHSPACE,
graphSpace,
graphName,
META_PATH_TASK,
taskId,
META_PATH_TASK_LOCK);
}
}

176
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/EdgeLabel.java
View File

@ -20,14 +20,21 @@ package org.apache.hugegraph.schema;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.schema.builder.SchemaBuilder;
import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.type.define.EdgeLabelType;
import org.apache.hugegraph.type.define.Frequency;
import org.apache.hugegraph.type.define.SchemaStatus;
import org.apache.hugegraph.util.E;
import com.google.common.base.Objects;
@ -39,6 +46,7 @@ public class EdgeLabel extends SchemaLabel {
private Id targetLabel = NONE_ID;
private Frequency frequency;
private List<Id> sortKeys;
private EdgeLabelType edgeLabelType;
public EdgeLabel(final HugeGraph graph, Id id, String name) {
super(graph, id, name);
@ -55,6 +63,10 @@ public class EdgeLabel extends SchemaLabel {
return this.frequency;
}
public void edgeLabelType(EdgeLabelType type) {
this.edgeLabelType = type;
}
public void frequency(Frequency frequency) {
this.frequency = frequency;
}
@ -164,4 +176,168 @@ public class EdgeLabel extends SchemaLabel {
Builder userdata(Map<String, Object> userdata);
}
@Override
public Map<String, Object> asMap() {
Map<String, Object> map = new HashMap<>();
if (this.sourceLabel() != null && this.sourceLabel() != NONE_ID) {
map.put(P.SOURCE_LABEL, this.sourceLabel().asString());
}
if (this.targetLabel() != null && this.targetLabel() != NONE_ID) {
map.put(P.TARGET_LABEL, this.targetLabel().asString());
}
if (this.properties() != null) {
map.put(P.PROPERTIES, this.properties());
}
if (this.nullableKeys() != null) {
map.put(P.NULLABLE_KEYS, this.nullableKeys());
}
if (this.indexLabels() != null) {
map.put(P.INDEX_LABELS, this.indexLabels());
}
if (this.ttlStartTime() != null) {
map.put(P.TT_START_TIME, this.ttlStartTime().asString());
}
if (this.sortKeys() != null) {
map.put(P.SORT_KEYS, this.sortKeys);
}
//map.put(P.EDGELABEL_TYPE, this.edgeLabelType);
//if (this.fatherId() != null) {
// map.put(P.FATHER_ID, this.fatherId().asString());
//}
map.put(P.ENABLE_LABEL_INDEX, this.enableLabelIndex());
map.put(P.TTL, String.valueOf(this.ttl()));
//map.put(P.LINKS, this.links());
map.put(P.FREQUENCY, this.frequency().toString());
return super.asMap(map);
}
@SuppressWarnings("unchecked")
public static EdgeLabel fromMap(Map<String, Object> map, HugeGraph graph) {
Id id = IdGenerator.of((int) map.get(EdgeLabel.P.ID));
String name = (String) map.get(EdgeLabel.P.NAME);
EdgeLabel edgeLabel = new EdgeLabel(graph, id, name);
for (Map.Entry<String, Object> entry : map.entrySet()) {
switch (entry.getKey()) {
case P.ID:
case P.NAME:
break;
case P.STATUS:
edgeLabel.status(
SchemaStatus.valueOf(((String) entry.getValue()).toUpperCase()));
break;
case P.USERDATA:
edgeLabel.userdata(new Userdata((Map<String, Object>) entry.getValue()));
break;
case P.PROPERTIES:
Set<Id> ids = ((List<Integer>) entry.getValue()).stream().map(
IdGenerator::of).collect(Collectors.toSet());
edgeLabel.properties(ids);
break;
case P.NULLABLE_KEYS:
ids = ((List<Integer>) entry.getValue()).stream().map(
IdGenerator::of).collect(Collectors.toSet());
edgeLabel.nullableKeys(ids);
break;
case P.INDEX_LABELS:
ids = ((List<Integer>) entry.getValue()).stream().map(
IdGenerator::of).collect(Collectors.toSet());
edgeLabel.addIndexLabels(ids.toArray(new Id[0]));
break;
case P.ENABLE_LABEL_INDEX:
boolean enableLabelIndex = (Boolean) entry.getValue();
edgeLabel.enableLabelIndex(enableLabelIndex);
break;
case P.TTL:
long ttl = Long.parseLong((String) entry.getValue());
edgeLabel.ttl(ttl);
break;
case P.TT_START_TIME:
long ttlStartTime =
Long.parseLong((String) entry.getValue());
edgeLabel.ttlStartTime(IdGenerator.of(ttlStartTime));
break;
//case P.LINKS:
// // TODO: serialize and deserialize
// List<Map> list = (List<Map>) entry.getValue();
// for (Map m : list) {
// for (Object key : m.keySet()) {
// Id sid = IdGenerator.of(Long.parseLong((String) key));
// Id tid = IdGenerator.of(Long.parseLong(String.valueOf(m.get(key))));
// edgeLabel.links(Pair.of(sid, tid));
// }
// }
// break;
case P.SOURCE_LABEL:
long sourceLabel =
Long.parseLong((String) entry.getValue());
edgeLabel.sourceLabel(IdGenerator.of(sourceLabel));
break;
case P.TARGET_LABEL:
long targetLabel =
Long.parseLong((String) entry.getValue());
edgeLabel.targetLabel(IdGenerator.of(targetLabel));
break;
//case P.FATHER_ID:
// long fatherId =
// Long.parseLong((String) entry.getValue());
// edgeLabel.fatherId(IdGenerator.of(fatherId));
// break;
//case P.EDGELABEL_TYPE:
// EdgeLabelType edgeLabelType =
// EdgeLabelType.valueOf(
// ((String) entry.getValue()).toUpperCase());
// edgeLabel.edgeLabelType(edgeLabelType);
// break;
case P.FREQUENCY:
Frequency frequency =
Frequency.valueOf(((String) entry.getValue()).toUpperCase());
edgeLabel.frequency(frequency);
break;
case P.SORT_KEYS:
ids = ((List<Integer>) entry.getValue()).stream().map(
IdGenerator::of).collect(Collectors.toSet());
edgeLabel.sortKeys(ids.toArray(new Id[0]));
break;
default:
throw new AssertionError(String.format(
"Invalid key '%s' for edge label",
entry.getKey()));
}
}
return edgeLabel;
}
public static final class P {
public static final String ID = "id";
public static final String NAME = "name";
public static final String STATUS = "status";
public static final String USERDATA = "userdata";
public static final String PROPERTIES = "properties";
public static final String NULLABLE_KEYS = "nullableKeys";
public static final String INDEX_LABELS = "indexLabels";
public static final String ENABLE_LABEL_INDEX = "enableLabelIndex";
public static final String TTL = "ttl";
public static final String TT_START_TIME = "ttlStartTime";
public static final String LINKS = "links";
public static final String SOURCE_LABEL = "sourceLabel";
public static final String TARGET_LABEL = "targetLabel";
public static final String EDGELABEL_TYPE = "edgeLabelType";
public static final String FATHER_ID = "fatherId";
public static final String FREQUENCY = "frequency";
public static final String SORT_KEYS = "sortKeys";
}
}

75
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/IndexLabel.java
View File

@ -20,8 +20,10 @@ package org.apache.hugegraph.schema;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
@ -29,6 +31,7 @@ import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.schema.builder.SchemaBuilder;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.type.define.IndexType;
import org.apache.hugegraph.type.define.SchemaStatus;
import org.apache.hugegraph.util.E;
import com.google.common.base.Objects;
@ -164,6 +167,8 @@ public class IndexLabel extends SchemaElement {
public static IndexLabel label(HugeType type) {
switch (type) {
case TASK:
case SERVER:
case VERTEX:
return VL_IL;
case EDGE:
@ -280,4 +285,74 @@ public class IndexLabel extends SchemaElement {
Builder rebuild(boolean rebuild);
}
@Override
public Map<String, Object> asMap() {
HashMap<String, Object> map = new HashMap<>();
map.put(P.BASE_TYPE, this.baseType().name());
map.put(P.BASE_VALUE, this.baseValue().asString());
map.put(P.INDEX_TYPE, this.indexType().name());
map.put(P.INDEX_FIELDS, this.indexFields());
return super.asMap(map);
}
@SuppressWarnings("unchecked")
public static IndexLabel fromMap(Map<String, Object> map, HugeGraph graph) {
Id id = IdGenerator.of((int) map.get(IndexLabel.P.ID));
String name = (String) map.get(IndexLabel.P.NAME);
IndexLabel indexLabel = new IndexLabel(graph, id, name);
for (Map.Entry<String, Object> entry : map.entrySet()) {
switch (entry.getKey()) {
case P.ID:
case P.NAME:
break;
case P.STATUS:
indexLabel.status(
SchemaStatus.valueOf(((String) entry.getValue()).toUpperCase()));
break;
case P.USERDATA:
indexLabel.userdata(new Userdata((Map<String, Object>) entry.getValue()));
break;
case P.BASE_TYPE:
HugeType hugeType =
HugeType.valueOf(((String) entry.getValue()).toUpperCase());
indexLabel.baseType(hugeType);
break;
case P.BASE_VALUE:
long sourceLabel =
Long.parseLong((String) entry.getValue());
indexLabel.baseValue(IdGenerator.of(sourceLabel));
break;
case P.INDEX_TYPE:
IndexType indexType =
IndexType.valueOf(((String) entry.getValue()).toUpperCase());
indexLabel.indexType(indexType);
break;
case P.INDEX_FIELDS:
List<Id> ids = ((List<Integer>) entry.getValue()).stream().map(
IdGenerator::of).collect(Collectors.toList());
indexLabel.indexFields(ids.toArray(new Id[0]));
break;
default:
throw new AssertionError(String.format(
"Invalid key '%s' for index label",
entry.getKey()));
}
}
return indexLabel;
}
public static final class P {
public static final String ID = "id";
public static final String NAME = "name";
public static final String STATUS = "status";
public static final String USERDATA = "userdata";
public static final String BASE_TYPE = "baseType";
public static final String BASE_VALUE = "baseValue";
public static final String INDEX_TYPE = "indexType";
public static final String INDEX_FIELDS = "indexFields";
}
}

84
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/PropertyKey.java
View File

@ -20,6 +20,7 @@ package org.apache.hugegraph.schema;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@ -28,6 +29,7 @@ import java.util.Set;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.HugeException;
import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.exception.NotSupportException;
import org.apache.hugegraph.schema.builder.SchemaBuilder;
import org.apache.hugegraph.type.HugeType;
@ -35,6 +37,7 @@ import org.apache.hugegraph.type.Propertiable;
import org.apache.hugegraph.type.define.AggregateType;
import org.apache.hugegraph.type.define.Cardinality;
import org.apache.hugegraph.type.define.DataType;
import org.apache.hugegraph.type.define.SchemaStatus;
import org.apache.hugegraph.type.define.WriteType;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.LongEncoding;
@ -409,4 +412,85 @@ public class PropertyKey extends SchemaElement implements Propertiable {
Builder userdata(Map<String, Object> userdata);
}
@Override
public Map<String, Object> asMap() {
Map<String, Object> map = new HashMap<>();
if (this.dataType != null) {
map.put(P.DATA_TYPE, this.dataType.string());
}
if (this.cardinality != null) {
map.put(P.CARDINALITY, this.cardinality.string());
}
if (this.aggregateType != null) {
map.put(P.AGGREGATE_TYPE, this.aggregateType.string());
}
if (this.writeType != null) {
map.put(P.WRITE_TYPE, this.writeType.string());
}
return super.asMap(map);
}
@SuppressWarnings("unchecked")
public static PropertyKey fromMap(Map<String, Object> map, HugeGraph graph) {
Id id = IdGenerator.of((int) map.get(P.ID));
String name = (String) map.get(P.NAME);
PropertyKey propertyKey = new PropertyKey(graph, id, name);
for (Map.Entry<String, Object> entry : map.entrySet()) {
switch (entry.getKey()) {
case P.ID:
case P.NAME:
break;
case P.STATUS:
propertyKey.status(
SchemaStatus.valueOf(((String) entry.getValue()).toUpperCase()));
break;
case P.USERDATA:
propertyKey.userdata(new Userdata((Map<String, Object>) entry.getValue()));
break;
case P.AGGREGATE_TYPE:
propertyKey.aggregateType(
AggregateType.valueOf(((String) entry.getValue()).toUpperCase()));
break;
case P.WRITE_TYPE:
propertyKey.writeType(
WriteType.valueOf(((String) entry.getValue()).toUpperCase()));
break;
case P.DATA_TYPE:
propertyKey.dataType(
DataType.valueOf(((String) entry.getValue()).toUpperCase()));
break;
case P.CARDINALITY:
propertyKey.cardinality(
Cardinality.valueOf(((String) entry.getValue()).toUpperCase()));
break;
default:
throw new AssertionError(String.format(
"Invalid key '%s' for property key",
entry.getKey()));
}
}
return propertyKey;
}
public static final class P {
public static final String ID = "id";
public static final String NAME = "name";
public static final String STATUS = "status";
public static final String USERDATA = "userdata";
public static final String DATA_TYPE = "data_type";
public static final String CARDINALITY = "cardinality";
public static final String AGGREGATE_TYPE = "aggregate_type";
public static final String WRITE_TYPE = "write_type";
}
}

32
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/SchemaElement.java
View File

@ -47,6 +47,11 @@ public abstract class SchemaElement implements Nameable, Typeable,
protected static final int ILN_IL_ID = -6;
protected static final int OLAP_VL_ID = -7;
// OLAP_ID means all of vertex label ids
public static final Id OLAP_ID = IdGenerator.of(-7);
// OLAP means all of vertex label names
public static final String OLAP = "~olap";
public static final Id NONE_ID = IdGenerator.ZERO;
public static final String UNDEF = "~undefined";
@ -217,4 +222,31 @@ public abstract class SchemaElement implements Nameable, Typeable,
return this.task;
}
}
public abstract Map<String, Object> asMap();
public Map<String, Object> asMap(Map<String, Object> map) {
E.checkState(this.id != null,
"Property key id can't be null");
E.checkState(this.name != null,
"Property key name can't be null");
E.checkState(this.status != null,
"Property status can't be null");
map.put(P.ID, this.id);
map.put(P.NAME, this.name);
map.put(P.STATUS, this.status.string());
map.put(P.USERDATA, this.userdata);
return map;
}
public static final class P {
public static final String ID = "id";
public static final String NAME = "name";
public static final String STATUS = "status";
public static final String USERDATA = "userdata";
}
}

5
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/SchemaManager.java
View File

@ -20,6 +20,7 @@ package org.apache.hugegraph.schema;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.tinkerpop.gremlin.structure.Graph;
@ -34,10 +35,10 @@ import org.apache.hugegraph.util.E;
public class SchemaManager {
private final SchemaTransaction transaction;
private final ISchemaTransaction transaction;
private HugeGraph graph;
public SchemaManager(SchemaTransaction transaction, HugeGraph graph) {
public SchemaManager(ISchemaTransaction transaction, HugeGraph graph) {
E.checkNotNull(transaction, "transaction");
E.checkNotNull(graph, "graph");
this.transaction = transaction;

112
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/VertexLabel.java
View File

@ -20,8 +20,11 @@ package org.apache.hugegraph.schema;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
@ -29,6 +32,8 @@ import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.schema.builder.SchemaBuilder;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.type.define.IdStrategy;
import org.apache.hugegraph.type.define.SchemaStatus;
import com.google.common.base.Objects;
public class VertexLabel extends SchemaLabel {
@ -132,4 +137,111 @@ public class VertexLabel extends SchemaLabel {
Builder userdata(Map<String, Object> userdata);
}
@Override
public Map<String, Object> asMap() {
HashMap<String, Object> map = new HashMap();
map.put(P.PROPERTIES, this.properties());
map.put(P.NULLABLE_KEYS, this.nullableKeys());
map.put(P.INDEX_LABELS, this.indexLabels());
map.put(P.ENABLE_LABEL_INDEX, this.enableLabelIndex());
map.put(P.TTL, String.valueOf(this.ttl()));
map.put(P.TT_START_TIME, this.ttlStartTime().asString());
map.put(P.ID_STRATEGY, this.idStrategy().string());
map.put(P.PRIMARY_KEYS, this.primaryKeys());
return super.asMap(map);
}
@SuppressWarnings("unchecked")
public static VertexLabel fromMap(Map<String, Object> map, HugeGraph graph) {
Id id = IdGenerator.of((int) map.get(VertexLabel.P.ID));
String name = (String) map.get(VertexLabel.P.NAME);
VertexLabel vertexLabel = new VertexLabel(graph, id, name);
for (Map.Entry<String, Object> entry : map.entrySet()) {
switch (entry.getKey()) {
case P.ID:
case P.NAME:
break;
case P.STATUS:
vertexLabel.status(
SchemaStatus.valueOf(((String) entry.getValue()).toUpperCase()));
break;
case P.USERDATA:
vertexLabel.userdata(new Userdata((Map<String, Object>) entry.getValue()));
break;
case P.PROPERTIES:
Set<Id> ids = ((List<Integer>) entry.getValue()).stream().map(
IdGenerator::of).collect(Collectors.toSet());
vertexLabel.properties(ids);
break;
case P.NULLABLE_KEYS:
ids = ((List<Integer>) entry.getValue()).stream().map(
IdGenerator::of).collect(Collectors.toSet());
vertexLabel.nullableKeys(ids);
break;
case P.INDEX_LABELS:
ids = ((List<Integer>) entry.getValue()).stream().map(
IdGenerator::of).collect(Collectors.toSet());
vertexLabel.addIndexLabels(ids.toArray(new Id[0]));
break;
case P.ENABLE_LABEL_INDEX:
boolean enableLabelIndex = (Boolean) entry.getValue();
vertexLabel.enableLabelIndex(enableLabelIndex);
break;
case P.TTL:
long ttl = Long.parseLong((String) entry.getValue());
vertexLabel.ttl(ttl);
break;
case P.TT_START_TIME:
long ttlStartTime =
Long.parseLong((String) entry.getValue());
vertexLabel.ttlStartTime(IdGenerator.of(ttlStartTime));
break;
case P.ID_STRATEGY:
IdStrategy idStrategy =
IdStrategy.valueOf(((String) entry.getValue()).toUpperCase());
vertexLabel.idStrategy(idStrategy);
break;
case P.PRIMARY_KEYS:
ids = ((List<Integer>) entry.getValue()).stream().map(
IdGenerator::of).collect(Collectors.toSet());
vertexLabel.primaryKeys(ids.toArray(new Id[0]));
break;
default:
throw new AssertionError(String.format(
"Invalid key '%s' for vertex label",
entry.getKey()));
}
}
return vertexLabel;
}
public static final class P {
public static final String ID = "id";
public static final String NAME = "name";
public static final String STATUS = "status";
public static final String USERDATA = "userdata";
public static final String PROPERTIES = "properties";
public static final String NULLABLE_KEYS = "nullableKeys";
public static final String INDEX_LABELS = "indexLabels";
public static final String ENABLE_LABEL_INDEX = "enableLabelIndex";
public static final String TTL = "ttl";
public static final String TT_START_TIME = "ttlStartTime";
public static final String ID_STRATEGY = "idStrategy";
public static final String PRIMARY_KEYS = "primaryKeys";
}
}

5
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/AbstractBuilder.java
View File

@ -22,6 +22,7 @@ import java.util.function.Function;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.schema.IndexLabel;
import org.apache.hugegraph.schema.PropertyKey;
@ -38,10 +39,10 @@ import org.apache.hugegraph.util.LockUtil;
public abstract class AbstractBuilder {
private final SchemaTransaction transaction;
private final ISchemaTransaction transaction;
private final HugeGraph graph;
public AbstractBuilder(SchemaTransaction transaction, HugeGraph graph) {
public AbstractBuilder(ISchemaTransaction transaction, HugeGraph graph) {
E.checkNotNull(transaction, "transaction");
E.checkNotNull(graph, "graph");
this.transaction = transaction;

5
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/EdgeLabelBuilder.java
View File

@ -29,6 +29,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.schema.PropertyKey;
import org.apache.hugegraph.schema.Userdata;
@ -62,7 +63,7 @@ public class EdgeLabelBuilder extends AbstractBuilder
private Userdata userdata;
private boolean checkExist;
public EdgeLabelBuilder(SchemaTransaction transaction,
public EdgeLabelBuilder(ISchemaTransaction transaction,
HugeGraph graph, String name) {
super(transaction, graph);
E.checkNotNull(name, "name");
@ -81,7 +82,7 @@ public class EdgeLabelBuilder extends AbstractBuilder
this.checkExist = true;
}
public EdgeLabelBuilder(SchemaTransaction transaction,
public EdgeLabelBuilder(ISchemaTransaction transaction,
HugeGraph graph, EdgeLabel copy) {
super(transaction, graph);
E.checkNotNull(copy, "copy");

5
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/IndexLabelBuilder.java
View File

@ -27,6 +27,7 @@ import java.util.function.BiPredicate;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.HugeException;
import org.apache.hugegraph.HugeGraph;
@ -64,7 +65,7 @@ public class IndexLabelBuilder extends AbstractBuilder
private boolean checkExist;
private boolean rebuild;
public IndexLabelBuilder(SchemaTransaction transaction,
public IndexLabelBuilder(ISchemaTransaction transaction,
HugeGraph graph, String name) {
super(transaction, graph);
E.checkNotNull(name, "name");
@ -79,7 +80,7 @@ public class IndexLabelBuilder extends AbstractBuilder
this.rebuild = true;
}
public IndexLabelBuilder(SchemaTransaction transaction,
public IndexLabelBuilder(ISchemaTransaction transaction,
HugeGraph graph, IndexLabel copy) {
super(transaction, graph);
E.checkNotNull(copy, "copy");

5
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/PropertyKeyBuilder.java
View File

@ -22,6 +22,7 @@ import java.util.concurrent.TimeoutException;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.schema.PropertyKey;
import org.apache.hugegraph.schema.SchemaElement;
@ -53,7 +54,7 @@ public class PropertyKeyBuilder extends AbstractBuilder
private boolean checkExist;
private Userdata userdata;
public PropertyKeyBuilder(SchemaTransaction transaction,
public PropertyKeyBuilder(ISchemaTransaction transaction,
HugeGraph graph, String name) {
super(transaction, graph);
E.checkNotNull(name, "name");
@ -67,7 +68,7 @@ public class PropertyKeyBuilder extends AbstractBuilder
this.checkExist = true;
}
public PropertyKeyBuilder(SchemaTransaction transaction,
public PropertyKeyBuilder(ISchemaTransaction transaction,
HugeGraph graph, PropertyKey copy) {
super(transaction, graph);
E.checkNotNull(copy, "copy");

5
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/schema/builder/VertexLabelBuilder.java
View File

@ -29,6 +29,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.backend.tx.ISchemaTransaction;
import org.apache.hugegraph.backend.tx.SchemaTransaction;
import org.apache.hugegraph.schema.PropertyKey;
import org.apache.hugegraph.schema.Userdata;
@ -59,7 +60,7 @@ public class VertexLabelBuilder extends AbstractBuilder
private Userdata userdata;
private boolean checkExist;
public VertexLabelBuilder(SchemaTransaction transaction,
public VertexLabelBuilder(ISchemaTransaction transaction,
HugeGraph graph, String name) {
super(transaction, graph);
E.checkNotNull(name, "name");
@ -76,7 +77,7 @@ public class VertexLabelBuilder extends AbstractBuilder
this.checkExist = true;
}
public VertexLabelBuilder(SchemaTransaction transaction,
public VertexLabelBuilder(ISchemaTransaction transaction,
HugeGraph graph, VertexLabel copy) {
super(transaction, graph);
E.checkNotNull(copy, "copy");

512
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/space/GraphSpace.java Normal file
View File

@ -0,0 +1,512 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.space;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.hugegraph.HugeException;
import org.apache.hugegraph.util.E;
public class GraphSpace {
public static final String DEFAULT_GRAPH_SPACE_SERVICE_NAME = "DEFAULT";
public static final String DEFAULT_NICKNAME = "默认图空间";
public static final String DEFAULT_GRAPH_SPACE_DESCRIPTION =
"The system default graph space";
public static final String DEFAULT_CREATOR_NAME = "anonymous";
public static final int DEFAULT_CPU_LIMIT = 4;
public static final int DEFAULT_MEMORY_LIMIT = 8;
public static final int DEFAULT_STORAGE_LIMIT = 100;
public static final int DEFAULT_MAX_GRAPH_NUMBER = 100;
public static final int DEFAULT_MAX_ROLE_NUMBER = 100;
private final String creator;
public int storageLimit; // GB
public String oltpNamespace;
private String name;
private String nickname;
private String description;
private int cpuLimit;
private int memoryLimit; // GB
private int computeCpuLimit;
private int computeMemoryLimit; // GB
private String olapNamespace;
private String storageNamespace;
private int maxGraphNumber;
private int maxRoleNumber;
private Boolean auth;
private Map<String, Object> configs;
private int cpuUsed;
private int memoryUsed; // GB
private int storageUsed; // GB
private int graphNumberUsed;
private int roleNumberUsed;
private String operatorImagePath = ""; // path of compute operator image
private String internalAlgorithmImageUrl = "";
private Date createTime;
private Date updateTime;
public GraphSpace(String name) {
E.checkArgument(name != null && !StringUtils.isEmpty(name),
"The name of graph space can't be null or empty");
this.name = name;
this.nickname = DEFAULT_NICKNAME;
this.maxGraphNumber = DEFAULT_MAX_GRAPH_NUMBER;
this.maxRoleNumber = DEFAULT_MAX_ROLE_NUMBER;
this.cpuLimit = DEFAULT_CPU_LIMIT;
this.memoryLimit = DEFAULT_MEMORY_LIMIT;
this.storageLimit = DEFAULT_STORAGE_LIMIT;
this.computeCpuLimit = DEFAULT_CPU_LIMIT;
this.computeMemoryLimit = DEFAULT_MEMORY_LIMIT;
this.auth = false;
this.creator = DEFAULT_CREATOR_NAME;
this.configs = new HashMap<>();
}
public GraphSpace(String name, String nickname, String description,
int cpuLimit,
int memoryLimit, int storageLimit, int maxGraphNumber,
int maxRoleNumber, boolean auth, String creator,
Map<String, Object> config) {
E.checkArgument(name != null && !StringUtils.isEmpty(name),
"The name of graph space can't be null or empty");
E.checkArgument(cpuLimit > 0, "The cpu limit must > 0");
E.checkArgument(memoryLimit > 0, "The memory limit must > 0");
E.checkArgument(storageLimit > 0, "The storage limit must > 0");
E.checkArgument(maxGraphNumber > 0, "The max graph number must > 0");
this.name = name;
this.nickname = nickname;
this.description = description;
this.cpuLimit = cpuLimit;
this.memoryLimit = memoryLimit;
this.storageLimit = storageLimit;
this.maxGraphNumber = maxGraphNumber;
this.maxRoleNumber = maxRoleNumber;
this.auth = auth;
if (config == null) {
this.configs = new HashMap<>();
} else {
this.configs = config;
}
this.createTime = new Date();
this.updateTime = this.createTime;
this.creator = creator;
}
public GraphSpace(String name, String nickname, String description,
int cpuLimit,
int memoryLimit, int storageLimit, int maxGraphNumber,
int maxRoleNumber, String oltpNamespace,
String olapNamespace, String storageNamespace,
int cpuUsed, int memoryUsed, int storageUsed,
int graphNumberUsed, int roleNumberUsed,
boolean auth, String creator, Map<String, Object> config) {
E.checkArgument(name != null && !StringUtils.isEmpty(name),
"The name of graph space can't be null or empty");
E.checkArgument(cpuLimit > 0, "The cpu limit must > 0");
E.checkArgument(memoryLimit > 0, "The memory limit must > 0");
E.checkArgument(storageLimit > 0, "The storage limit must > 0");
E.checkArgument(maxGraphNumber > 0, "The max graph number must > 0");
this.name = name;
this.nickname = nickname;
this.description = description;
this.cpuLimit = cpuLimit;
this.memoryLimit = memoryLimit;
this.storageLimit = storageLimit;
this.maxGraphNumber = maxGraphNumber;
this.maxRoleNumber = maxRoleNumber;
this.oltpNamespace = oltpNamespace;
this.olapNamespace = olapNamespace;
this.storageNamespace = storageNamespace;
this.cpuUsed = cpuUsed;
this.memoryUsed = memoryUsed;
this.storageUsed = storageUsed;
this.graphNumberUsed = graphNumberUsed;
this.roleNumberUsed = roleNumberUsed;
this.auth = auth;
this.creator = creator;
this.configs = new HashMap<>();
if (config != null) {
this.configs = config;
}
}
public String name() {
return this.name;
}
public void name(String name) {
this.name = name;
}
public String nickname() {
return this.nickname;
}
public void nickname(String nickname) {
this.nickname = nickname;
}
public String description() {
return this.description;
}
public void description(String description) {
this.description = description;
}
public int cpuLimit() {
return this.cpuLimit;
}
public void cpuLimit(int cpuLimit) {
E.checkArgument(cpuLimit > 0,
"The cpu limit must be > 0, but got: %s", cpuLimit);
this.cpuLimit = cpuLimit;
}
public int memoryLimit() {
return this.memoryLimit;
}
public void memoryLimit(int memoryLimit) {
E.checkArgument(memoryLimit > 0,
"The memory limit must be > 0, but got: %s",
memoryLimit);
this.memoryLimit = memoryLimit;
}
public int storageLimit() {
return this.storageLimit;
}
public void storageLimit(int storageLimit) {
E.checkArgument(storageLimit > 0,
"The storage limit must be > 0, but got: %s",
storageLimit);
this.storageLimit = storageLimit;
}
public void setStorageUsed(int storageUsed) {
this.storageUsed = storageUsed;
}
public int computeCpuLimit() {
return this.computeCpuLimit;
}
public void computeCpuLimit(int computeCpuLimit) {
E.checkArgument(computeCpuLimit >= 0,
"The compute cpu limit must be >= 0, but got: %s", computeCpuLimit);
this.computeCpuLimit = computeCpuLimit;
}
public int computeMemoryLimit() {
return this.computeMemoryLimit;
}
public void computeMemoryLimit(int computeMemoryLimit) {
E.checkArgument(computeMemoryLimit >= 0,
"The compute memory limit must be >= 0, but got: %s",
computeMemoryLimit);
this.computeMemoryLimit = computeMemoryLimit;
}
public String oltpNamespace() {
return this.oltpNamespace;
}
public void oltpNamespace(String oltpNamespace) {
this.oltpNamespace = oltpNamespace;
}
public String olapNamespace() {
return this.olapNamespace;
}
public void olapNamespace(String olapNamespace) {
this.olapNamespace = olapNamespace;
}
public String storageNamespace() {
return this.storageNamespace;
}
public void storageNamespace(String storageNamespace) {
this.storageNamespace = storageNamespace;
}
public int maxGraphNumber() {
return this.maxGraphNumber;
}
public void maxGraphNumber(int maxGraphNumber) {
this.maxGraphNumber = maxGraphNumber;
}
public int maxRoleNumber() {
return this.maxRoleNumber;
}
public void maxRoleNumber(int maxRoleNumber) {
this.maxRoleNumber = maxRoleNumber;
}
public int graphNumberUsed() {
return this.graphNumberUsed;
}
public void graphNumberUsed(int graphNumberUsed) {
this.graphNumberUsed = graphNumberUsed;
}
public int roleNumberUsed() {
return this.roleNumberUsed;
}
public void roleNumberUsed(int roleNumberUsed) {
this.roleNumberUsed = roleNumberUsed;
}
public boolean auth() {
return this.auth;
}
public void auth(boolean auth) {
this.auth = auth;
}
public Map<String, Object> configs() {
return this.configs;
}
public void configs(Map<String, Object> configs) {
this.configs.putAll(configs);
}
public void operatorImagePath(String path) {
this.operatorImagePath = path;
}
public String operatorImagePath() {
return this.operatorImagePath;
}
public void internalAlgorithmImageUrl(String url) {
if (StringUtils.isNotBlank(url)) {
this.internalAlgorithmImageUrl = url;
}
}
public String internalAlgorithmImageUrl() {
return this.internalAlgorithmImageUrl;
}
public Date createTime() {
return this.createTime;
}
public Date updateTime() {
return this.updateTime;
}
public String creator() {
return this.creator;
}
public void updateTime(Date update) {
this.updateTime = update;
}
public void createTime(Date create) {
this.createTime = create;
}
public void refreshUpdate() {
this.updateTime = new Date();
}
public Map<String, Object> info() {
Map<String, Object> infos = new LinkedHashMap<>();
infos.put("name", this.name);
infos.put("nickname", this.nickname);
infos.put("description", this.description);
infos.put("cpu_limit", this.cpuLimit);
infos.put("memory_limit", this.memoryLimit);
infos.put("storage_limit", this.storageLimit);
infos.put("compute_cpu_limit", this.computeCpuLimit);
infos.put("compute_memory_limit", this.computeMemoryLimit);
infos.put("oltp_namespace", this.oltpNamespace);
infos.put("olap_namespace", this.olapNamespace);
infos.put("storage_namespace", this.storageNamespace);
infos.put("max_graph_number", this.maxGraphNumber);
infos.put("max_role_number", this.maxRoleNumber);
infos.putAll(this.configs);
// sources used info is not automatically updated, it could be
// updated by pdClient of GraphManager
infos.put("cpu_used", this.cpuUsed);
infos.put("memory_used", this.memoryUsed);
infos.put("storage_used", this.storageUsed);
float storageUserPercent = Float.parseFloat(
String.format("%.2f", (float) this.storageUsed /
((float) this.storageLimit * 1.0)));
infos.put("storage_percent", storageUserPercent);
infos.put("graph_number_used", this.graphNumberUsed);
infos.put("role_number_used", this.roleNumberUsed);
infos.put("auth", this.auth);
infos.put("operator_image_path", this.operatorImagePath);
infos.put("internal_algorithm_image_url", this.internalAlgorithmImageUrl);
infos.put("create_time", this.createTime);
infos.put("update_time", this.updateTime);
infos.put("creator", this.creator);
return infos;
}
private synchronized void incrCpuUsed(int acquiredCount) {
if (acquiredCount < 0) {
throw new HugeException("cannot increase cpu used since acquired count is negative");
}
this.cpuUsed += acquiredCount;
}
private synchronized void decrCpuUsed(int releasedCount) {
if (releasedCount < 0) {
throw new HugeException("cannot decrease cpu used since released count is negative");
}
if (cpuUsed < releasedCount) {
cpuUsed = 0;
} else {
this.cpuUsed -= releasedCount;
}
}
private synchronized void incrMemoryUsed(int acquiredCount) {
if (acquiredCount < 0) {
throw new HugeException("cannot increase memory used since acquired count is negative");
}
this.memoryUsed += acquiredCount;
}
private synchronized void decrMemoryUsed(int releasedCount) {
if (releasedCount < 0) {
throw new HugeException("cannot decrease memory used since released count is negative");
}
if (memoryUsed < releasedCount) {
this.memoryUsed = 0;
} else {
this.memoryUsed -= releasedCount;
}
}
/**
* Only limit the resource usage for oltp service under k8s
*
* @param service
* @return
*/
public boolean tryOfferResourceFor(Service service) {
if (!service.k8s()) {
return true;
}
int count = service.count();
int leftCpu = this.cpuLimit - this.cpuUsed;
int leftMemory = this.memoryLimit - this.memoryUsed;
int acquiredCpu = service.cpuLimit() * count;
int acquiredMemory = service.memoryLimit() * count;
if (acquiredCpu > leftCpu ||
acquiredMemory > leftMemory) {
return false;
}
this.incrCpuUsed(acquiredCpu);
this.incrMemoryUsed(acquiredMemory);
return true;
}
public void recycleResourceFor(Service service) {
int count = service.count();
this.decrCpuUsed(service.cpuLimit() * count);
this.decrMemoryUsed(service.memoryLimit() * count);
}
public boolean tryOfferGraph() {
return this.tryOfferGraph(1);
}
public boolean tryOfferGraph(int count) {
if (this.graphNumberUsed + count > this.maxGraphNumber) {
return false;
}
this.graphNumberUsed += count;
return true;
}
public void recycleGraph() {
this.recycleGraph(1);
}
public void recycleGraph(int count) {
this.graphNumberUsed -= count;
}
public boolean tryOfferRole() {
return this.tryOfferRole(1);
}
public boolean tryOfferRole(int count) {
if (this.roleNumberUsed + count > this.maxRoleNumber) {
return false;
}
this.roleNumberUsed += count;
return true;
}
public void recycleRole() {
this.recycleRole(1);
}
public void recycleRole(int count) {
this.roleNumberUsed -= count;
}
}

140
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/space/SchemaTemplate.java Normal file
View File

@ -0,0 +1,140 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.space;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Map;
import org.apache.hugegraph.util.E;
import com.google.common.collect.ImmutableMap;
public class SchemaTemplate {
public static SimpleDateFormat FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
protected Date createTime;
protected Date updateTime;
protected String creator;
private final String name;
private String schema;
public SchemaTemplate(String name, String schema) {
E.checkArgument(name != null && !name.isEmpty(),
"The name of schema template can't be null or empty");
E.checkArgument(schema != null && !schema.isEmpty(),
"The schema template can't be null or empty");
this.name = name;
this.schema = schema;
this.createTime = new Date();
this.updateTime = createTime;
}
public SchemaTemplate(String name, String schema, Date create, String creator) {
E.checkArgument(name != null && !name.isEmpty(),
"The name of schema template can't be null or empty");
E.checkArgument(schema != null && !schema.isEmpty(),
"The schema template can't be null or empty");
this.name = name;
this.schema = schema;
this.createTime = create;
this.updateTime = createTime;
this.creator = creator;
}
public static SchemaTemplate fromMap(Map<String, String> map) {
try {
SchemaTemplate template = new SchemaTemplate(map.get("name"),
map.get("schema"),
FORMATTER.parse(map.get("create")),
map.get("creator"));
template.updateTime(FORMATTER.parse(map.get("update")));
return template;
} catch (ParseException e) {
e.printStackTrace();
}
return null;
}
public String name() {
return this.name;
}
public String schema() {
return this.schema;
}
public void schema(String schema) {
this.schema = schema;
}
public Date create() {
return this.createTime;
}
public Date createTime() {
return this.createTime;
}
public Date update() {
return this.updateTime;
}
public Date updateTime() {
return this.updateTime;
}
public void create(Date create) {
this.createTime = create;
}
public String creator() {
return this.creator;
}
public void creator(String creator) {
this.creator = creator;
}
public void updateTime(Date updateTime) {
this.updateTime = updateTime;
}
public void refreshUpdateTime() {
this.updateTime = new Date();
}
public Map<String, String> asMap() {
String createStr = FORMATTER.format(this.createTime);
String updateStr = FORMATTER.format(this.updateTime);
return new ImmutableMap.Builder<String, String>()
.put("name", this.name)
.put("schema", this.schema)
.put("create", createStr)
.put("create_time", createStr)
.put("update", updateStr)
.put("update_time", updateStr)
.put("creator", this.creator)
.build();
}
}

361
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/space/Service.java Normal file
View File

@ -0,0 +1,361 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.space;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.hugegraph.util.E;
public class Service {
public static final int DEFAULT_COUNT = 1;
public static final String DEFAULT_ROUTE_TYPE = "NodePort";
public static final int DEFAULT_PORT = 0;
public static final int DEFAULT_CPU_LIMIT = 4;
public static final int DEFAULT_MEMORY_LIMIT = 8;
public static final int DEFAULT_STORAGE_LIMIT = 100;
private final String creator;
private String name;
private ServiceType type;
private DeploymentType deploymentType;
private String description;
private Status status;
private int count;
private int running;
private int cpuLimit;
private int memoryLimit; // GB
private int storageLimit; // GB
private String routeType;
private int port;
private Set<String> urls = new HashSet<>();
private Set<String> serverDdsUrls = new HashSet<>();
private Set<String> serverNodePortUrls = new HashSet<>();
private String serviceId;
private String pdServiceId;
private Date createTime;
private Date updateTime;
public Service(String name, String creator, ServiceType type,
DeploymentType deploymentType) {
E.checkArgument(name != null && !StringUtils.isEmpty(name),
"The name of service can't be null or empty");
E.checkArgumentNotNull(type, "The type of service can't be null");
E.checkArgumentNotNull(deploymentType,
"The deployment type of service can't be null");
this.name = name;
this.type = type;
this.deploymentType = deploymentType;
this.status = Status.UNKNOWN;
this.count = DEFAULT_COUNT;
this.running = 0;
this.routeType = DEFAULT_ROUTE_TYPE;
this.port = DEFAULT_PORT;
this.cpuLimit = DEFAULT_CPU_LIMIT;
this.memoryLimit = DEFAULT_MEMORY_LIMIT;
this.storageLimit = DEFAULT_STORAGE_LIMIT;
this.creator = creator;
this.createTime = new Date();
this.updateTime = this.createTime;
}
public Service(String name, String creator, String description, ServiceType type,
DeploymentType deploymentType, int count, int running,
int cpuLimit, int memoryLimit, int storageLimit,
String routeType, int port, Set<String> urls) {
E.checkArgument(name != null && !StringUtils.isEmpty(name),
"The name of service can't be null or empty");
E.checkArgumentNotNull(type, "The type of service can't be null");
this.name = name;
this.description = description;
this.type = type;
this.status = Status.UNKNOWN;
this.deploymentType = deploymentType;
this.count = count;
this.running = running;
this.cpuLimit = cpuLimit;
this.memoryLimit = memoryLimit;
this.storageLimit = storageLimit;
this.routeType = routeType;
this.port = port;
this.urls = urls;
this.creator = creator;
this.createTime = new Date();
this.updateTime = this.createTime;
}
public String name() {
return this.name;
}
public String description() {
return this.description;
}
public void description(String description) {
this.description = description;
}
public ServiceType type() {
return this.type;
}
public void type(ServiceType type) {
this.type = type;
}
public DeploymentType deploymentType() {
return this.deploymentType;
}
public void deploymentType(DeploymentType deploymentType) {
this.deploymentType = deploymentType;
}
public Status status() {
return this.status;
}
public void status(Status status) {
this.status = status;
}
public int count() {
return this.count;
}
public void count(int count) {
E.checkArgument(count > 0,
"The service count must be > 0, but got: %s", count);
this.count = count;
}
public int running() {
return this.running;
}
public void running(int running) {
E.checkArgument(running <= this.count,
"The running count must be < count %s, but got: %s",
this.count, running);
this.running = running;
}
public int cpuLimit() {
return this.cpuLimit;
}
public void cpuLimit(int cpuLimit) {
E.checkArgument(cpuLimit > 0,
"The cpu limit must be > 0, but got: %s", cpuLimit);
this.cpuLimit = cpuLimit;
}
public int memoryLimit() {
return this.memoryLimit;
}
public void memoryLimit(int memoryLimit) {
E.checkArgument(memoryLimit > 0,
"The memory limit must be > 0, but got: %s",
memoryLimit);
this.memoryLimit = memoryLimit;
}
public int storageLimit() {
return this.storageLimit;
}
public void storageLimit(int storageLimit) {
E.checkArgument(storageLimit > 0,
"The storage limit must be > 0, but got: %s",
storageLimit);
this.storageLimit = storageLimit;
}
public String routeType() {
return this.routeType;
}
public void routeType(String routeType) {
this.routeType = routeType;
}
public int port() {
return this.port;
}
public void port(int port) {
this.port = port;
}
public Set<String> urls() {
if (this.urls == null) {
this.urls = new HashSet<>();
}
return this.urls;
}
public void urls(Set<String> urls) {
this.urls = urls;
}
public Set<String> serverDdsUrls() {
if (this.serverDdsUrls == null) {
this.serverDdsUrls = new HashSet<>();
}
return this.serverDdsUrls;
}
public void serverDdsUrls(Set<String> urls) {
this.serverDdsUrls = urls;
}
public Set<String> serverNodePortUrls() {
if (this.serverNodePortUrls == null) {
this.serverNodePortUrls = new HashSet<>();
}
return this.serverNodePortUrls;
}
public void serverNodePortUrls(Set<String> urls) {
this.serverNodePortUrls = urls;
}
public void url(String url) {
if (this.urls == null) {
this.urls = new HashSet<>();
}
this.urls.add(url);
}
public boolean manual() {
return DeploymentType.MANUAL.equals(this.deploymentType);
}
public boolean k8s() {
return DeploymentType.K8S.equals(this.deploymentType);
}
public String creator() {
return this.creator;
}
public Date createdTime() {
return this.createTime;
}
public Date updateTime() {
return this.updateTime;
}
public void createTime(Date create) {
this.createTime = create;
}
public void updateTime(Date update) {
this.updateTime = update;
}
public void refreshUpdate() {
this.updateTime = new Date();
}
public boolean sameService(Service other) {
if (other.deploymentType == DeploymentType.K8S ||
this.deploymentType == DeploymentType.K8S) {
return true;
}
return (this.name.equals(other.name) &&
this.type.equals(other.type) &&
this.deploymentType == other.deploymentType &&
this.urls.equals(other.urls) &&
this.port == other.port);
}
public Map<String, Object> info() {
Map<String, Object> infos = new LinkedHashMap<>();
infos.put("name", this.name);
infos.put("type", this.type);
infos.put("deployment_type", this.deploymentType);
infos.put("description", this.description);
infos.put("status", this.status);
infos.put("count", this.count);
infos.put("running", this.running);
infos.put("cpu_limit", this.cpuLimit);
infos.put("memory_limit", this.memoryLimit);
infos.put("storage_limit", this.storageLimit);
infos.put("route_type", this.routeType);
infos.put("port", this.port);
infos.put("urls", this.urls);
infos.put("server_dds_urls", this.serverDdsUrls);
infos.put("server_node_port_urls", this.serverNodePortUrls);
infos.put("service_id", this.serviceId);
infos.put("pd_service_id", this.pdServiceId);
infos.put("creator", this.creator);
infos.put("create_time", this.createTime);
infos.put("update_time", this.updateTime);
return infos;
}
public String serviceId() {
return this.serviceId;
}
public void serviceId(String serviceId) {
this.serviceId = serviceId;
}
public String pdServiceId() {
return this.pdServiceId;
}
public void pdServiceId(String serviceId) {
this.pdServiceId = serviceId;
}
public enum DeploymentType {
MANUAL,
K8S,
}
public enum ServiceType {
OLTP,
OLAP,
STORAGE
}
public enum Status {
UNKNOWN,
STARTING,
RUNNING,
STOPPED
}
}

65
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java
View File

@ -540,4 +540,69 @@ public class HugeEdge extends HugeElement implements Edge, Cloneable {
return edge;
}
public static HugeEdge constructEdgeWithoutLabel(HugeVertex ownerVertex,
boolean isOutEdge,
String sortValues,
Id otherVertexId) {
HugeGraph graph = ownerVertex.graph();
HugeVertex otherVertex = new HugeVertex(graph, otherVertexId,
VertexLabel.NONE);
ownerVertex.propNotLoaded();
otherVertex.propNotLoaded();
HugeEdge edge = new HugeEdge(graph, null, EdgeLabel.NONE);
edge.name(sortValues);
edge.vertices(isOutEdge, ownerVertex, otherVertex);
edge.assignId();
if (isOutEdge) {
ownerVertex.addOutEdge(edge);
otherVertex.addInEdge(edge.switchOwner());
} else {
ownerVertex.addInEdge(edge);
otherVertex.addOutEdge(edge.switchOwner());
}
return edge;
}
public static HugeEdge constructEdgeWithoutGraph(HugeVertex ownerVertex,
boolean isOutEdge,
EdgeLabel edgeLabel,
String sortValues,
Id otherVertexId) {
Id ownerLabelId = edgeLabel.sourceLabel();
Id otherLabelId = edgeLabel.targetLabel();
VertexLabel srcLabel = new VertexLabel(null, ownerLabelId, "UNDEF");
VertexLabel tgtLabel = new VertexLabel(null, otherLabelId, "UNDEF");
VertexLabel otherVertexLabel;
if (isOutEdge) {
ownerVertex.correctVertexLabel(srcLabel);
otherVertexLabel = tgtLabel;
} else {
ownerVertex.correctVertexLabel(tgtLabel);
otherVertexLabel = srcLabel;
}
HugeVertex otherVertex = new HugeVertex(null, otherVertexId,
otherVertexLabel);
ownerVertex.propNotLoaded();
otherVertex.propNotLoaded();
HugeEdge edge = new HugeEdge(null, null, edgeLabel);
edge.name(sortValues);
edge.vertices(isOutEdge, ownerVertex, otherVertex);
edge.assignId();
if (isOutEdge) {
ownerVertex.addOutEdge(edge);
otherVertex.addInEdge(edge.switchOwner());
} else {
ownerVertex.addInEdge(edge);
otherVertex.addOutEdge(edge.switchOwner());
}
return edge;
}
}

15
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeIndex.java
View File

@ -44,6 +44,7 @@ public class HugeIndex implements GraphType, Cloneable {
private Object fieldValues;
private IndexLabel indexLabel;
private Set<IdWithExpiredTime> elementIds;
private static final int HUGE_TYPE_CODE_LENGTH = 1;
public HugeIndex(HugeGraph graph, IndexLabel indexLabel) {
E.checkNotNull(graph, "graph");
@ -210,11 +211,12 @@ public class HugeIndex implements GraphType, Cloneable {
* index label in front(hugegraph-1317)
*/
String strIndexLabelId = IdGenerator.asStoredString(indexLabelId);
return SplicingIdGenerator.splicing(strIndexLabelId, value);
return SplicingIdGenerator.splicing(type.string(), strIndexLabelId, value);
} else {
assert type.isRangeIndex();
int length = type.isRange4Index() ? 4 : 8;
BytesBuffer buffer = BytesBuffer.allocate(4 + length);
BytesBuffer buffer = BytesBuffer.allocate(HUGE_TYPE_CODE_LENGTH + 4 + length);
buffer.write(type.code());
buffer.writeInt(SchemaElement.schemaId(indexLabelId));
if (fieldValues != null) {
E.checkState(fieldValues instanceof Number,
@ -234,15 +236,16 @@ public class HugeIndex implements GraphType, Cloneable {
if (type.isStringIndex()) {
Id idObject = IdGenerator.of(id, IdType.STRING);
String[] parts = SplicingIdGenerator.parse(idObject);
E.checkState(parts.length == 2, "Invalid secondary index id");
Id label = IdGenerator.ofStoredString(parts[0], IdType.LONG);
E.checkState(parts.length == 3, "Invalid secondary index id");
Id label = IdGenerator.ofStoredString(parts[1], IdType.LONG);
indexLabel = IndexLabel.label(graph, label);
values = parts[1];
values = parts[2];
} else {
assert type.isRange4Index() || type.isRange8Index();
final int labelLength = 4;
E.checkState(id.length > labelLength, "Invalid range index id");
BytesBuffer buffer = BytesBuffer.wrap(id);
buffer.read(HUGE_TYPE_CODE_LENGTH);
Id label = IdGenerator.of(buffer.readInt());
indexLabel = IndexLabel.label(graph, label);
List<Id> fields = indexLabel.indexFields();
@ -252,7 +255,7 @@ public class HugeIndex implements GraphType, Cloneable {
"Invalid range index field type");
Class<?> clazz = dataType.isNumber() ?
dataType.clazz() : DataType.LONG.clazz();
values = bytes2number(buffer.read(id.length - labelLength), clazz);
values = bytes2number(buffer.read(id.length - labelLength - HUGE_TYPE_CODE_LENGTH), clazz);
}
HugeIndex index = new HugeIndex(graph, indexLabel);
index.fieldValues(values);

8
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeVertex.java
View File

@ -43,6 +43,8 @@ import org.apache.hugegraph.perf.PerfUtil.Watched;
import org.apache.hugegraph.schema.EdgeLabel;
import org.apache.hugegraph.schema.PropertyKey;
import org.apache.hugegraph.schema.VertexLabel;
import org.apache.hugegraph.task.HugeServerInfo;
import org.apache.hugegraph.task.HugeTask;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.type.define.Cardinality;
import org.apache.hugegraph.type.define.CollectionType;
@ -90,6 +92,12 @@ public class HugeVertex extends HugeElement implements Vertex, Cloneable {
@Override
public HugeType type() {
if (label != null && label.name().equals(HugeTask.P.TASK)) {
return HugeType.TASK;
}
if (label != null && label.name().equals(HugeServerInfo.P.SERVER)) {
return HugeType.SERVER;
}
return HugeType.VERTEX;
}

11
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/ServerInfoManager.java
View File

@ -319,7 +319,7 @@ public class ServerInfoManager {
private HugeServerInfo serverInfo(Id server) {
return this.call(() -> {
Iterator<Vertex> vertices = this.tx().queryVertices(server);
Iterator<Vertex> vertices = this.tx().queryServerInfos(server);
Vertex vertex = QueryResults.one(vertices);
if (vertex == null) {
return null;
@ -347,7 +347,7 @@ public class ServerInfoManager {
}
LOG.info("Remove server info: {}", server);
return this.call(() -> {
Iterator<Vertex> vertices = this.tx().queryVertices(server);
Iterator<Vertex> vertices = this.tx().queryServerInfos(server);
Vertex vertex = QueryResults.one(vertices);
if (vertex == null) {
return null;
@ -382,7 +382,12 @@ public class ServerInfoManager {
private Iterator<HugeServerInfo> serverInfos(Map<String, Object> conditions,
long limit, String page) {
return this.call(() -> {
ConditionQuery query = new ConditionQuery(HugeType.VERTEX);
ConditionQuery query;
if (this.graph.backendStoreFeatures().supportsTaskAndServerVertex()) {
query = new ConditionQuery(HugeType.SERVER);
} else {
query = new ConditionQuery(HugeType.VERTEX);
}
if (page != null) {
query.page(page);
}

15
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java
View File

@ -523,7 +523,7 @@ public class StandardTaskScheduler implements TaskScheduler {
public <V> HugeTask<V> findTask(Id id) {
HugeTask<V> result = this.call(() -> {
Iterator<Vertex> vertices = this.tx().queryVertices(id);
Iterator<Vertex> vertices = this.tx().queryTaskInfos(id);
Vertex vertex = QueryResults.one(vertices);
if (vertex == null) {
return null;
@ -573,7 +573,7 @@ public class StandardTaskScheduler implements TaskScheduler {
}
return this.call(() -> {
Iterator<Vertex> vertices = this.tx().queryVertices(id);
Iterator<Vertex> vertices = this.tx().queryTaskInfos(id);
HugeVertex vertex = (HugeVertex) QueryResults.one(vertices);
if (vertex == null) {
return null;
@ -666,7 +666,12 @@ public class StandardTaskScheduler implements TaskScheduler {
private <V> Iterator<HugeTask<V>> queryTask(Map<String, Object> conditions,
long limit, String page) {
return this.call(() -> {
ConditionQuery query = new ConditionQuery(HugeType.VERTEX);
ConditionQuery query;
if (this.graph.backendStoreFeatures().supportsTaskAndServerVertex()) {
query = new ConditionQuery(HugeType.TASK);
} else {
query = new ConditionQuery(HugeType.VERTEX);
}
if (page != null) {
query.page(page);
}
@ -691,7 +696,7 @@ public class StandardTaskScheduler implements TaskScheduler {
private <V> Iterator<HugeTask<V>> queryTask(List<Id> ids) {
return this.call(() -> {
Object[] idArray = ids.toArray(new Id[0]);
Iterator<Vertex> vertices = this.tx().queryVertices(idArray);
Iterator<Vertex> vertices = this.tx().queryTaskInfos(idArray);
Iterator<HugeTask<V>> tasks =
new MapperIterator<>(vertices, HugeTask::fromVertex);
// Convert iterator to list to avoid across thread tx accessed
@ -756,7 +761,7 @@ public class StandardTaskScheduler implements TaskScheduler {
public void deleteIndex(HugeVertex vertex) {
// Delete the old record if exist
Iterator<Vertex> old = this.queryVertices(vertex.id());
Iterator<Vertex> old = this.queryTaskInfos(vertex.id());
HugeVertex oldV = (HugeVertex) QueryResults.one(old);
if (oldV == null) {
return;

64
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/type/HugeTableType.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.type;
import java.util.HashMap;
import java.util.Map;
import org.apache.hugegraph.type.define.SerialEnum;
public enum HugeTableType implements SerialEnum {
UNKNOWN(0, "UNKNOWN"),
/* Schema types */
VERTEX(1, "V"), // 顶点表
OUT_EDGE(2, "OE"), // 出边表
IN_EDGE(3, "IE"), // 入边表
ALL_INDEX_TABLE(4, "INDEX"), // 索引表
TASK_INFO_TABLE(5, "TASK"), // 任务信息表
OLAP_TABLE(6, "OLAP"), // OLAP 表
SERVER_INFO_TABLE(7, "SERVER"); // SERVER 信息表
private static final Map<String, HugeTableType> ALL_NAME = new HashMap<>();
static {
SerialEnum.register(HugeTableType.class);
for (HugeTableType type : values()) {
ALL_NAME.put(type.name, type);
}
}
private byte type = 0;
private String name;
HugeTableType(int type, String name) {
assert type < 256;
this.type = (byte) type;
this.name = name;
}
@Override
public byte code() {
return this.type;
}
public String string() {
return this.name;
}
}

9
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/type/HugeType.java
View File

@ -65,7 +65,8 @@ public enum HugeType implements SerialEnum {
SHARD_INDEX(175, "HI"),
UNIQUE_INDEX(178, "UI"),
TASK(180, "T"),
TASK(180, "TASK"),
SERVER(181, "SERVER"),
// System schema
SYS_SCHEMA(250, "SS"),
@ -115,7 +116,7 @@ public enum HugeType implements SerialEnum {
}
public boolean isVertex() {
return this == HugeType.VERTEX;
return this == HugeType.VERTEX || this == HugeType.TASK || this == HugeType.SERVER;
}
public boolean isEdge() {
@ -192,4 +193,8 @@ public enum HugeType implements SerialEnum {
public static HugeType fromCode(byte code) {
return SerialEnum.fromCode(HugeType.class, code);
}
public boolean isLabelIndex() {
return this == VERTEX_LABEL_INDEX || this == EDGE_LABEL_INDEX;
}
}

70
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/type/define/EdgeLabelType.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.type.define;
public enum EdgeLabelType implements SerialEnum {
NORMAL(1, "NORMAL"),
PARENT(2, "PARENT"),
SUB(3, "SUB"),
GENERAL(4, "GENERAL"),
;
static {
SerialEnum.register(EdgeLabelType.class);
}
private final byte code;
private final String name;
EdgeLabelType(int code, String name) {
assert code < 256;
this.code = (byte) code;
this.name = name;
}
@Override
public byte code() {
return this.code;
}
public String string() {
return this.name;
}
public boolean normal() {
return this == NORMAL;
}
public boolean parent() {
return this == PARENT;
}
public boolean sub() {
return this == SUB;
}
public boolean general() {
return this == GENERAL;
}
}

5
hugegraph-server/hugegraph-dist/pom.xml
View File

@ -98,6 +98,11 @@
<artifactId>hugegraph-postgresql</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-hstore</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>

2
hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties
View File

@ -19,7 +19,7 @@ edge.cache_type=l2
#vertex.default_label=vertex
backend=rocksdb
backend=hstore
serializer=binary
store=hugegraph

12
hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/RegisterUtil.java vendored
View File

@ -91,6 +91,9 @@ public class RegisterUtil {
case "postgresql":
registerPostgresql();
break;
case "hstore":
registerHstore();
break;
default:
throw new HugeException("Unsupported backend type '%s'", backend);
}
@ -215,4 +218,13 @@ public class RegisterUtil {
}
}
}
public static void registerHstore() {
// Register config
OptionSpace.register("hstore",
"org.apache.hugegraph.backend.store.hstore.HstoreOptions");
// Register backend
BackendProviderFactory.register("hstore",
"org.apache.hugegraph.backend.store.hstore.HstoreProvider");
}
}

2
hugegraph-server/hugegraph-dist/src/main/resources/backend.properties
View File

@ -15,4 +15,4 @@
# under the License.
#
backends=[cassandra, scylladb, rocksdb, mysql, palo, hbase, postgresql]
backends=[cassandra, scylladb, rocksdb, mysql, palo, hbase, postgresql, hstore]

51
hugegraph-server/hugegraph-example/src/main/java/org/apache/hugegraph/example/ExampleNew.java Normal file
View File

@ -0,0 +1,51 @@
package org.apache.hugegraph.example;
import static org.apache.hugegraph.backend.page.PageState.EMPTY_BYTES;
import org.apache.hugegraph.pd.client.PDConfig;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgKvStore;
import org.apache.hugegraph.store.HgStoreClient;
import org.apache.hugegraph.store.HgStoreSession;
public class ExampleNew {
public static void main(String[] args) throws Exception {
testScanTable("hugegraph", "g+v"); // why should with "g+"?
}
private static void testScanTable(String graph, String table) {
/*
* Valid table is:
* g+v
* g+oe
* g+ie
* g+olap
* g+task
* g+index
* g+server
*/
HgStoreClient storeClient = HgStoreClient.create(
PDConfig.of("127.0.0.1:8686").setEnableCache(false));
String storeTemplate = "%s/g";
String store = String.format(storeTemplate, graph);
HgStoreSession session = storeClient.openSession(store);
try (HgKvIterator<HgKvEntry> iterators = session.scanIterator(table,
0, 100000000,
HgKvStore.SCAN_HASHCODE,
EMPTY_BYTES)) {
int count = 0;
while (iterators.hasNext()) {
count++;
HgKvEntry next = iterators.next();
System.out.println(new String(next.key()) +
" <====> " +
new String(next.value()));
}
System.out.println(count);
} catch (Exception e) {
e.printStackTrace();
}
}
}

2
hugegraph-server/hugegraph-hbase/src/main/java/org/apache/hugegraph/backend/store/hbase/HbaseStore.java
View File

@ -96,7 +96,7 @@ public abstract class HbaseStore extends AbstractBackendStore<HbaseSessions.Sess
@Override
protected final HbaseTable table(HugeType type) {
assert type != null;
HbaseTable table = this.tables.get(type);
HbaseTable table = this.tables.get(convertTaskOrServerToVertex(type));
if (table == null) {
throw new BackendException("Unsupported table type: %s", type);
}

50
hugegraph-server/hugegraph-hstore/pom.xml Normal file
View File

@ -0,0 +1,50 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>hugegraph-server</artifactId>
<groupId>org.apache.hugegraph</groupId>
<version>${revision}</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hugegraph-hstore</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-core</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-client</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-pd-client</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</project>

133
hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreFeatures.java Normal file
View File

@ -0,0 +1,133 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.store.hstore;
import org.apache.hugegraph.backend.store.BackendFeatures;
public class HstoreFeatures implements BackendFeatures {
@Override
public boolean supportsScanToken() {
return false;
}
@Override
public boolean supportsScanKeyPrefix() {
return true;
}
@Override
public boolean supportsScanKeyRange() {
return true;
}
@Override
public boolean supportsQuerySchemaByName() {
return false;
}
@Override
public boolean supportsQueryByLabel() {
return false;
}
@Override
public boolean supportsQueryWithInCondition() {
return false;
}
@Override
public boolean supportsQueryWithRangeCondition() {
return true;
}
@Override
public boolean supportsQuerySortByInputIds() {
return true;
}
@Override
public boolean supportsQueryWithOrderBy() {
return true;
}
@Override
public boolean supportsQueryWithContains() {
return false;
}
@Override
public boolean supportsQueryWithContainsKey() {
return false;
}
@Override
public boolean supportsQueryByPage() {
return true;
}
@Override
public boolean supportsDeleteEdgeByLabel() {
return false;
}
@Override
public boolean supportsUpdateVertexProperty() {
// Vertex properties are stored in a cell(column value)
return false;
}
@Override
public boolean supportsMergeVertexProperty() {
return false;
}
@Override
public boolean supportsUpdateEdgeProperty() {
// Edge properties are stored in a cell(column value)
return false;
}
@Override
public boolean supportsTransaction() {
return false;
}
@Override
public boolean supportsNumberType() {
return false;
}
@Override
public boolean supportsAggregateProperty() {
return false;
}
@Override
public boolean supportsTtl() {
return false;
}
@Override
public boolean supportsOlapProperties() {
return true;
}
@Override
public boolean supportsTaskAndServerVertex() { return true; }
}

279
hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreNodePartitionerImpl.java Normal file
View File

@ -0,0 +1,279 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.store.hstore;
import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.common.KVPair;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.common.PartitionUtils;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.client.HgNodePartition;
import org.apache.hugegraph.store.client.HgNodePartitionerBuilder;
import org.apache.hugegraph.store.client.HgStoreNode;
import org.apache.hugegraph.store.client.HgStoreNodeManager;
import org.apache.hugegraph.store.client.HgStoreNodeNotifier;
import org.apache.hugegraph.store.client.HgStoreNodePartitioner;
import org.apache.hugegraph.store.client.HgStoreNodeProvider;
import org.apache.hugegraph.store.client.HgStoreNotice;
import org.apache.hugegraph.store.client.type.HgNodeStatus;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
public class HstoreNodePartitionerImpl implements HgStoreNodePartitioner,
HgStoreNodeProvider,
HgStoreNodeNotifier {
private static final Logger LOG = Log.logger(HstoreNodePartitionerImpl.class);
private PDClient pdClient;
private HgStoreNodeManager nodeManager;
protected HstoreNodePartitionerImpl() {
}
public HstoreNodePartitionerImpl(String pdPeers) {
pdClient = HstoreSessionsImpl.getDefaultPdClient();
}
public HstoreNodePartitionerImpl(HgStoreNodeManager nodeManager,
String pdPeers) {
this(pdPeers);
this.nodeManager = nodeManager;
}
public void setPDClient(PDClient pdClient) {
this.pdClient = pdClient;
}
/**
* 查询分区信息，结果通过HgNodePartitionerBuilder返回
*/
@Override
public int partition(HgNodePartitionerBuilder builder, String graphName,
byte[] startKey, byte[] endKey) {
try {
HashSet<HgNodePartition> partitions = null;
if (HgStoreClientConst.ALL_PARTITION_OWNER == startKey) {
List<Metapb.Store> stores = pdClient.getActiveStores(graphName);
partitions = new HashSet<>(stores.size());
for (Metapb.Store store : stores) {
partitions.add(HgNodePartition.of(store.getId(), -1));
}
} else if (endKey == HgStoreClientConst.EMPTY_BYTES
|| startKey == endKey || Arrays.equals(startKey, endKey)) {
KVPair<Metapb.Partition, Metapb.Shard> partShard =
pdClient.getPartition(graphName, startKey);
Metapb.Shard leader = partShard.getValue();
partitions = new HashSet<>(1);
partitions.add(HgNodePartition.of(leader.getStoreId(),
pdClient.keyToCode(graphName, startKey)));
} else {
LOG.warn(
"StartOwnerkey is not equal to endOwnerkey, which is meaningless!!, It is" +
" a error!!");
List<Metapb.Store> stores = pdClient.getActiveStores(graphName);
for (Metapb.Store store : stores) {
partitions.add(HgNodePartition.of(store.getId(), -1));
}
}
builder.setPartitions(partitions);
} catch (PDException e) {
LOG.error("An error occurred while getting partition information :{}", e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
return 0;
}
@Override
public int partition(HgNodePartitionerBuilder builder, String graphName,
int startKey, int endKey) {
try {
HashSet<HgNodePartition> partitions = new HashSet<>();
Metapb.Partition partition = null;
while ((partition == null || partition.getEndKey() < endKey)
&& startKey < PartitionUtils.MAX_VALUE) {
KVPair<Metapb.Partition, Metapb.Shard> partShard =
pdClient.getPartitionByCode(graphName, startKey);
if (partShard != null) {
partition = partShard.getKey();
Metapb.Shard leader = partShard.getValue();
partitions.add(HgNodePartition.of(leader.getStoreId(), startKey,
(int) partition.getStartKey(),
(int) partition.getEndKey()));
startKey = (int) partition.getEndKey();
} else {
break;
}
}
builder.setPartitions(partitions);
} catch (PDException e) {
LOG.error("An error occurred while getting partition information :{}", e.getMessage());
throw new RuntimeException(e.getMessage(), e);
}
return 0;
}
/**
* 查询hgstore信息
*
* @return hgstore
*/
@Override
public HgStoreNode apply(String graphName, Long nodeId) {
try {
Metapb.Store store = pdClient.getStore(nodeId);
return nodeManager.getNodeBuilder().setNodeId(store.getId())
.setAddress(store.getAddress()).build();
} catch (PDException e) {
throw new RuntimeException(e.getMessage(), e);
}
}
/**
* 通知更新缓存
*/
@Override
public int notice(String graphName, HgStoreNotice storeNotice) {
LOG.warn(storeNotice.toString());
if (storeNotice.getPartitionLeaders() != null) {
storeNotice.getPartitionLeaders().forEach((partId, leader) -> {
pdClient.updatePartitionLeader(graphName, partId, leader);
LOG.warn("updatePartitionLeader:{}-{}-{}",
graphName, partId, leader);
});
}
if (storeNotice.getPartitionIds() != null) {
storeNotice.getPartitionIds().forEach(partId -> {
pdClient.invalidPartitionCache(graphName, partId);
});
}
if (!storeNotice.getNodeStatus().equals(
HgNodeStatus.PARTITION_COMMON_FAULT)
&& !storeNotice.getNodeStatus().equals(
HgNodeStatus.NOT_PARTITION_LEADER)) {
pdClient.invalidPartitionCache();
LOG.warn("invalidPartitionCache:{} ", storeNotice.getNodeStatus());
}
return 0;
}
public Metapb.Graph delGraph(String graphName) {
try {
return pdClient.delGraph(graphName);
} catch (PDException e) {
LOG.error("delGraph {} exception, {}", graphName, e.getMessage());
}
return null;
}
public void setNodeManager(HgStoreNodeManager nodeManager) {
this.nodeManager = nodeManager;
}
}
class FakeHstoreNodePartitionerImpl extends HstoreNodePartitionerImpl {
private static final Logger LOG = Log.logger(HstoreNodePartitionerImpl.class);
private static final int partitionCount = 3;
private static final Map<Integer, Long> leaderMap = new ConcurrentHashMap<>();
private static final Map<Long, String> storeMap = new ConcurrentHashMap<>();
HgStoreNodeManager nodeManager;
private final String hstorePeers;
public FakeHstoreNodePartitionerImpl(String pdPeers) {
this.hstorePeers = pdPeers;
// store列表
for (String address : hstorePeers.split(",")) {
storeMap.put((long) address.hashCode(), address);
}
// 分区列表
for (int i = 0; i < partitionCount; i++) {
leaderMap.put(i, storeMap.keySet().iterator().next());
}
}
public FakeHstoreNodePartitionerImpl(HgStoreNodeManager nodeManager,
String peers) {
this(peers);
this.nodeManager = nodeManager;
}
@Override
public int partition(HgNodePartitionerBuilder builder, String graphName,
byte[] startKey, byte[] endKey) {
int startCode = PartitionUtils.calcHashcode(startKey);
HashSet<HgNodePartition> partitions = new HashSet<>(storeMap.size());
if (ALL_PARTITION_OWNER == startKey) {
storeMap.forEach((k, v) -> {
partitions.add(HgNodePartition.of(k, -1));
});
} else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey ||
Arrays.equals(startKey, endKey)) {
partitions.add(
HgNodePartition.of(leaderMap.get(startCode % partitionCount), startCode));
} else {
LOG.error("OwnerKey转成HashCode后已经无序了， 按照OwnerKey范围查询没意义");
storeMap.forEach((k, v) -> {
partitions.add(HgNodePartition.of(k, -1));
});
}
builder.setPartitions(partitions);
return 0;
}
@Override
public HgStoreNode apply(String graphName, Long nodeId) {
return nodeManager.getNodeBuilder().setNodeId(nodeId)
.setAddress(storeMap.get(nodeId)).build();
}
@Override
public int notice(String graphName, HgStoreNotice storeNotice) {
if (storeNotice.getPartitionLeaders() != null
&& storeNotice.getPartitionLeaders().size() > 0) {
leaderMap.putAll(storeNotice.getPartitionLeaders());
}
return 0;
}
public static class NodePartitionerFactory {
public static HstoreNodePartitionerImpl getNodePartitioner(
HugeConfig config, HgStoreNodeManager nodeManager) {
if (config.get(HstoreOptions.PD_FAKE)) {
return new FakeHstoreNodePartitionerImpl(nodeManager,
config.get(HstoreOptions.HSTORE_PEERS));
} else {
return new HstoreNodePartitionerImpl(nodeManager,
config.get(HstoreOptions.PD_PEERS)
);
}
}
}
}

70
hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreOptions.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.store.hstore;
import static org.apache.hugegraph.config.OptionChecker.disallowEmpty;
import org.apache.hugegraph.config.ConfigOption;
import org.apache.hugegraph.config.OptionHolder;
public class HstoreOptions extends OptionHolder {
public static final ConfigOption<String> PD_PEERS = new ConfigOption<>(
"pd.peers",
"The addresses of pd nodes, separated with commas.",
disallowEmpty(),
"localhost:8686"
);
public static final ConfigOption<Boolean> PD_FAKE = new ConfigOption<>(
"pd.fake",
"Enable the fake PD service.",
disallowEmpty(),
false
);
public static final ConfigOption<String> HSTORE_PEERS = new ConfigOption<>(
"hstore.peers",
"The addresses of store nodes, separated with commas.",
disallowEmpty(),
"localhost:9080"
);
public static final ConfigOption<Integer> PARTITION_COUNT = new ConfigOption<>(
"hstore.partition_count",
"Number of partitions, which PD controls partitions based on.",
disallowEmpty(),
0
);
public static final ConfigOption<Integer> SHARD_COUNT = new ConfigOption<>(
"hstore.shard_count",
"Number of copies, which PD controls partition copies based on.",
disallowEmpty(),
0
);
private static volatile HstoreOptions instance;
private HstoreOptions() {
super();
}
public static synchronized HstoreOptions instance() {
if (instance == null) {
instance = new HstoreOptions();
instance.registerOptions();
}
return instance;
}
}

54
hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreProvider.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.store.hstore;
import org.apache.hugegraph.backend.store.AbstractBackendStoreProvider;
import org.apache.hugegraph.backend.store.BackendStore;
import org.apache.hugegraph.config.HugeConfig;
public class HstoreProvider extends AbstractBackendStoreProvider {
protected String namespace() {
return this.graph();
}
@Override
public String type() {
return "hstore";
}
@Override
public String driverVersion() {
return "1.13";
}
@Override
protected BackendStore newSchemaStore(HugeConfig config, String store) {
return new HstoreStore.HstoreSchemaStore(this, this.namespace(), store);
}
@Override
protected BackendStore newGraphStore(HugeConfig config, String store) {
return new HstoreStore.HstoreGraphStore(this, this.namespace(), store);
}
@Override
protected BackendStore newSystemStore(HugeConfig config, String store) {
return null;
}
}

206
hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java Executable file
View File

@ -0,0 +1,206 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.store.hstore;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hugegraph.backend.query.Query;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.backend.store.BackendEntry.BackendColumnIterator;
import org.apache.hugegraph.backend.store.BackendSession.AbstractBackendSession;
import org.apache.hugegraph.backend.store.BackendSessionPool;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.type.define.GraphMode;
public abstract class HstoreSessions extends BackendSessionPool {
public HstoreSessions(HugeConfig config, String database, String store) {
super(config, database + "/" + store);
}
public abstract Set<String> openedTables();
public abstract void createTable(String... tables);
public abstract void dropTable(String... tables);
public abstract boolean existsTable(String table);
public abstract void truncateTable(String table);
public abstract void clear();
@Override
public abstract Session session();
public interface Countable {
public long count();
}
/**
* Session for Hstore
*/
public static abstract class Session extends AbstractBackendSession {
public static final int SCAN_ANY = 0x80;
public static final int SCAN_PREFIX_BEGIN = 0x01;
public static final int SCAN_PREFIX_END = 0x02;
public static final int SCAN_GT_BEGIN = 0x04;
public static final int SCAN_GTE_BEGIN = 0x0c;
public static final int SCAN_LT_END = 0x10;
public static final int SCAN_LTE_END = 0x30;
public static final int SCAN_KEY_ONLY = 0x40;
public static final int SCAN_HASHCODE = 0x100;
private HugeConfig conf;
private String graphName;
public static boolean matchScanType(int expected, int actual) {
return (expected & actual) == expected;
}
public abstract void createTable(String tableName);
public abstract void dropTable(String tableName);
public abstract boolean existsTable(String tableName);
public abstract void truncateTable(String tableName);
public abstract void deleteGraph();
public abstract Pair<byte[], byte[]> keyRange(String table);
public abstract void put(String table, byte[] ownerKey,
byte[] key, byte[] value);
public abstract void increase(String table, byte[] ownerKey,
byte[] key, byte[] value);
public abstract void delete(String table, byte[] ownerKey, byte[] key);
public abstract void deletePrefix(String table, byte[] ownerKey,
byte[] key);
public abstract void deleteRange(String table, byte[] ownerKeyFrom,
byte[] ownerKeyTo, byte[] keyFrom,
byte[] keyTo);
public abstract byte[] get(String table, byte[] key);
public abstract byte[] get(String table, byte[] ownerKey, byte[] key);
public abstract BackendColumnIterator scan(String table);
public abstract BackendColumnIterator scan(String table,
byte[] ownerKey,
byte[] prefix);
public BackendColumnIterator scan(String table, byte[] ownerKeyFrom,
byte[] ownerKeyTo, byte[] keyFrom,
byte[] keyTo) {
return this.scan(table, ownerKeyFrom, ownerKeyTo, keyFrom, keyTo,
SCAN_LT_END);
}
public abstract List<BackendColumnIterator> scan(String table,
List<HgOwnerKey> keys,
int scanType,
long limit,
byte[] query);
public abstract BackendEntry.BackendIterator<BackendColumnIterator> scan(String table,
Iterator<HgOwnerKey> keys,
int scanType,
Query queryParam,
byte[] query);
public abstract BackendColumnIterator scan(String table,
byte[] ownerKeyFrom,
byte[] ownerKeyTo,
byte[] keyFrom,
byte[] keyTo,
int scanType);
public abstract BackendColumnIterator scan(String table,
byte[] ownerKeyFrom,
byte[] ownerKeyTo,
byte[] keyFrom,
byte[] keyTo,
int scanType,
byte[] query);
public abstract BackendColumnIterator scan(String table,
byte[] ownerKeyFrom,
byte[] ownerKeyTo,
byte[] keyFrom,
byte[] keyTo,
int scanType,
byte[] query,
byte[] position);
public abstract BackendColumnIterator scan(String table,
int codeFrom,
int codeTo,
int scanType,
byte[] query);
public abstract BackendColumnIterator scan(String table,
int codeFrom,
int codeTo,
int scanType,
byte[] query,
byte[] position);
public abstract BackendColumnIterator getWithBatch(String table,
List<HgOwnerKey> keys);
public abstract void merge(String table, byte[] ownerKey,
byte[] key, byte[] value);
public abstract void setMode(GraphMode mode);
public abstract void truncate() throws Exception;
public abstract BackendColumnIterator scan(String table,
byte[] conditionQueryToByte);
public HugeConfig getConf() {
return conf;
}
public void setConf(HugeConfig conf) {
this.conf = conf;
}
public String getGraphName() {
return graphName;
}
public void setGraphName(String graphName) {
this.graphName = graphName;
}
public abstract void beginTx();
}
}

782
hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessionsImpl.java Executable file
View File

@ -0,0 +1,782 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.store.hstore;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hugegraph.backend.query.Query;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.backend.store.BackendEntry.BackendColumn;
import org.apache.hugegraph.backend.store.BackendEntry.BackendColumnIterator;
import org.apache.hugegraph.backend.store.BackendEntryIterator;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.client.PDConfig;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.HgStoreClient;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.testutil.Assert;
import org.apache.hugegraph.type.define.GraphMode;
import org.apache.hugegraph.util.Bytes;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.StringEncoding;
public class HstoreSessionsImpl extends HstoreSessions {
private static final Set<String> infoInitializedGraph =
Collections.synchronizedSet(new HashSet<>());
private static int tableCode = 0;
private static volatile Boolean initializedNode = Boolean.FALSE;
private static volatile PDClient defaultPdClient;
private static volatile HgStoreClient hgStoreClient;
private final HugeConfig config;
private final HstoreSession session;
private final Map<String, Integer> tables;
private final AtomicInteger refCount;
private final String graphName;
public HstoreSessionsImpl(HugeConfig config, String database,
String store) {
super(config, database, store);
this.config = config;
this.graphName = database + "/" + store;
this.initStoreNode(config);
this.session = new HstoreSession(this.config, graphName);
this.tables = new ConcurrentHashMap<>();
this.refCount = new AtomicInteger(1);
}
public static HgStoreClient getHgStoreClient() {
return hgStoreClient;
}
public static PDClient getDefaultPdClient() {
return defaultPdClient;
}
public static byte[] encode(String string) {
return StringEncoding.encode(string);
}
public static String decode(byte[] bytes) {
return StringEncoding.decode(bytes);
}
private void initStoreNode(HugeConfig config) {
if (!initializedNode) {
synchronized (this) {
if (!initializedNode) {
PDConfig pdConfig =
PDConfig.of(config.get(HstoreOptions.PD_PEERS))
.setEnableCache(true);
defaultPdClient = PDClient.create(pdConfig);
hgStoreClient =
HgStoreClient.create(defaultPdClient);
initializedNode = Boolean.TRUE;
}
}
}
}
@Override
public void open() throws Exception {
if (!infoInitializedGraph.contains(this.graphName)) {
synchronized (infoInitializedGraph) {
if (!infoInitializedGraph.contains(this.graphName)) {
Integer partitionCount =
this.config.get(HstoreOptions.PARTITION_COUNT);
Assert.assertTrue("The value of hstore.partition_count" +
" cannot be less than 0.",
partitionCount > -1);
defaultPdClient.setGraph(Metapb.Graph.newBuilder()
.setGraphName(
this.graphName)
.setPartitionCount(
partitionCount)
.build());
infoInitializedGraph.add(this.graphName);
}
}
}
this.session.open();
}
@Override
protected boolean opened() {
return this.session != null;
}
@Override
public Set<String> openedTables() {
return this.tables.keySet();
}
@Override
public synchronized void createTable(String... tables) {
for (String table : tables) {
this.session.createTable(table);
this.tables.put(table, tableCode++);
}
}
@Override
public synchronized void dropTable(String... tables) {
for (String table : tables) {
this.session.dropTable(table);
this.tables.remove(table);
}
}
@Override
public boolean existsTable(String table) {
return this.session.existsTable(table);
}
@Override
public void truncateTable(String table) {
this.session.truncateTable(table);
}
@Override
public void clear() {
this.session.deleteGraph();
try {
hgStoreClient.getPdClient().delGraph(this.graphName);
} catch (PDException e) {
}
}
@Override
public final Session session() {
return (Session) super.getOrNewSession();
}
@Override
protected final Session newSession() {
return new HstoreSession(this.config(), this.graphName);
}
@Override
protected synchronized void doClose() {
this.checkValid();
if (this.refCount != null) {
if (this.refCount.decrementAndGet() > 0) {
return;
}
if (this.refCount.get() != 0) {
return;
}
}
assert this.refCount.get() == 0;
this.tables.clear();
this.session.close();
}
private void checkValid() {
}
private static class ColumnIterator<T extends HgKvIterator> implements
BackendColumnIterator,
Countable {
private final T iter;
private final byte[] keyBegin;
private final byte[] keyEnd;
private final int scanType;
private final String table;
private final byte[] value;
private boolean gotNext;
private byte[] position;
public ColumnIterator(String table, T results) {
this(table, results, null, null, 0);
}
public ColumnIterator(String table, T results, byte[] keyBegin,
byte[] keyEnd, int scanType) {
E.checkNotNull(results, "results");
this.table = table;
this.iter = results;
this.keyBegin = keyBegin;
this.keyEnd = keyEnd;
this.scanType = scanType;
this.value = null;
if (this.iter.hasNext()) {
this.iter.next();
this.gotNext = true;
this.position = iter.position();
} else {
this.gotNext = false;
this.position = null;
}
if (!ArrayUtils.isEmpty(this.keyBegin) ||
!ArrayUtils.isEmpty(this.keyEnd)) {
this.checkArguments();
}
}
public T iter() {
return iter;
}
private void checkArguments() {
E.checkArgument(!(this.match(Session.SCAN_PREFIX_BEGIN) &&
this.match(Session.SCAN_PREFIX_END)),
"Can't set SCAN_PREFIX_WITH_BEGIN and " +
"SCAN_PREFIX_WITH_END at the same time");
E.checkArgument(!(this.match(Session.SCAN_PREFIX_BEGIN) &&
this.match(Session.SCAN_GT_BEGIN)),
"Can't set SCAN_PREFIX_WITH_BEGIN and " +
"SCAN_GT_BEGIN/SCAN_GTE_BEGIN at the same time");
E.checkArgument(!(this.match(Session.SCAN_PREFIX_END) &&
this.match(Session.SCAN_LT_END)),
"Can't set SCAN_PREFIX_WITH_END and " +
"SCAN_LT_END/SCAN_LTE_END at the same time");
if (this.match(Session.SCAN_PREFIX_BEGIN) && !matchHash()) {
E.checkArgument(this.keyBegin != null,
"Parameter `keyBegin` can't be null " +
"if set SCAN_PREFIX_WITH_BEGIN");
E.checkArgument(this.keyEnd == null,
"Parameter `keyEnd` must be null " +
"if set SCAN_PREFIX_WITH_BEGIN");
}
if (this.match(Session.SCAN_PREFIX_END) && !matchHash()) {
E.checkArgument(this.keyEnd != null,
"Parameter `keyEnd` can't be null " +
"if set SCAN_PREFIX_WITH_END");
}
if (this.match(Session.SCAN_GT_BEGIN) && !matchHash()) {
E.checkArgument(this.keyBegin != null,
"Parameter `keyBegin` can't be null " +
"if set SCAN_GT_BEGIN or SCAN_GTE_BEGIN");
}
if (this.match(Session.SCAN_LT_END) && !matchHash()) {
E.checkArgument(this.keyEnd != null,
"Parameter `keyEnd` can't be null " +
"if set SCAN_LT_END or SCAN_LTE_END");
}
}
private boolean matchHash() {
return this.scanType == Session.SCAN_HASHCODE;
}
private boolean match(int expected) {
return Session.matchScanType(expected, this.scanType);
}
@Override
public boolean hasNext() {
if (gotNext) {
this.position = this.iter.position();
} else {
this.position = null;
}
return gotNext;
}
private boolean filter(byte[] key) {
if (this.match(Session.SCAN_PREFIX_BEGIN)) {
/*
* Prefix with `keyBegin`?
* TODO: use custom prefix_extractor instead
* or use ReadOptions.prefix_same_as_start
*/
return Bytes.prefixWith(key, this.keyBegin);
} else if (this.match(Session.SCAN_PREFIX_END)) {
/*
* Prefix with `keyEnd`?
* like the following query for range index:
* key > 'age:20' and prefix with 'age'
*/
assert this.keyEnd != null;
return Bytes.prefixWith(key, this.keyEnd);
} else if (this.match(Session.SCAN_LT_END)) {
/*
* Less (equal) than `keyEnd`?
* NOTE: don't use BytewiseComparator due to signed byte
*/
if ((this.scanType | Session.SCAN_HASHCODE) != 0) {
return true;
}
assert this.keyEnd != null;
if (this.match(Session.SCAN_LTE_END)) {
// Just compare the prefix, can be there are excess tail
key = Arrays.copyOfRange(key, 0, this.keyEnd.length);
return Bytes.compare(key, this.keyEnd) <= 0;
} else {
return Bytes.compare(key, this.keyEnd) < 0;
}
} else {
assert this.match(Session.SCAN_ANY) || this.match(Session.SCAN_GT_BEGIN) ||
this.match(
Session.SCAN_GTE_BEGIN) : "Unknown scan type";
return true;
}
}
@Override
public BackendColumn next() {
BackendEntryIterator.checkInterrupted();
if (!this.hasNext()) {
throw new NoSuchElementException();
}
BackendColumn col =
BackendColumn.of(this.iter.key(),
this.iter.value());
if (this.iter.hasNext()) {
gotNext = true;
this.iter.next();
} else {
gotNext = false;
}
return col;
}
@Override
public long count() {
long count = 0L;
while (this.hasNext()) {
this.next();
count++;
BackendEntryIterator.checkInterrupted();
}
return count;
}
@Override
public byte[] position() {
return this.position;
}
@Override
public void close() {
if (this.iter != null) {
this.iter.close();
}
}
}
/**
* HstoreSession implement for hstore
*/
private final class HstoreSession extends Session {
private static final boolean TRANSACTIONAL = true;
private final HgStoreSession graph;
int changedSize = 0;
public HstoreSession(HugeConfig conf, String graphName) {
setGraphName(graphName);
setConf(conf);
this.graph = hgStoreClient.openSession(graphName);
}
@Override
public void open() {
this.opened = true;
}
@Override
public void close() {
this.opened = false;
}
@Override
public boolean closed() {
return !this.opened;
}
@Override
public void reset() {
if (this.changedSize != 0) {
this.rollback();
this.changedSize = 0;
}
}
/**
* Any change in the session
*/
@Override
public boolean hasChanges() {
return this.changedSize > 0;
}
/**
* Commit all updates(put/delete) to DB
*/
@Override
public Integer commit() {
int commitSize = this.changedSize;
if (TRANSACTIONAL) {
this.graph.commit();
}
this.changedSize = 0;
return commitSize;
}
/**
* Rollback all updates(put/delete) not committed
*/
@Override
public void rollback() {
if (TRANSACTIONAL) {
this.graph.rollback();
}
this.changedSize = 0;
}
@Override
public void createTable(String tableName) {
this.graph.createTable(tableName);
}
@Override
public void dropTable(String tableName) {
this.graph.dropTable(tableName);
}
@Override
public boolean existsTable(String tableName) {
return this.graph.existsTable(tableName);
}
@Override
public void truncateTable(String tableName) {
this.graph.deleteTable(tableName);
}
@Override
public void deleteGraph() {
this.graph.deleteGraph(this.getGraphName());
}
@Override
public Pair<byte[], byte[]> keyRange(String table) {
return null;
}
private void prepare() {
if (!this.hasChanges() && TRANSACTIONAL) {
this.graph.beginTx();
}
this.changedSize++;
}
/**
* Add a KV record to a table
*/
@Override
public void put(String table, byte[] ownerKey, byte[] key,
byte[] value) {
prepare();
this.graph.put(table, HgOwnerKey.of(ownerKey, key), value);
}
@Override
public synchronized void increase(String table, byte[] ownerKey,
byte[] key, byte[] value) {
prepare();
this.graph.merge(table, HgOwnerKey.of(ownerKey, key), value);
}
@Override
public void delete(String table, byte[] ownerKey, byte[] key) {
prepare();
this.graph.delete(table, HgOwnerKey.of(ownerKey, key));
}
@Override
public void deletePrefix(String table, byte[] ownerKey, byte[] key) {
prepare();
this.graph.deletePrefix(table, HgOwnerKey.of(ownerKey, key));
}
/**
* Delete a range of keys from a table
*/
@Override
public void deleteRange(String table, byte[] ownerKeyFrom,
byte[] ownerKeyTo, byte[] keyFrom,
byte[] keyTo) {
prepare();
this.graph.deleteRange(table, HgOwnerKey.of(ownerKeyFrom, keyFrom),
HgOwnerKey.of(ownerKeyTo, keyTo));
}
@Override
public byte[] get(String table, byte[] key) {
return this.graph.get(table, HgOwnerKey.of(
HgStoreClientConst.ALL_PARTITION_OWNER, key));
}
@Override
public byte[] get(String table, byte[] ownerKey, byte[] key) {
byte[] values = this.graph.get(table, HgOwnerKey.of(ownerKey, key));
return values != null ? values : new byte[0];
}
@Override
public void beginTx() {
this.graph.beginTx();
}
@Override
public BackendColumnIterator scan(String table) {
assert !this.hasChanges();
return new ColumnIterator<>(table, this.graph.scanIterator(table));
}
@Override
public BackendColumnIterator scan(String table,
byte[] conditionQueryToByte) {
assert !this.hasChanges();
HgKvIterator results =
this.graph.scanIterator(table, conditionQueryToByte);
return new ColumnIterator<>(table, results);
}
@Override
public BackendColumnIterator scan(String table, byte[] ownerKey,
byte[] prefix) {
assert !this.hasChanges();
HgKvIterator<HgKvEntry> result = this.graph.scanIterator(table,
HgOwnerKey.of(
ownerKey,
prefix));
return new ColumnIterator<>(table, result);
}
@Override
public List<BackendColumnIterator> scan(String table,
List<HgOwnerKey> keys,
int scanType, long limit,
byte[] query) {
HgScanQuery scanQuery = HgScanQuery.prefixOf(table, keys).builder()
.setScanType(scanType)
.setQuery(query)
.setPerKeyLimit(limit).build();
List<HgKvIterator<HgKvEntry>> scanIterators =
this.graph.scanBatch(scanQuery);
LinkedList<BackendColumnIterator> columnIterators =
new LinkedList<>();
scanIterators.forEach(item -> {
columnIterators.add(
new ColumnIterator<>(table, item));
});
return columnIterators;
}
@Override
public BackendEntry.BackendIterator<BackendColumnIterator> scan(
String table,
Iterator<HgOwnerKey> keys,
int scanType, Query queryParam, byte[] query) {
//ScanOrderType orderType;
//switch (queryParam.orderType()) {
// case ORDER_NONE:
// orderType = ScanOrderType.ORDER_NONE;
// break;
// case ORDER_WITHIN_VERTEX:
// orderType = ScanOrderType.ORDER_WITHIN_VERTEX;
// break;
// case ORDER_STRICT:
// orderType = ScanOrderType.ORDER_STRICT;
// break;
// default:
// throw new RuntimeException("not implement");
//}
//HgScanQuery scanQuery = HgScanQuery.prefixIteratorOf(table, keys)
// .builder()
// .setScanType(scanType)
// .setQuery(query)
// .setPerKeyMax(queryParam.limit())
// .setOrderType(orderType)
// .setOnlyKey(
// !queryParam.withProperties())
// .setSkipDegree(
// queryParam.skipDegree())
// .build();
//KvCloseableIterator<HgKvIterator<HgKvEntry>> scanIterators =
// this.graph.scanBatch2(scanQuery);
//return new BackendEntry.BackendIterator<>() {
// @Override
// public void close() {
// scanIterators.close();
// }
//
// @Override
// public byte[] position() {
// throw new NotImplementedException();
// }
//
// @Override
// public boolean hasNext() {
// return scanIterators.hasNext();
// }
//
// @Override
// public BackendColumnIterator next() {
// return new ColumnIterator<HgKvIterator>(table,
// scanIterators.next());
// }
//};
return null;
}
@Override
public BackendColumnIterator scan(String table, byte[] ownerKeyFrom,
byte[] ownerKeyTo,
byte[] keyFrom, byte[] keyTo,
int scanType) {
assert !this.hasChanges();
HgKvIterator result = this.graph.scanIterator(table, HgOwnerKey.of(
ownerKeyFrom, keyFrom),
HgOwnerKey.of(
ownerKeyTo,
keyTo), 0,
scanType,
null);
return new ColumnIterator<>(table, result, keyFrom,
keyTo, scanType);
}
@Override
public BackendColumnIterator scan(String table, byte[] ownerKeyFrom,
byte[] ownerKeyTo,
byte[] keyFrom, byte[] keyTo,
int scanType, byte[] query) {
assert !this.hasChanges();
HgKvIterator<HgKvEntry> result = this.graph.scanIterator(table,
HgOwnerKey.of(
ownerKeyFrom,
keyFrom),
HgOwnerKey.of(
ownerKeyTo,
keyTo),
0,
scanType,
query);
return new ColumnIterator<>(table, result, keyFrom, keyTo,
scanType);
}
@Override
public BackendColumnIterator scan(String table, byte[] ownerKeyFrom,
byte[] ownerKeyTo,
byte[] keyFrom, byte[] keyTo,
int scanType, byte[] query,
byte[] position) {
assert !this.hasChanges();
HgKvIterator<HgKvEntry> result = this.graph.scanIterator(table,
HgOwnerKey.of(
ownerKeyFrom,
keyFrom),
HgOwnerKey.of(
ownerKeyTo,
keyTo),
0,
scanType,
query);
result.seek(position);
return new ColumnIterator<>(table, result, keyFrom, keyTo,
scanType);
}
@Override
public BackendColumnIterator scan(String table, int codeFrom,
int codeTo, int scanType,
byte[] query) {
assert !this.hasChanges();
HgKvIterator<HgKvEntry> iterator =
this.graph.scanIterator(table, codeFrom, codeTo, 256,
new byte[0]);
return new ColumnIterator<>(table, iterator, new byte[0],
new byte[0], scanType);
}
@Override
public BackendColumnIterator scan(String table, int codeFrom,
int codeTo, int scanType,
byte[] query, byte[] position) {
assert !this.hasChanges();
HgKvIterator<HgKvEntry> iterator =
this.graph.scanIterator(table, codeFrom, codeTo, 256,
new byte[0]);
iterator.seek(position);
return new ColumnIterator<>(table, iterator, new byte[0],
new byte[0], scanType);
}
@Override
public BackendColumnIterator getWithBatch(String table,
List<HgOwnerKey> keys) {
assert !this.hasChanges();
HgKvIterator<HgKvEntry> kvIterator =
this.graph.batchPrefix(table, keys);
return new ColumnIterator<>(table, kvIterator);
}
@Override
public void merge(String table, byte[] ownerKey, byte[] key,
byte[] value) {
prepare();
this.graph.merge(table, HgOwnerKey.of(ownerKey, key), value);
}
@Override
public void setMode(GraphMode mode) {
// no need to set pd mode
}
@Override
public void truncate() throws Exception {
this.graph.truncate();
HstoreSessionsImpl.getDefaultPdClient()
.resetIdByKey(this.getGraphName());
}
}
}

811
hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreStore.java Normal file
View File

@ -0,0 +1,811 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.backend.store.hstore;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hugegraph.backend.id.Id;
import org.apache.hugegraph.backend.id.IdGenerator;
import org.apache.hugegraph.backend.query.IdPrefixQuery;
import org.apache.hugegraph.backend.query.IdQuery;
import org.apache.hugegraph.backend.query.Query;
import org.apache.hugegraph.backend.serializer.BinaryBackendEntry;
import org.apache.hugegraph.backend.serializer.BytesBuffer;
import org.apache.hugegraph.backend.store.AbstractBackendStore;
import org.apache.hugegraph.backend.store.BackendAction;
import org.apache.hugegraph.backend.store.BackendEntry;
import org.apache.hugegraph.backend.store.BackendFeatures;
import org.apache.hugegraph.backend.store.BackendMutation;
import org.apache.hugegraph.backend.store.BackendStoreProvider;
import org.apache.hugegraph.backend.store.BackendTable;
import org.apache.hugegraph.backend.store.hstore.HstoreSessions.Session;
import org.apache.hugegraph.config.CoreOptions;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.iterator.CIter;
import org.apache.hugegraph.type.HugeTableType;
import org.apache.hugegraph.type.HugeType;
import org.apache.hugegraph.type.define.Action;
import org.apache.hugegraph.type.define.GraphMode;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
public abstract class HstoreStore extends AbstractBackendStore<Session> {
private static final Logger LOG = Log.logger(HstoreStore.class);
private static final Set<HugeType> INDEX_TYPES = ImmutableSet.of(
HugeType.SECONDARY_INDEX, HugeType.VERTEX_LABEL_INDEX,
HugeType.EDGE_LABEL_INDEX, HugeType.RANGE_INT_INDEX,
HugeType.RANGE_FLOAT_INDEX, HugeType.RANGE_LONG_INDEX,
HugeType.RANGE_DOUBLE_INDEX, HugeType.SEARCH_INDEX,
HugeType.SHARD_INDEX, HugeType.UNIQUE_INDEX
);
private static final BackendFeatures FEATURES = new HstoreFeatures();
private final String store, namespace;
private final BackendStoreProvider provider;
private final Map<Integer, HstoreTable> tables;
private final ReadWriteLock storeLock;
private boolean isGraphStore;
private HstoreSessions sessions;
public HstoreStore(final BackendStoreProvider provider,
String namespace, String store) {
this.tables = new HashMap<>();
this.provider = provider;
this.namespace = namespace;
this.store = store;
this.sessions = null;
this.storeLock = new ReentrantReadWriteLock();
this.registerMetaHandlers();
LOG.debug("Store loaded: {}", store);
}
private void registerMetaHandlers() {
this.registerMetaHandler("metrics", (session, meta, args) -> {
return ImmutableMap.of();
});
this.registerMetaHandler("mode", (session, meta, args) -> {
E.checkArgument(args.length == 1,
"The args count of %s must be 1", meta);
session.setMode((GraphMode) args[0]);
return null;
});
}
protected void registerTableManager(HugeTableType type, HstoreTable table) {
this.tables.put((int) type.code(), table);
}
@Override
protected final HstoreTable table(HugeType type) {
assert type != null;
HugeTableType table;
switch (type) {
case VERTEX:
table = HugeTableType.VERTEX;
break;
case EDGE_OUT:
table = HugeTableType.OUT_EDGE;
break;
case EDGE_IN:
table = HugeTableType.IN_EDGE;
break;
case OLAP:
table = HugeTableType.OLAP_TABLE;
break;
case TASK:
table = HugeTableType.TASK_INFO_TABLE;
break;
case SERVER:
table = HugeTableType.SERVER_INFO_TABLE;
break;
case SEARCH_INDEX:
case SHARD_INDEX:
case SECONDARY_INDEX:
case RANGE_INT_INDEX:
case RANGE_LONG_INDEX:
case RANGE_FLOAT_INDEX:
case RANGE_DOUBLE_INDEX:
case EDGE_LABEL_INDEX:
case VERTEX_LABEL_INDEX:
case UNIQUE_INDEX:
table = HugeTableType.ALL_INDEX_TABLE;
break;
default:
throw new AssertionError(String.format(
"Invalid type: %s", type));
}
return this.tables.get((int) table.code());
}
protected List<String> tableNames() {
return this.tables.values().stream()
.map(BackendTable::table)
.collect(Collectors.toList());
}
@Override
protected Session session(HugeType type) {
this.checkOpened();
return this.sessions.session();
}
public String namespace() {
return this.namespace;
}
@Override
public String store() {
return this.store;
}
@Override
public String database() {
return this.namespace;
}
@Override
public BackendStoreProvider provider() {
return this.provider;
}
@Override
public BackendFeatures features() {
return FEATURES;
}
@Override
public synchronized void open(HugeConfig config) {
E.checkNotNull(config, "config");
if (this.sessions == null) {
this.sessions = new HstoreSessionsImpl(config, this.namespace,
this.store);
}
String graphStore = config.get(CoreOptions.STORE_GRAPH);
this.isGraphStore = this.store.equals(graphStore);
assert this.sessions != null;
if (!this.sessions.closed()) {
LOG.debug("Store {} has been opened before", this.store);
this.sessions.useSession();
return;
}
try {
// NOTE: won't throw error even if connection refused
this.sessions.open();
} catch (Exception e) {
LOG.error("Failed to open Hstore '{}':{}", this.store, e);
}
this.sessions.session();
LOG.debug("Store opened: {}", this.store);
}
@Override
public void close() {
this.checkOpened();
this.sessions.close();
LOG.debug("Store closed: {}", this.store);
}
@Override
public boolean opened() {
this.checkConnectionOpened();
return this.sessions.session().opened();
}
@Override
public void mutate(BackendMutation mutation) {
Session session = this.sessions.session();
assert session.opened();
Map<HugeType, Map<Id, List<BackendAction>>> mutations = mutation.mutations();
Set<Map.Entry<HugeType, Map<Id, List<BackendAction>>>> entries = mutations.entrySet();
for (Map.Entry<HugeType, Map<Id, List<BackendAction>>> entry : entries) {
HugeType key = entry.getKey();
// in order to obtain the owner efficiently, special for edge
boolean isEdge = key.isEdge();
HstoreTable hTable = this.table(key);
Map<Id, List<BackendAction>> table = entry.getValue();
Collection<List<BackendAction>> values = table.values();
for (List<BackendAction> items : values) {
for (int i = 0; i < items.size(); i++) {
BackendAction item = items.get(i);
// set to ArrayList, use index to get item
this.mutate(session, item, hTable, isEdge);
}
}
}
}
private void mutate(Session session, BackendAction item,
HstoreTable hTable, boolean isEdge) {
BackendEntry entry = item.entry();
HstoreTable table;
if (!entry.olap()) {
// Oltp table
table = hTable;
} else {
if (entry.type().isIndex()) {
// Olap index
table = this.table(entry.type());
} else {
// Olap vertex
table = this.table(HugeType.OLAP);
}
session = this.session(HugeType.OLAP);
}
if (item.action().code() == Action.INSERT.code()) {
table.insert(session, entry, isEdge);
} else {
if (item.action().code() == Action.APPEND.code()) {
table.append(session, entry);
} else {
switch (item.action()) {
case DELETE:
table.delete(session, entry);
break;
case ELIMINATE:
table.eliminate(session, entry);
break;
case UPDATE_IF_PRESENT:
table.updateIfPresent(session, entry);
break;
case UPDATE_IF_ABSENT:
table.updateIfAbsent(session, entry);
break;
default:
throw new AssertionError(String.format(
"Unsupported mutate action: %s",
item.action()));
}
}
}
}
private HstoreTable getTableByQuery(Query query) {
HugeType tableType = HstoreTable.tableType(query);
HstoreTable table;
if (query.olap()) {
if (query.resultType().isIndex()) {
// Any index type is ok here
table = this.table(HugeType.SECONDARY_INDEX);
} else {
table = this.table(HugeType.OLAP);
}
} else {
table = this.table(tableType);
}
return table;
}
@Override
public Iterator<BackendEntry> query(Query query) {
Lock readLock = this.storeLock.readLock();
readLock.lock();
try {
this.checkOpened();
Session session = this.sessions.session();
HstoreTable table = getTableByQuery(query);
Iterator<BackendEntry> entries = table.query(session, query);
// Merge olap results as needed
entries = getBackendEntryIterator(entries, query);
return entries;
} finally {
readLock.unlock();
}
}
//@Override
//public Iterator<Iterator<BackendEntry>> query(Iterator<Query> queries,
// Function<Query, Query> queryWriter,
// HugeGraph hugeGraph) {
// if (queries == null || !queries.hasNext()) {
// return Collections.emptyIterator();
// }
//
// class QueryWrapper implements Iterator<IdPrefixQuery> {
// Query first;
// final Iterator<Query> queries;
// Iterator<Id> subEls;
// Query preQuery;
// Iterator<IdPrefixQuery> queryListIterator;
//
// QueryWrapper(Iterator<Query> queries, Query first) {
// this.queries = queries;
// this.first = first;
// }
//
// @Override
// public boolean hasNext() {
// return first != null || (this.subEls != null && this.subEls.hasNext())
// || (queryListIterator != null && queryListIterator.hasNext()) ||
// queries.hasNext();
// }
//
// @Override
// public IdPrefixQuery next() {
// if (queryListIterator != null && queryListIterator.hasNext()) {
// return queryListIterator.next();
// }
//
// Query q;
// if (first != null) {
// q = first;
// preQuery = q.copy();
// first = null;
// } else {
// if (this.subEls == null || !this.subEls.hasNext()) {
// q = queries.next();
// preQuery = q.copy();
// } else {
// q = preQuery.copy();
// }
// }
//
// assert q instanceof ConditionQuery;
// ConditionQuery cq = (ConditionQuery) q;
// ConditionQuery originQuery = (ConditionQuery) q.copy();
//
// List<IdPrefixQuery> queryList = Lists.newArrayList();
// if (hugeGraph != null) {
// for (ConditionQuery conditionQuery :
// ConditionQueryFlatten.flatten(cq)) {
// Id label = conditionQuery.condition(HugeKeys.LABEL);
// /* 父类型 + sortKeys： g.V("V.id").outE("parentLabel").has
// ("sortKey","value")转成 所有子类型 + sortKeys*/
// if ((this.subEls == null ||
// !this.subEls.hasNext()) && label != null &&
// hugeGraph.edgeLabel(label).isFather() &&
// conditionQuery.condition(HugeKeys.SUB_LABEL) ==
// null &&
// conditionQuery.condition(HugeKeys.OWNER_VERTEX) !=
// null &&
// conditionQuery.condition(HugeKeys.DIRECTION) !=
// null &&
// matchEdgeSortKeys(conditionQuery, false,
// hugeGraph)) {
// this.subEls =
// getSubLabelsOfParentEl(
// hugeGraph.edgeLabels(),
// label);
// }
//
// if (this.subEls != null &&
// this.subEls.hasNext()) {
// conditionQuery.eq(HugeKeys.SUB_LABEL,
// subEls.next());
// }
//
// HugeType hugeType = conditionQuery.resultType();
// if (hugeType != null && hugeType.isEdge() &&
// !conditionQuery.conditions().isEmpty()) {
// IdPrefixQuery idPrefixQuery =
// (IdPrefixQuery) queryWriter.apply(
// conditionQuery);
// idPrefixQuery.setOriginQuery(originQuery);
// queryList.add(idPrefixQuery);
// }
// }
//
// queryListIterator = queryList.iterator();
// if (queryListIterator.hasNext()) {
// return queryListIterator.next();
// }
// }
//
// Id ownerId = cq.condition(HugeKeys.OWNER_VERTEX);
// assert ownerId != null;
// BytesBuffer buffer =
// BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID);
// buffer.writeId(ownerId);
// return new IdPrefixQuery(cq, new BinaryBackendEntry.BinaryId(
// buffer.bytes(), ownerId));
// }
//
// private boolean matchEdgeSortKeys(ConditionQuery query,
// boolean matchAll,
// HugeGraph graph) {
// assert query.resultType().isEdge();
// Id label = query.condition(HugeKeys.LABEL);
// if (label == null) {
// return false;
// }
// List<Id> sortKeys = graph.edgeLabel(label).sortKeys();
// if (sortKeys.isEmpty()) {
// return false;
// }
// Set<Id> queryKeys = query.userpropKeys();
// for (int i = sortKeys.size(); i > 0; i--) {
// List<Id> subFields = sortKeys.subList(0, i);
// if (queryKeys.containsAll(subFields)) {
// if (queryKeys.size() == subFields.size() || !matchAll) {
// /*
// * Return true if:
// * matchAll=true and all queryKeys are in sortKeys
// * or
// * partial queryKeys are in sortKeys
// */
// return true;
// }
// }
// }
// return false;
// }
// }
// Query first = queries.next();
// List<HugeType> typeList = getHugeTypes(first);
// QueryWrapper idPrefixQueries = new QueryWrapper(queries, first);
//
// return query(typeList, idPrefixQueries);
//}
//private Iterator<Id> getSubLabelsOfParentEl(Collection<EdgeLabel> allEls,
// Id label) {
// List<Id> list = new ArrayList<>();
// for (EdgeLabel el : allEls) {
// if (el.edgeLabelType().sub() && el.fatherId().equals(label)) {
// list.add(el.id());
// }
// }
// return list.iterator();
//}
public List<CIter<BackendEntry>> query(List<HugeType> typeList,
List<IdPrefixQuery> queries) {
Lock readLock = this.storeLock.readLock();
readLock.lock();
LinkedList<CIter<BackendEntry>> results = new LinkedList<>();
try {
this.checkOpened();
Session session = this.sessions.session();
E.checkState(!CollectionUtils.isEmpty(queries) &&
!CollectionUtils.isEmpty(typeList),
"Please check query list or type list.");
HstoreTable table = null;
StringBuilder builder = new StringBuilder();
for (HugeType type : typeList) {
builder.append((table = this.table(type)).table()).append(",");
}
List<Iterator<BackendEntry>> iteratorList =
table.query(session, queries,
builder.substring(0, builder.length() - 1));
for (int i = 0; i < iteratorList.size(); i++) {
Iterator<BackendEntry> entries = iteratorList.get(i);
// Merge olap results as needed
Query query = queries.get(i);
entries = getBackendEntryIterator(entries, query);
if (entries instanceof CIter) {
results.add((CIter) entries);
}
}
return results;
} finally {
readLock.unlock();
}
}
public Iterator<Iterator<BackendEntry>> query(List<HugeType> typeList,
Iterator<IdPrefixQuery> queries) {
Lock readLock = this.storeLock.readLock();
readLock.lock();
try {
this.checkOpened();
Session session = this.sessions.session();
E.checkState(queries.hasNext() &&
!CollectionUtils.isEmpty(typeList),
"Please check query list or type list.");
HstoreTable table = null;
StringBuilder builder = new StringBuilder();
for (HugeType type : typeList) {
builder.append((table = this.table(type)).table()).append(",");
}
Iterator<Iterator<BackendEntry>> iterators =
table.query(session, queries,
builder.substring(0, builder.length() - 1));
return iterators;
} finally {
readLock.unlock();
}
}
private Iterator<BackendEntry> getBackendEntryIterator(
Iterator<BackendEntry> entries,
Query query) {
//HstoreTable table;
//Set<Id> olapPks = query.olapPks();
//if (this.isGraphStore && !olapPks.isEmpty()) {
// List<Iterator<BackendEntry>> iterators = new ArrayList<>();
// for (Id pk : olapPks) {
// // 构造olap表查询query condition
// Query q = this.constructOlapQueryCondition(pk, query);
// table = this.table(HugeType.OLAP);
// iterators.add(table.queryOlap(this.session(HugeType.OLAP), q));
// }
// entries = new MergeIterator<>(entries, iterators,
// BackendEntry::mergable);
//}
return entries;
}
/**
* 重新构造 查询olap表 query
* 由于 olap合并成一张表, 在写入olap数据, key在后面增加了pk
* 所以在此进行查询的时候,需要重新构造pk前缀
* 写入参考 BinarySerializer.writeOlapVertex
*
* @param pk
* @param query
* @return
*/
private Query constructOlapQueryCondition(Id pk, Query query) {
if (query instanceof IdQuery && !CollectionUtils.isEmpty((query).ids())) {
IdQuery q = (IdQuery) query.copy();
Iterator<Id> iterator = q.ids().iterator();
LinkedHashSet<Id> linkedHashSet = new LinkedHashSet<>();
while (iterator.hasNext()) {
Id id = iterator.next();
if (id instanceof BinaryBackendEntry.BinaryId) {
id = ((BinaryBackendEntry.BinaryId) id).origin();
}
// create binary id
BytesBuffer buffer =
BytesBuffer.allocate(1 + pk.length() + 1 + id.length());
buffer.writeId(pk);
id = new BinaryBackendEntry.BinaryId(
buffer.writeId(id).bytes(), id);
linkedHashSet.add(id);
}
q.resetIds();
q.query(linkedHashSet);
return q;
} else {
// create binary id
BytesBuffer buffer = BytesBuffer.allocate(1 + pk.length());
pk = new BinaryBackendEntry.BinaryId(
buffer.writeId(pk).bytes(), pk);
IdPrefixQuery idPrefixQuery = new IdPrefixQuery(HugeType.OLAP, pk);
return idPrefixQuery;
}
}
@Override
public Number queryNumber(Query query) {
this.checkOpened();
Session session = this.sessions.session();
HstoreTable table = this.table(HstoreTable.tableType(query));
return table.queryNumber(session, query);
}
@Override
public synchronized void init() {
Lock writeLock = this.storeLock.writeLock();
writeLock.lock();
try {
// Create tables with main disk
this.sessions.createTable(this.tableNames().toArray(new String[0]));
LOG.debug("Store initialized: {}", this.store);
} finally {
writeLock.unlock();
}
}
@Override
public void clear(boolean clearSpace) {
Lock writeLock = this.storeLock.writeLock();
writeLock.lock();
try {
// Drop tables with main disk
this.sessions.dropTable(this.tableNames().toArray(new String[0]));
if (clearSpace) {
this.sessions.clear();
}
LOG.debug("Store cleared: {}", this.store);
} finally {
writeLock.unlock();
}
}
@Override
public boolean initialized() {
return true;
}
@Override
public void truncate() {
try {
this.sessions.session().truncate();
} catch (Exception e) {
LOG.error("Store truncated failed", e);
return;
}
LOG.debug("Store truncated: {}", this.store);
}
@Override
public void beginTx() {
this.sessions.session().beginTx();
}
@Override
public void commitTx() {
this.checkOpened();
Session session = this.sessions.session();
session.commit();
}
@Override
public void rollbackTx() {
this.checkOpened();
Session session = this.sessions.session();
session.rollback();
}
private void checkConnectionOpened() {
}
@Override
public Id nextId(HugeType type) {
long counter = 0L;
counter = this.getCounter(type);
E.checkState(counter != 0L, "Please check whether '%s' is OK",
this.provider().type());
return IdGenerator.of(counter);
}
@Override
public void setCounterLowest(HugeType type, long lowest) {
this.increaseCounter(type, lowest);
}
/***************************** Store defines *****************************/
public static class HstoreSchemaStore extends HstoreStore {
public HstoreSchemaStore(BackendStoreProvider provider, String namespace, String store) {
super(provider, namespace, store);
}
@Override
public boolean isSchemaStore() {
return true;
}
@Override
public void increaseCounter(HugeType type, long num) {
throw new UnsupportedOperationException(
"HstoreSchemaStore.increaseCounter()");
}
@Override
public long getCounter(HugeType type) {
throw new UnsupportedOperationException(
"HstoreSchemaStore.getCounter()");
}
}
public static class HstoreGraphStore extends HstoreStore {
public HstoreGraphStore(BackendStoreProvider provider,
String namespace, String store) {
super(provider, namespace, store);
registerTableManager(HugeTableType.VERTEX,
new HstoreTables.Vertex(store));
registerTableManager(HugeTableType.OUT_EDGE,
HstoreTables.Edge.out(store));
registerTableManager(HugeTableType.IN_EDGE,
HstoreTables.Edge.in(store));
registerTableManager(HugeTableType.ALL_INDEX_TABLE,
new HstoreTables.IndexTable(store));
registerTableManager(HugeTableType.OLAP_TABLE,
new HstoreTables.OlapTable(store));
registerTableManager(HugeTableType.TASK_INFO_TABLE,
new HstoreTables.TaskInfo(store));
registerTableManager(HugeTableType.SERVER_INFO_TABLE,
new HstoreTables.ServerInfo(store));
}
@Override
public boolean isSchemaStore() {
return false;
}
@Override
public Id nextId(HugeType type) {
throw new UnsupportedOperationException(
"HstoreGraphStore.nextId()");
}
@Override
public void increaseCounter(HugeType type, long num) {
throw new UnsupportedOperationException(
"HstoreGraphStore.increaseCounter()");
}
@Override
public long getCounter(HugeType type) {
throw new UnsupportedOperationException(
"HstoreGraphStore.getCounter()");
}
@Override
public void createOlapTable(Id pkId) {
HstoreTable table = new HstoreTables.OlapTable(this.store());
LOG.info("Hstore create olap table {}", table.table());
super.sessions.createTable(table.table());
LOG.info("Hstore finish create olap table");
registerTableManager(HugeTableType.OLAP_TABLE, table);
LOG.info("OLAP table {} has been created", table.table());
}
@Override
public void checkAndRegisterOlapTable(Id pkId) {
HstoreTable table = new HstoreTables.OlapTable(this.store());
if (!super.sessions.existsTable(table.table())) {
LOG.error("Found exception: Table '{}' doesn't exist, we'll " +
"recreate it now. Please carefully check the recent" +
"operation in server and computer, then ensure the " +
"integrity of store file.", table.table());
this.createOlapTable(pkId);
} else {
registerTableManager(HugeTableType.OLAP_TABLE, table);
}
}
@Override
public void clearOlapTable(Id pkId) {
}
@Override
public void removeOlapTable(Id pkId) {
}
}
@Override
public String storedVersion() {
return "1.13";
}
}

Some files were not shown because too many files have changed in this diff Show More