anph
/
hugegraph-sync
forked from hugegraph/hugegraph-sync
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HugeGraph-929: implement RocksDB driver 

Change-Id: I9baa4b4e648848eb9f3f017ad0c8c429f07717b7
This commit is contained in:
Zhangmei Li 2017-11-23 19:39:33 +08:00 committed by liningrui
parent 6a0d790220
commit b7db9b527a
89 changed files with 4599 additions and 867 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hugegraph-api/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>hugegraph</artifactId>
<groupId>com.baidu.hugegraph</groupId>
<version>0.4.1-SNAPSHOT</version>
<version>0.4.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

2
hugegraph-cassandra/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>hugegraph</artifactId>
<groupId>com.baidu.hugegraph</groupId>
<version>0.4.1-SNAPSHOT</version>
<version>0.4.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

13
hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraBackendEntry.java
View File

@ -130,6 +130,7 @@ public class CassandraBackendEntry implements BackendEntry {
this.selfChanged = true;
}
@Override
public HugeType type() {
return this.row.type;
}
@ -143,7 +144,6 @@ public class CassandraBackendEntry implements BackendEntry {
return this.row.id;
}
@Override
public void id(Id id) {
this.row.id = id;
}
@ -153,7 +153,6 @@ public class CassandraBackendEntry implements BackendEntry {
return this.row.subId;
}
@Override
public void subId(Id subId) {
this.row.subId = subId;
}
@ -215,8 +214,18 @@ public class CassandraBackendEntry implements BackendEntry {
throw new RuntimeException("Not supported by Cassandra");
}
@Override
public void columns(BackendColumn... bytesColumns) {
throw new RuntimeException("Not supported by Cassandra");
}
@Override
public void merge(BackendEntry other) {
throw new RuntimeException("Not supported by Cassandra");
}
@Override
public void clear() {
throw new RuntimeException("Not supported by Cassandra");
}
}

61
hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraFeatures.java
View File

@ -24,7 +24,29 @@ import com.baidu.hugegraph.backend.store.BackendFeatures;
public class CassandraFeatures implements BackendFeatures {
@Override
public boolean supportsDeleteEdgeByLabel() {
public boolean supportsScanToken() {
return true;
}
@Override
public boolean supportsScanKeyPrefix() {
return false;
}
@Override
public boolean supportsScanKeyRange() {
return false;
}
@Override
public boolean supportsQuerySchemaByName() {
// Cassandra support secondary index
return true;
}
@Override
public boolean supportsQueryByLabel() {
// Cassandra support secondary index
return true;
}
@ -33,21 +55,31 @@ public class CassandraFeatures implements BackendFeatures {
return true;
}
@Override
public boolean supportsQueryWithOrderBy() {
return true;
}
@Override
public boolean supportsQueryWithContains() {
return true;
}
@Override
public boolean supportsQueryWithContainsKey() {
return true;
}
@Override
public boolean supportsDeleteEdgeByLabel() {
return true;
}
@Override
public boolean supportsUpdateEdgeProperty() {
return true;
}
@Override
public boolean supportsOrderByQuery() {
return true;
}
@Override
public boolean supportsScan() {
return true;
}
@Override
public boolean supportsTransaction() {
// Cassandra support tx(atomicity level) with batch API
@ -56,12 +88,7 @@ public class CassandraFeatures implements BackendFeatures {
}
@Override
public boolean supportsQueryByContains() {
return true;
}
@Override
public boolean supportsQueryByContainsKey() {
public boolean supportsNumberType() {
return true;
}
}

10
hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSerializer.java
View File

@ -59,15 +59,11 @@ import com.google.common.collect.ImmutableMap;
public class CassandraSerializer extends AbstractSerializer {
@Override
public CassandraBackendEntry newBackendEntry(HugeType type, Id id) {
return new CassandraBackendEntry(type, id);
}
@Override
public BackendEntry newBackendEntry(Id id) {
return newBackendEntry(null, id);
}
protected CassandraBackendEntry newBackendEntry(HugeElement e) {
return newBackendEntry(e.type(), e.id());
}
@ -84,8 +80,7 @@ public class CassandraSerializer extends AbstractSerializer {
@Override
protected BackendEntry convertEntry(BackendEntry backendEntry) {
if (!(backendEntry instanceof CassandraBackendEntry)) {
throw new BackendException(
"CassandraSerializer just supports CassandraBackendEntry");
throw new BackendException("Not supported by CassandraSerializer");
}
return backendEntry;
}
@ -152,7 +147,6 @@ public class CassandraSerializer extends AbstractSerializer {
String sortValues = row.column(HugeKeys.SORT_VALUES);
String targetVertexId = row.column(HugeKeys.OTHER_VERTEX);
if (vertex == null) {
Id id = IdGenerator.of(sourceVertexId);
vertex = new HugeVertex(graph, id, null);

113
hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java
View File

@ -20,41 +20,30 @@
package com.baidu.hugegraph.backend.store.cassandra;
import java.util.Collection;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import com.baidu.hugegraph.backend.BackendException;
import com.baidu.hugegraph.backend.store.BackendStore.TxState;
import com.baidu.hugegraph.backend.store.BackendSessionPool;
import com.baidu.hugegraph.config.CassandraOptions;
import com.baidu.hugegraph.config.HugeConfig;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.Log;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.Metadata;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.SocketOptions;
import com.datastax.driver.core.Statement;
import com.datastax.driver.core.exceptions.InvalidQueryException;
public class CassandraSessionPool {
private static final Logger LOG = Log.logger(CassandraStore.class);
public class CassandraSessionPool extends BackendSessionPool {
private static final int SECOND = 1000;
private Cluster cluster;
private String keyspace;
private ThreadLocal<Session> threadLocalSession;
private AtomicInteger sessionCount;
public CassandraSessionPool(String keyspace) {
this.cluster = null;
this.keyspace = keyspace;
this.threadLocalSession = new ThreadLocal<>();
this.sessionCount = new AtomicInteger(0);
}
public synchronized void open(HugeConfig config) {
@ -91,51 +80,21 @@ public class CassandraSessionPool {
}
public final synchronized Session session() {
Session session = this.threadLocalSession.get();
if (session == null) {
E.checkState(this.cluster != null,
"Cassandra cluster has not been initialized");
session = new Session(this.cluster.connect(this.keyspace));
this.threadLocalSession.set(session);
this.sessionCount.incrementAndGet();
LOG.debug("Now(after connect()) session count is: {}",
this.sessionCount.get());
}
return session;
return (Session) super.getOrNewSession();
}
public void useSession() {
Session session = this.threadLocalSession.get();
if (session == null) {
return;
}
session.attach();
@Override
protected final synchronized Session newSession() {
E.checkState(this.cluster != null,
"Cassandra cluster has not been initialized");
return new Session();
}
public void closeSession() {
Session session = this.threadLocalSession.get();
if (session == null) {
return;
@Override
protected synchronized void doClose() {
if (this.cluster != null && !this.cluster.isClosed()) {
this.cluster.close();
}
if (session.detach() <= 0) {
session.close();
this.threadLocalSession.remove();
this.sessionCount.decrementAndGet();
}
}
public synchronized void close() {
try {
this.closeSession();
} finally {
if (this.sessionCount.get() == 0 &&
this.cluster != null &&
!this.cluster.isClosed()) {
this.cluster.close();
}
}
LOG.debug("Now(after close()) session count is: {}",
this.sessionCount.get());
}
public final void checkClusterConnected() {
@ -158,36 +117,29 @@ public class CassandraSessionPool {
* The Session class is a wrapper of driver Session
* Expect every thread hold a its own session(wrapper)
*/
public final class Session {
public final class Session extends BackendSessionPool.Session {
private com.datastax.driver.core.Session session;
private BatchStatement batch;
private int refs;
private TxState txState;
public Session(com.datastax.driver.core.Session session) {
this.session = session;
public Session() {
this.session = null;
this.batch = new BatchStatement();
this.refs = 1;
this.txState = TxState.CLEAN;
}
private int attach() {
return ++this.refs;
}
private int detach() {
return --this.refs;
try {
this.open();
} catch (InvalidQueryException ignored) {}
}
public BatchStatement add(Statement statement) {
return this.batch.add(statement);
}
@Override
public void clear() {
this.batch.clear();
}
@Override
public ResultSet commit() {
return this.session.execute(this.batch);
}
@ -204,11 +156,24 @@ public class CassandraSessionPool {
return this.session.execute(statement, args);
}
public void open() {
this.session = cluster().connect(keyspace());
}
@Override
public boolean closed() {
if (this.session == null) {
return true;
}
return this.session.isClosed();
}
private void close() {
@Override
public void close() {
assert this.closeable();
if (this.session == null) {
return;
}
this.session.close();
}
@ -220,14 +185,6 @@ public class CassandraSessionPool {
return this.batch.getStatements();
}
public TxState txState() {
return this.txState;
}
public void txState(TxState state) {
this.txState = state;
}
public String keyspace() {
return CassandraSessionPool.this.keyspace;
}

15
hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java
View File

@ -20,6 +20,7 @@
package com.baidu.hugegraph.backend.store.cassandra;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@ -112,7 +113,7 @@ public abstract class CassandraStore implements BackendStore {
try {
LOG.debug("Store connect with keyspace: {}", this.keyspace);
try {
this.sessions.session();
this.sessions.session().open();
} catch (InvalidQueryException e) {
// TODO: the error message may be changed in different versions
if (!e.getMessage().contains(String.format(
@ -120,8 +121,7 @@ public abstract class CassandraStore implements BackendStore {
throw e;
}
LOG.info("Failed to connect keyspace: {}, " +
"try init keyspace later", this.keyspace);
this.sessions.closeSession();
"try to init keyspace later", this.keyspace);
}
} catch (Throwable e) {
try {
@ -211,7 +211,7 @@ public abstract class CassandraStore implements BackendStore {
}
@Override
public Iterable<BackendEntry> query(Query query) {
public Iterator<BackendEntry> query(Query query) {
this.checkSessionConnected();
CassandraTable table = this.table(query.resultType());
@ -234,8 +234,9 @@ public abstract class CassandraStore implements BackendStore {
@Override
public void init() {
this.checkClusterConnected();
this.initKeyspace();
this.checkSessionConnected();
this.initTables();
LOG.info("Store initialized: {}", this.name);
@ -246,6 +247,7 @@ public abstract class CassandraStore implements BackendStore {
this.checkClusterConnected();
if (this.existsKeyspace()) {
this.checkSessionConnected();
this.clearTables();
this.clearKeyspace();
}
@ -363,6 +365,7 @@ public abstract class CassandraStore implements BackendStore {
session.close();
}
}
this.sessions.session().open();
}
protected void clearKeyspace() {
@ -402,7 +405,7 @@ public abstract class CassandraStore implements BackendStore {
assert type != null;
CassandraTable table = this.tables.get(type);
if (table == null) {
throw new BackendException("Unsupported type: %s", type.name());
throw new BackendException("Unsupported table type: %s", type);
}
return table;
}

4
hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStoreProvider.java
View File

@ -49,7 +49,7 @@ public class CassandraStoreProvider extends AbstractBackendStoreProvider {
BackendStore store = this.stores.get(name);
E.checkNotNull(store, "store");
E.checkState(store instanceof CassandraStore.CassandraSchemaStore,
"SchemaStore must be a instance of CassandraSchemaStore");
"SchemaStore must be an instance of CassandraSchemaStore");
return store;
}
@ -66,7 +66,7 @@ public class CassandraStoreProvider extends AbstractBackendStoreProvider {
BackendStore store = this.stores.get(name);
E.checkNotNull(store, "store");
E.checkState(store instanceof CassandraStore.CassandraGraphStore,
"GraphStore must be a instance of CassandraGraphStore");
"GraphStore must be an instance of CassandraGraphStore");
return store;
}

6
hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java
View File

@ -107,13 +107,13 @@ public abstract class CassandraTable {
});
}
public Iterable<BackendEntry> query(CassandraSessionPool.Session session,
public Iterator<BackendEntry> query(CassandraSessionPool.Session session,
Query query) {
List<BackendEntry> rs = new ArrayList<>();
if (query.limit() == 0 && query.limit() != Query.NO_LIMIT) {
LOG.debug("Return empty result(limit=0) for query {}", query);
return rs;
return rs.iterator();
}
List<Select> selections = query2Select(this.table, query);
@ -127,7 +127,7 @@ public abstract class CassandraTable {
}
LOG.debug("Return {} for query {}", rs, query);
return rs;
return rs.iterator();
}
protected List<Select> query2Select(String table, Query query) {

4
hugegraph-core/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph</artifactId>
<version>0.4.1-SNAPSHOT</version>
<version>0.4.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>hugegraph-core</artifactId>
@ -19,7 +19,7 @@
<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-common</artifactId>
<version>1.3.4-SNAPSHOT</version>
<version>1.3.6-SNAPSHOT</version>
</dependency>
<!-- TinkerPop -->
<dependency>

33
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java vendored
View File

@ -19,6 +19,8 @@
package com.baidu.hugegraph.backend.cache;
import java.util.Iterator;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.store.BackendEntry;
@ -29,6 +31,9 @@ import com.baidu.hugegraph.backend.store.BackendStoreProvider;
import com.baidu.hugegraph.config.HugeConfig;
import com.baidu.hugegraph.type.HugeType;
/**
* This class is unused now, just for debug or test
*/
public class CachedBackendStore implements BackendStore {
private BackendStore store = null;
@ -104,7 +109,7 @@ public class CachedBackendStore implements BackendStore {
@SuppressWarnings("unchecked")
@Override
public Iterable<BackendEntry> query(Query query) {
public Iterator<BackendEntry> query(Query query) {
if (query.empty()) {
return this.store.query(query);
}
@ -112,10 +117,10 @@ public class CachedBackendStore implements BackendStore {
QueryId id = new QueryId(query);
Object result = this.cache.get(id);
if (result != null) {
return (Iterable<BackendEntry>) result;
return (Iterator<BackendEntry>) result;
} else {
Iterable<BackendEntry> rs = this.store.query(query);
if (rs.iterator().hasNext()) {
Iterator<BackendEntry> rs = this.store.query(query);
if (rs.hasNext()) {
this.cache.update(id, rs);
}
return rs;
@ -150,8 +155,8 @@ public class CachedBackendStore implements BackendStore {
@Override
public int compareTo(Id o) {
// TODO Auto-generated method stub
return hashCode() - o.hashCode();
// TODO: improve
return this.hashCode() - o.hashCode();
}
@Override
@ -161,13 +166,13 @@ public class CachedBackendStore implements BackendStore {
@Override
public long asLong() {
// TODO Auto-generated method stub
// TODO: improve
return 0;
}
@Override
public byte[] asBytes() {
// TODO Auto-generated method stub
// TODO: improve
return null;
}
@ -175,5 +180,17 @@ public class CachedBackendStore implements BackendStore {
public String toString() {
return this.id.toString();
}
@Override
public int length() {
// TODO: improve
return 32;
}
@Override
public boolean number() {
// TODO: improve
return false;
}
}
}

4
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java vendored
View File

@ -159,10 +159,10 @@ public class CachedGraphTransaction extends GraphTransaction {
@Override
public void removeEdges(EdgeLabel edgeLabel) {
super.removeEdges(edgeLabel);
// TODO: Use a more precise strategy to update the edge cache
this.edgesCache.clear();
super.removeEdges(edgeLabel);
}
@Override

12
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/Id.java
View File

@ -19,13 +19,15 @@
package com.baidu.hugegraph.backend.id;
import com.baidu.hugegraph.type.HugeType;
public interface Id extends Comparable<Id> {
public abstract String asString();
public String asString();
public abstract long asLong();
public long asLong();
public abstract byte[] asBytes();
public byte[] asBytes();
public int length();
public boolean number();
}

29
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java
View File

@ -22,7 +22,6 @@ package com.baidu.hugegraph.backend.id;
import com.baidu.hugegraph.schema.SchemaElement;
import com.baidu.hugegraph.structure.HugeEdge;
import com.baidu.hugegraph.structure.HugeVertex;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.NumericUtil;
import com.baidu.hugegraph.util.StringEncoding;
@ -55,8 +54,8 @@ public abstract class IdGenerator {
return new LongId(id);
}
public static Id of(byte[] id) {
return new StringId(id);
public static Id of(byte[] bytes, boolean number) {
return number ? new LongId(bytes) : new StringId(bytes);
}
public static Id of(SchemaElement element) {
@ -120,7 +119,12 @@ public abstract class IdGenerator {
}
public StringId(byte[] bytes) {
this.id = StringEncoding.decodeString(bytes);
this.id = StringEncoding.decode(bytes);
}
@Override
public boolean number() {
return false;
}
@Override
@ -135,7 +139,12 @@ public abstract class IdGenerator {
@Override
public byte[] asBytes() {
return StringEncoding.encodeString(this.id);
return StringEncoding.encode(this.id);
}
@Override
public int length() {
return this.id.length();
}
@Override
@ -174,6 +183,11 @@ public abstract class IdGenerator {
this.id = NumericUtil.bytesToLong(bytes);
}
@Override
public boolean number() {
return true;
}
@Override
public String asString() {
return String.valueOf(this.id);
@ -189,6 +203,11 @@ public abstract class IdGenerator {
return NumericUtil.longToBytes(this.id);
}
@Override
public int length() {
return Long.BYTES;
}
@Override
public int compareTo(Id other) {
long otherId = ((LongId) other).id;

35
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/SnowflakeIdGenerator.java
View File

@ -39,7 +39,7 @@ public class SnowflakeIdGenerator extends IdGenerator {
synchronized (SnowflakeIdGenerator.class) {
if (instance == null) {
// TODO: workerId, datacenterId should read from conf
instance = new SnowflakeIdGenerator(1, 1);
instance = new SnowflakeIdGenerator(0, 0);
}
}
}
@ -52,7 +52,7 @@ public class SnowflakeIdGenerator extends IdGenerator {
public Id generate() {
if (this.idWorker == null) {
throw new HugeException("Please initialize before using it");
throw new HugeException("Please initialize before using");
}
return this.generate(this.idWorker.nextId());
}
@ -90,7 +90,7 @@ public class SnowflakeIdGenerator extends IdGenerator {
private long workerId;
private long datacenterId;
private long sequence = 0L;
private long sequence = 0L; // AtomicLong
private long lastTimestamp = -1L;
private static final long WORKER_BIT = 5L;
@ -120,33 +120,32 @@ public class SnowflakeIdGenerator extends IdGenerator {
}
this.workerId = workerId;
this.datacenterId = datacenterId;
LOG.debug("Worker starting. timestamp left shift {}," +
LOG.debug("Id Worker starting. timestamp left shift {}," +
"datacenter id bits {}, worker id bits {}," +
"sequence bits {}, workerid {}",
TIMESTAMP_SHIFT,
DC_BIT,
WORKER_BIT,
SEQUENCE_BIT,
workerId);
"sequence bits {}",
TIMESTAMP_SHIFT, DC_BIT, WORKER_BIT, SEQUENCE_BIT);
LOG.info("Id Worker starting. datacenter id {}, worker id {}",
datacenterId, workerId);
}
public synchronized long nextId() {
long timestamp = TimeUtil.timeGen();
if (timestamp < this.lastTimestamp) {
LOG.error("Clock is moving backwards, " +
"rejecting requests until {}.",
this.lastTimestamp);
throw new HugeException("Clock moved backwards. Refusing to " +
"generate id for %d milliseconds",
this.lastTimestamp - timestamp);
if (timestamp > this.lastTimestamp) {
this.sequence = 0L;
} else if (timestamp == this.lastTimestamp) {
this.sequence = (this.sequence + 1) & SEQUENCE_MASK;
if (this.sequence == 0) {
timestamp = TimeUtil.tillNextMillis(this.lastTimestamp);
}
} else {
this.sequence = 0L;
assert timestamp < this.lastTimestamp;
LOG.error("Clock is moving backwards, " +
"rejecting requests until {}.",
this.lastTimestamp);
throw new HugeException("Clock moved backwards. Refusing to " +
"generate id for %d milliseconds",
this.lastTimestamp - timestamp);
}
this.lastTimestamp = timestamp;

6
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Condition.java
View File

@ -395,7 +395,9 @@ public abstract class Condition {
// Single-type value or a list of single-type value
protected Object value;
// The key serialized(code/string) by backend store.
protected Object serialKey;
// The value serialized(code/string) by backend store.
protected Object serialValue;
@Override
@ -481,10 +483,6 @@ public abstract class Condition {
public static class SyspropRelation extends Relation {
/*
* Column name. TODO: the key should be serialized(code/string) by
* backend store private Object key.
*/
private HugeKeys key;
public SyspropRelation(HugeKeys key, Object value) {

4
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java
View File

@ -331,8 +331,8 @@ public class ConditionQuery extends IdQuery {
}
@Override
public ConditionQuery clone() {
ConditionQuery query = (ConditionQuery) super.clone();
public ConditionQuery copy() {
ConditionQuery query = (ConditionQuery) super.copy();
query.conditions = new LinkedHashSet<>(this.conditions);
return query;
}

3
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQueryFlatten.java
View File

@ -68,7 +68,6 @@ public class ConditionQueryFlatten {
return result;
}
@SuppressWarnings("unchecked")
private static Set<Set<Relation>> and(Set<Set<Relation>> left,
Set<Set<Relation>> right) {
Set<Set<Relation>> result = new HashSet<>();
@ -92,7 +91,7 @@ public class ConditionQueryFlatten {
private static ConditionQuery queryFromRelations(ConditionQuery query,
Set<Relation> relations) {
ConditionQuery q = query.clone();
ConditionQuery q = query.copy();
q.resetConditions();
for (Relation relation : relations) {
q.query(relation);

4
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/IdQuery.java
View File

@ -82,8 +82,8 @@ public class IdQuery extends Query {
}
@Override
public IdQuery clone() {
IdQuery query = (IdQuery) super.clone();
public IdQuery copy() {
IdQuery query = (IdQuery) super.copy();
query.ids = new LinkedHashSet<>(this.ids);
return query;
}

5
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java
View File

@ -140,10 +140,9 @@ public class Query implements Cloneable {
return true;
}
@Override
public Query clone() {
public Query copy() {
try {
return (Query) super.clone();
return (Query) this.clone();
} catch (CloneNotSupportedException e) {
throw new BackendException(e);
}

3
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/AbstractSerializer.java
View File

@ -21,6 +21,7 @@ package com.baidu.hugegraph.backend.serializer;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.type.HugeType;
public abstract class AbstractSerializer
implements GraphSerializer, SchemaSerializer {
@ -29,5 +30,5 @@ public abstract class AbstractSerializer
return entry;
}
public abstract BackendEntry newBackendEntry(Id id);
public abstract BackendEntry newBackendEntry(HugeType type, Id id);
}

109
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinaryBackendEntry.java
View File

@ -19,39 +19,47 @@
package com.baidu.hugegraph.backend.serializer;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.StringEncoding;
public class BinaryBackendEntry implements BackendEntry {
private Id id;
private static final byte[] EMPTY_BYTES = new byte[]{};
private final HugeType type;
private final BinaryId id;
private Id subId;
private Collection<BackendColumn> columns;
public BinaryBackendEntry(Id id) {
public BinaryBackendEntry(HugeType type, BinaryId id) {
this.type = type;
this.id = id;
this.columns = new ArrayList<>();
}
@Override
public Id id() {
public HugeType type() {
return this.type;
}
@Override
public BinaryId id() {
return this.id;
}
@Override
public void id(Id id) {
this.id = id;
}
@Override
public Id subId() {
return this.subId;
}
@Override
public void subId(Id subId) {
this.subId = subId;
}
@ -71,12 +79,17 @@ public class BinaryBackendEntry implements BackendEntry {
}
public void column(BackendColumn column) {
if (this.columns == null) {
this.columns = new ArrayList<>();
}
this.columns.add(column);
}
public void column(byte[] name, byte[] value) {
E.checkNotNull(name, "name");
BackendColumn col = new BackendColumn();
col.name = name;
col.value = value != null ? value : EMPTY_BYTES;
this.columns.add(col);
}
@Override
public Collection<BackendColumn> columns() {
return this.columns;
@ -84,7 +97,12 @@ public class BinaryBackendEntry implements BackendEntry {
@Override
public void columns(Collection<BackendColumn> bytesColumns) {
this.columns = bytesColumns;
this.columns.addAll(bytesColumns);
}
@Override
public void columns(BackendColumn... bytesColumns) {
this.columns.addAll(Arrays.asList(bytesColumns));
}
@Override
@ -99,6 +117,11 @@ public class BinaryBackendEntry implements BackendEntry {
}
}
@Override
public void clear() {
this.columns.clear();
}
@Override
public boolean equals(Object obj) {
if (!(obj instanceof BinaryBackendEntry)) {
@ -116,4 +139,64 @@ public class BinaryBackendEntry implements BackendEntry {
}
return true;
}
protected static class BinaryId implements Id {
private final byte[] bytes;
private final Id id;
public BinaryId(byte[] bytes, Id id) {
this.bytes = bytes;
this.id = id;
}
public Id origin() {
return this.id;
}
@Override
public String asString() {
throw new UnsupportedOperationException();
}
@Override
public long asLong() {
throw new UnsupportedOperationException();
}
@Override
public boolean number() {
throw new UnsupportedOperationException();
}
@Override
public int compareTo(Id other) {
throw new UnsupportedOperationException();
}
@Override
public byte[] asBytes() {
return this.bytes;
}
@Override
public int length() {
return this.bytes.length;
}
@Override
public int hashCode() {
return ByteBuffer.wrap(this.bytes).hashCode();
}
@Override
public boolean equals(Object other) {
return ByteBuffer.wrap(this.bytes).equals(other);
}
@Override
public String toString() {
return StringEncoding.decode(this.bytes);
}
}
}

618
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java
View File

@ -19,123 +19,268 @@
package com.baidu.hugegraph.backend.serializer;
import java.nio.ByteBuffer;
import java.util.Collection;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.commons.lang.NotImplementedException;
import org.apache.tinkerpop.gremlin.structure.Direction;
import com.baidu.hugegraph.HugeGraph;
import com.baidu.hugegraph.backend.BackendException;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.id.IdGenerator;
import com.baidu.hugegraph.backend.id.SplicingIdGenerator;
import com.baidu.hugegraph.backend.query.ConditionQuery;
import com.baidu.hugegraph.backend.query.IdQuery;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry.BinaryId;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.backend.store.BackendEntry.BackendColumn;
import com.baidu.hugegraph.exception.NotFoundException;
import com.baidu.hugegraph.schema.EdgeLabel;
import com.baidu.hugegraph.schema.IndexLabel;
import com.baidu.hugegraph.schema.PropertyKey;
import com.baidu.hugegraph.schema.SchemaElement;
import com.baidu.hugegraph.schema.VertexLabel;
import com.baidu.hugegraph.structure.HugeEdge;
import com.baidu.hugegraph.structure.HugeEdgeProperty;
import com.baidu.hugegraph.structure.HugeElement;
import com.baidu.hugegraph.structure.HugeIndex;
import com.baidu.hugegraph.structure.HugeProperty;
import com.baidu.hugegraph.structure.HugeVertex;
import com.baidu.hugegraph.structure.HugeVertexProperty;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.type.define.Cardinality;
import com.baidu.hugegraph.type.define.HugeKeys;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.JsonUtil;
import com.baidu.hugegraph.util.StringEncoding;
public class BinarySerializer extends AbstractSerializer {
@Override
public BackendEntry newBackendEntry(Id id) {
return new BinaryBackendEntry(id);
public BinaryBackendEntry newBackendEntry(HugeType type, Id id) {
BytesBuffer buffer = BytesBuffer.allocate(1 + id.length());
BinaryId bid = new BinaryId(buffer.writeId(id).bytes(), id);
return new BinaryBackendEntry(type, bid);
}
private BinaryBackendEntry newBackendEntry(HugeVertex vertex) {
return newBackendEntry(vertex.type(), vertex.id());
}
private BinaryBackendEntry newBackendEntry(HugeEdge edge) {
BinaryId id = new BinaryId(formatEdgeName(edge),
edge.idWithDirection());
return new BinaryBackendEntry(HugeType.EDGE, id);
}
@SuppressWarnings("unused")
private BinaryBackendEntry newBackendEntry(SchemaElement elem) {
Id id = IdGenerator.of(elem);
return newBackendEntry(elem.type(), id);
}
@Override
public BackendEntry writeId(HugeType type, Id id) {
return null;
protected BinaryBackendEntry convertEntry(BackendEntry entry) {
assert entry instanceof BinaryBackendEntry;
return (BinaryBackendEntry) entry;
}
@Override
public Query writeQuery(Query query) {
return null;
protected byte[] formatSyspropName(Id id, HugeKeys col) {
BytesBuffer buffer = BytesBuffer.allocate(1 + id.length() + 1 + 1);
byte sysprop = HugeType.SYS_PROPERTY.code();
return buffer.writeId(id).write(sysprop).write(col.code()).bytes();
}
protected byte[] formatSystemPropertyName(HugeKeys col) {
return new byte[] {HugeType.SYS_PROPERTY.code(), col.code()};
protected byte[] formatSyspropName(BinaryId id, HugeKeys col) {
BytesBuffer buffer = BytesBuffer.allocate(id.length() + 1 + 1);
byte sysprop = HugeType.SYS_PROPERTY.code();
return buffer.write(id.asBytes()).write(sysprop)
.write(col.code()).bytes();
}
private BackendColumn formatLabel(VertexLabel vertexLabel) {
protected BackendColumn formatLabel(HugeElement elem) {
BackendColumn col = new BackendColumn();
col.name = this.formatSystemPropertyName(HugeKeys.LABEL);
// TODO: save label name or id?
col.value = StringEncoding.encodeString(vertexLabel.name());
col.name = this.formatSyspropName(elem.id(), HugeKeys.LABEL);
// TODO: change to save label id
col.value = StringEncoding.encode(elem.label());
return col;
}
private VertexLabel parseLabel(BackendColumn col, HugeGraph graph) {
String label = StringEncoding.decodeString(col.value);
return graph.vertexLabel(label);
protected byte[] formatPropertyName(HugeProperty<?> prop) {
Id id = prop.element().id();
byte[] name = StringEncoding.encode(prop.key());
BytesBuffer buffer = BytesBuffer.allocate(1 + id.length() +
1 + name.length);
buffer.writeId(id);
buffer.write(prop.type().code());
buffer.write(name);
return buffer.bytes();
}
private byte[] formatPropertyName(HugeProperty<?> prop) {
// With encoded bytes
byte[] name = StringEncoding.encodeString(prop.key());
ByteBuffer buffer = ByteBuffer.allocate(3 + name.length);
buffer.put(prop.type().code());
// WriteString(name, buffer);
buffer.put(name);
return buffer.array();
protected byte[] formatPropertyValue(HugeProperty<?> prop) {
// TODO: serialize to bin instead of json
return StringEncoding.encode(JsonUtil.toJson(prop.value()));
}
private byte[] formatPropertyValue(HugeProperty<?> prop) {
// With encoded bytes
Object value = prop.value();
// TODO: serialize any object, not only string
return StringEncoding.encodeString(value.toString());
}
private BackendColumn formatProperty(HugeProperty<?> prop) {
protected BackendColumn formatProperty(HugeProperty<?> prop) {
BackendColumn col = new BackendColumn();
col.name = this.formatPropertyName(prop);
col.value = this.formatPropertyValue(prop);
return col;
}
private Object parsePropertyValue(byte[] bytes) {
// TODO: deserialize any object, not only string
return StringEncoding.decodeString(bytes);
protected void parseProperty(String name, byte[] val, HugeElement owner) {
PropertyKey pkey = owner.graph().propertyKey(name);
// Parse value
Object value = JsonUtil.fromJson(StringEncoding.decode(val),
pkey.clazz());
// Set properties of vertex/edge
if (pkey.cardinality() == Cardinality.SINGLE) {
owner.addProperty(pkey.name(), value);
} else {
if (!(value instanceof Collection)) {
throw new BackendException(
"Invalid value of non-single property: %s", value);
}
for (Object v : (Collection<?>) value) {
v = JsonUtil.castNumber(v, pkey.dataType().clazz());
owner.addProperty(pkey.name(), v);
}
}
}
private void parseColumn(BackendColumn col, HugeVertex vertex) {
ByteBuffer buffer = ByteBuffer.wrap(col.name);
byte type = buffer.get();
// Property
protected byte[] formatEdgeName(HugeEdge edge) {
// source-vertex + dir + edge-label + sort-values + target-vertex
BytesBuffer buffer = BytesBuffer.allocate(256);
buffer.writeId(edge.ownerVertex().id());
buffer.write(edge.type().code());
buffer.writeString(edge.label()); // TODO: change to id
buffer.writeString(edge.name()); // TODO: write if need
buffer.writeId(edge.otherVertex().id());
return buffer.bytes();
}
protected byte[] formatEdgeValue(HugeEdge edge) {
BytesBuffer buffer = BytesBuffer.allocate(6 * edge.getProperties().size());
// Write edge id
//buffer.writeId(edge.id());
// Write edge properties size
buffer.writeInt(edge.getProperties().size());
// Write edge properties data
for (HugeProperty<?> property : edge.getProperties().values()) {
buffer.writeString(property.key());
buffer.writeBytes(this.formatPropertyValue(property));
}
return buffer.bytes();
}
protected BackendColumn formatEdge(HugeEdge edge) {
BackendColumn col = new BackendColumn();
col.name = this.formatEdgeName(edge);
col.value = this.formatEdgeValue(edge);
return col;
}
protected void parseEdge(BackendColumn col, HugeVertex vertex,
HugeGraph graph) {
// source-vertex + dir + edge-label + sort-values + target-vertex
BytesBuffer buffer = BytesBuffer.wrap(col.name);
Id ownerVertexId = buffer.readId();
byte type = buffer.read();
String labelName = buffer.readString(); // TODO: change to id
String sk = buffer.readString();
Id otherVertexId = buffer.readId();
if (vertex == null) {
vertex = new HugeVertex(graph, ownerVertexId, null);
}
boolean isOutEdge = (type == HugeType.EDGE_OUT.code());
EdgeLabel edgeLabel = graph.edgeLabel(labelName);
VertexLabel srcLabel = graph.vertexLabel(edgeLabel.sourceLabel());
VertexLabel tgtLabel = graph.vertexLabel(edgeLabel.targetLabel());
HugeVertex otherVertex;
if (isOutEdge) {
vertex.vertexLabel(srcLabel);
otherVertex = new HugeVertex(graph, otherVertexId, tgtLabel);
} else {
vertex.vertexLabel(tgtLabel);
otherVertex = new HugeVertex(graph, otherVertexId, srcLabel);
}
HugeEdge edge = new HugeEdge(graph, null, edgeLabel);
if (isOutEdge) {
edge.sourceVertex(vertex);
edge.targetVertex(otherVertex);
vertex.addOutEdge(edge);
otherVertex.addInEdge(edge.switchOwner());
} else {
edge.sourceVertex(otherVertex);
edge.targetVertex(vertex);
vertex.addInEdge(edge);
otherVertex.addOutEdge(edge.switchOwner());
}
vertex.propNotLoaded();
otherVertex.propNotLoaded();
// Write edge-id + edge-properties
buffer = BytesBuffer.wrap(col.value);
//Id id = buffer.readId();
// Write edge properties
int size = buffer.readInt();
for (int i = 0; i < size; i++) {
this.parseProperty(buffer.readString(), buffer.readBytes(), edge);
}
edge.name(sk);
edge.assignId();
}
protected void parseColumn(BackendColumn col, HugeVertex vertex) {
BytesBuffer buffer = BytesBuffer.wrap(col.name);
buffer.readId();
byte type = buffer.read();
// Parse property
if (type == HugeType.PROPERTY.code()) {
String name = readStringFromRemaining(buffer);
Object value = parsePropertyValue(col.value);
vertex.addProperty(name, value);
} else if (type == HugeType.EDGE_IN.code() ||
type == HugeType.EDGE_OUT.code()) {
// TODO: parse edge
;
String name = buffer.readStringFromRemaining();
this.parseProperty(name, col.value, vertex);
}
// Parse edge
else if (type == HugeType.EDGE_IN.code() ||
type == HugeType.EDGE_OUT.code()) {
this.parseEdge(col, vertex, vertex.graph());
}
}
@Override
public BackendEntry writeVertex(HugeVertex vertex) {
BinaryBackendEntry entry = new BinaryBackendEntry(vertex.id());
BinaryBackendEntry entry = newBackendEntry(vertex);
// Write label column
entry.column(this.formatLabel(vertex.vertexLabel()));
// Add all properties of a Vertex
for (HugeProperty<?> prop : vertex.getProperties().values()) {
entry.column(this.formatProperty(prop));
if (vertex.removed()) {
return entry;
}
// Add all edges of a Vertex
for (@SuppressWarnings("unused") Edge edge : vertex.getEdges()) {
// TODO: format edge
// Write vertex label
entry.column(this.formatLabel(vertex));
// Write all properties of a Vertex
for (HugeProperty<?> prop : vertex.getProperties().values()) {
entry.column(this.formatProperty(prop));
}
return entry;
@ -143,20 +288,31 @@ public class BinarySerializer extends AbstractSerializer {
@Override
public BackendEntry writeVertexProperty(HugeVertexProperty<?> prop) {
return null;
BinaryBackendEntry entry = newBackendEntry(prop.element());
entry.column(this.formatProperty(prop));
entry.subId(IdGenerator.of(prop.key()));
return entry;
}
@Override
public HugeVertex readVertex(BackendEntry bytesEntry, HugeGraph graph) {
E.checkNotNull(graph, "serializer graph");
assert bytesEntry instanceof BinaryBackendEntry;
BinaryBackendEntry entry = (BinaryBackendEntry) bytesEntry;
if (bytesEntry == null) {
return null;
}
BinaryBackendEntry entry = this.convertEntry(bytesEntry);
// Parse label
byte[] labelCol = this.formatSystemPropertyName(HugeKeys.LABEL);
VertexLabel label = this.parseLabel(entry.column(labelCol), graph);
final byte[] VL = this.formatSyspropName(entry.id(), HugeKeys.LABEL);
BackendColumn vl = entry.column(VL);
VertexLabel label = null;
if (vl != null) {
// TODO: change to read vl-id
label = graph.vertexLabel(StringEncoding.decode(vl.value));
}
HugeVertex vertex = new HugeVertex(graph, entry.id(), label);
// Parse id
Id id = entry.id().origin();
HugeVertex vertex = new HugeVertex(graph, id, label);
// Parse all properties and edges of a Vertex
for (BackendColumn col : entry.columns()) {
@ -166,102 +322,294 @@ public class BinarySerializer extends AbstractSerializer {
return vertex;
}
@Override
public BackendEntry writeEdge(HugeEdge edge) {
BinaryBackendEntry entry = newBackendEntry(edge);
entry.column(this.formatEdge(edge));
return entry;
}
@Override
public BackendEntry writeEdgeProperty(HugeEdgeProperty<?> prop) {
// TODO: entry.column(this.formatProperty(prop));
throw new NotImplementedException("Unsupported writeEdgeProperty()");
}
@Override
public HugeEdge readEdge(BackendEntry entry, HugeGraph graph) {
throw new NotImplementedException("Unsupported readEdge()");
}
@Override
public BackendEntry writeIndex(HugeIndex index) {
BinaryBackendEntry entry;
if (index.fieldValues() == null && index.elementIds().size() == 0) {
/*
* When field-values is null and elementIds size is 0, it is
* meaningful for deletion of index data in secondary/search index.
* TODO: improve
*/
Id id = IdGenerator.of(index.indexLabelName());
entry = new BinaryBackendEntry(index.type(),
new BinaryId(id.asBytes(), id));
} else {
Id id = index.id();
byte[] indexId = id.asBytes();
E.checkArgument(indexId.length <= BytesBuffer.UINT8_MAX,
"Index key must be less than 256, but got: %s",
indexId.length);
Id elemId = index.elementId();
BytesBuffer buffer = BytesBuffer.allocate(indexId.length +
1 + elemId.length() + 1);
buffer.write(indexId);
buffer.writeId(elemId);
buffer.writeUInt8(indexId.length);
// Ensure the original look of the index key
entry = new BinaryBackendEntry(index.type(),
new BinaryId(indexId, id));
entry.column(buffer.bytes(), null);
entry.subId(elemId);
}
return entry;
}
@Override
public HugeIndex readIndex(BackendEntry bytesEntry, HugeGraph graph) {
if (bytesEntry == null) {
return null;
}
BinaryBackendEntry entry = this.convertEntry(bytesEntry);
// TODO: parse index id from entry.id() as bytes instead of string
HugeIndex index = HugeIndex.parseIndexId(graph, entry.type(),
entry.id().origin());
for (BackendColumn col : entry.columns()) {
if (col.name.length <= 0) {
// Ignore
continue;
}
int idLength = col.name[col.name.length - 1];
BytesBuffer buffer = BytesBuffer.wrap(col.name);
buffer.read(idLength);
index.elementIds(buffer.readId());
}
return index;
}
@Override
public BackendEntry writeId(HugeType type, Id id) {
return newBackendEntry(type, id);
}
@Override
public Query writeQuery(Query query) {
HugeType type = query.resultType();
// Serialize edge condition query (TODO: add VEQ(for EOUT/EIN))
if (type == HugeType.EDGE && !query.conditions().isEmpty()) {
HugeKeys[] keys = new HugeKeys[] {
HugeKeys.OWNER_VERTEX,
HugeKeys.DIRECTION,
HugeKeys.LABEL,
HugeKeys.SORT_VALUES,
HugeKeys.OTHER_VERTEX
};
int count = 0;
BytesBuffer buffer = BytesBuffer.allocate(256);
for (HugeKeys key : keys) {
Object value = ((ConditionQuery) query).condition(key);
if (value != null) {
count++;
} else {
if (key == HugeKeys.DIRECTION) {
value = Direction.OUT;
} else {
break;
}
}
if (key == HugeKeys.OWNER_VERTEX ||
key == HugeKeys.OTHER_VERTEX) {
Id id = HugeElement.getIdValue(value);
buffer.writeId(id);
} else if (key == HugeKeys.DIRECTION) {
byte t = value == Direction.OUT ?
HugeType.EDGE_OUT.code() :
HugeType.EDGE_IN.code();
buffer.write(t);
} else if (value instanceof String) {
buffer.writeString((String) value);
} else {
assert false : value.getClass();
}
}
if (count > 0) {
assert count == query.conditions().size();
IdQuery result = new IdQuery(type, query);
result.query(new BinaryId(buffer.bytes(), null));
return result;
}
}
// Serialize id in query
if (query instanceof IdQuery) {
IdQuery result = (IdQuery) query.copy();
result.resetIds();
for (Id id : query.ids()) {
if (type == HugeType.EDGE) {
// Serialize edge id query (TODO: add class EdgeId)
result.query(edgeId(id, Direction.OUT));
} else {
BytesBuffer buffer = BytesBuffer.allocate(1 + id.length());
result.query(new BinaryId(buffer.writeId(id).bytes(), id));
}
}
return result;
}
return query;
}
private static BinaryId edgeId(Id id, Direction dir) {
BytesBuffer buffer = BytesBuffer.allocate(256);
// TODO: improve Id split()
String[] idParts = SplicingIdGenerator.split(id);
// Ensure edge id with Direction
// NOTE: we assume the id without Direction if it contains 4 parts
if (idParts.length == 4) {
if (dir == Direction.IN) {
// Swap source-vertex and target-vertex
String tmp = idParts[0];
idParts[0] = idParts[3];
idParts[3] = tmp;
}
buffer.writeId(IdGenerator.of(idParts[0])); // long or string
buffer.write(dir == Direction.OUT ?
HugeType.EDGE_OUT.code() :
HugeType.EDGE_IN.code());
buffer.writeString(idParts[1]); // TODO: change to id
buffer.writeString(idParts[2]); // TODO: write if need
buffer.writeId(IdGenerator.of(idParts[3]));
} else if (idParts.length == 5) {
buffer.writeId(IdGenerator.of(idParts[0])); // long or string
buffer.write(idParts[1].equals("OUT") ?
HugeType.EDGE_OUT.code() :
HugeType.EDGE_IN.code());
buffer.writeString(idParts[2]); // TODO: change to id
buffer.writeString(idParts[3]); // TODO: write if need
buffer.writeId(IdGenerator.of(idParts[4]));
} else {
throw new NotFoundException("Unsupported ID format: %s", id);
}
return new BinaryId(buffer.bytes(), id);
}
public static BinaryId splitIdKey(HugeType type, byte[] bytes) {
// TODO: maybe we can find a better way to parse index id
if (type == HugeType.SECONDARY_INDEX || type == HugeType.SEARCH_INDEX) {
int idLength = bytes.length > 0 ? bytes[bytes.length - 1] : 0;
BytesBuffer buffer = BytesBuffer.wrap(bytes);
byte[] id = buffer.read(idLength);
return new BinaryId(id, IdGenerator.of(id, false));
}
return BytesBuffer.wrap(bytes).asId();
}
// TODO: remove these methods when improving schema serialize
private static String splitKeyId(byte[] bytes) {
BytesBuffer buffer = BytesBuffer.wrap(bytes);
buffer.readId();
return buffer.readStringFromRemaining();
}
private static byte[] joinIdKey(Id id, String key) {
int size = 1 + id.length() + key.length();
BytesBuffer buffer = BytesBuffer.allocate(size);
buffer.writeId(id);
buffer.writeStringToRemaining(key);
return buffer.bytes();
}
private BackendEntry text2bin(BackendEntry entry) {
BinaryBackendEntry bin = newBackendEntry(entry.type(), entry.id());
TextBackendEntry text = (TextBackendEntry) entry;
for (String name : text.columnNames()) {
String value = text.column(name);
bin.column(joinIdKey(entry.id(), name),
StringEncoding.encode(value));
}
return bin;
}
private BackendEntry bin2text(BackendEntry entry) {
if (entry == null) {
return null;
}
BinaryBackendEntry bin = (BinaryBackendEntry) entry;
TextBackendEntry text = new TextBackendEntry(null, bin.id().origin());
for (BackendColumn col : bin.columns()) {
String name = splitKeyId(col.name);
String value = StringEncoding.decode(col.value);
text.column(name, value);
}
return text;
}
// TODO: improve schema serialize
private final TextSerializer textSerializer = new TextSerializer();
@Override
public BackendEntry writeVertexLabel(VertexLabel vertexLabel) {
// TODO Auto-generated method stub
return null;
}
@Override
public BackendEntry writeEdgeLabel(EdgeLabel edgeLabel) {
// TODO Auto-generated method stub
return null;
}
@Override
public BackendEntry writePropertyKey(PropertyKey propertyKey) {
// TODO Auto-generated method stub
return null;
return text2bin(this.textSerializer.writeVertexLabel(vertexLabel));
}
@Override
public VertexLabel readVertexLabel(BackendEntry entry) {
// TODO Auto-generated method stub
return null;
return this.textSerializer.readVertexLabel(bin2text(entry));
}
@Override
public BackendEntry writeEdgeLabel(EdgeLabel edgeLabel) {
// TODO Auto-generated method stub
return text2bin(this.textSerializer.writeEdgeLabel(edgeLabel));
}
@Override
public EdgeLabel readEdgeLabel(BackendEntry entry) {
// TODO Auto-generated method stub
return null;
return this.textSerializer.readEdgeLabel(bin2text(entry));
}
@Override
public BackendEntry writePropertyKey(PropertyKey propertyKey) {
// TODO Auto-generated method stub
return text2bin(this.textSerializer.writePropertyKey(propertyKey));
}
@Override
public PropertyKey readPropertyKey(BackendEntry entry) {
// TODO Auto-generated method stub
return null;
return this.textSerializer.readPropertyKey(bin2text(entry));
}
@Override
public BackendEntry writeIndexLabel(IndexLabel indexLabel) {
return null;
// TODO Auto-generated method stub
return text2bin(this.textSerializer.writeIndexLabel(indexLabel));
}
@Override
public IndexLabel readIndexLabel(BackendEntry entry) {
return null;
}
protected static void writeString(byte[] bytes, ByteBuffer buffer) {
assert bytes.length < Short.MAX_VALUE;
buffer.putShort((short) bytes.length);
buffer.put(bytes);
}
protected static void writeString(String value, ByteBuffer buffer) {
byte[] bytes = StringEncoding.encodeString(value);
writeString(bytes, buffer);
}
protected static String readString(ByteBuffer buffer) {
short length = buffer.getShort();
byte[] bytes = new byte[length];
buffer.get(bytes);
return StringEncoding.decodeString(bytes);
}
protected static String readStringFromRemaining(ByteBuffer buffer) {
byte[] bytes = new byte[buffer.remaining()];
buffer.get(bytes);
return StringEncoding.decodeString(bytes);
}
@Override
public BackendEntry writeIndex(HugeIndex index) {
return null;
}
@Override
public HugeIndex readIndex(BackendEntry entry, HugeGraph graph) {
E.checkNotNull(graph, "serializer graph");
return null;
}
@Override
public BackendEntry writeEdge(HugeEdge edge) {
// TODO Auto-generated method stub
return null;
}
@Override
public BackendEntry writeEdgeProperty(HugeEdgeProperty<?> prop) {
return null;
}
@Override
public HugeEdge readEdge(BackendEntry entry, HugeGraph graph) {
E.checkNotNull(graph, "serializer graph");
// TODO Auto-generated method stub
return null;
return this.textSerializer.readIndexLabel(bin2text(entry));
}
}

337
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BytesBuffer.java Normal file
View File

@ -0,0 +1,337 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.backend.serializer;
import java.nio.ByteBuffer;
import java.util.Arrays;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.id.IdGenerator;
import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry.BinaryId;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.StringEncoding;
/**
* class BytesBuffer is a util for read/write binary
*/
public final class BytesBuffer {
public static final int BYTE_LEN = Byte.BYTES;
public static final int SHORT_LEN = Short.BYTES;
public static final int INT_LEN = Integer.BYTES;
public static final int LONG_LEN = Long.BYTES;
public static final int CHAR_LEN = Character.BYTES;
public static final int FLOAT_LEN = Float.BYTES;
public static final int DOUBLE_LEN = Double.BYTES;
public static final int UINT8_MAX = ((byte) -1) & 0xff;
public static final int UINT16_MAX = ((short) -1) & 0xffff;
public static final long UINT32_MAX = (-1) & 0xffffffffL;
public static final int DEFAULT_CAPACITY = 64;
public static final int MAX_BUFFER_CAPACITY = 128 * 1024 * 1024; // 128M
private ByteBuffer buffer;
public BytesBuffer() {
this(DEFAULT_CAPACITY);
}
public BytesBuffer(int capacity) {
E.checkArgument(capacity <= MAX_BUFFER_CAPACITY,
"Capacity exceeds max buffer capacity: %s",
MAX_BUFFER_CAPACITY);
this.buffer = ByteBuffer.allocate(capacity);
}
public BytesBuffer(ByteBuffer buffer) {
E.checkNotNull(buffer, "buffer");
this.buffer = buffer;
}
public static BytesBuffer allocate(int capacity) {
return new BytesBuffer(capacity);
}
public static BytesBuffer wrap(byte[] array) {
return new BytesBuffer(ByteBuffer.wrap(array));
}
public static BytesBuffer wrap(byte[] array, int offset, int length) {
return new BytesBuffer(ByteBuffer.wrap(array, offset, length));
}
public ByteBuffer asByteBuffer() {
return this.buffer;
}
public byte[] array() {
return this.buffer.array();
}
public byte[] bytes() {
byte[] bytes = this.buffer.array();
if (this.buffer.position() == bytes.length) {
return bytes;
} else {
return Arrays.copyOf(bytes, this.buffer.position());
}
}
private void require(int size) {
// Does need to resize?
if (this.buffer.capacity() - this.buffer.position() >= size) {
return;
}
// Extra capacity as buffer
int newcapacity = size + this.buffer.capacity() + DEFAULT_CAPACITY;
E.checkArgument(newcapacity <= MAX_BUFFER_CAPACITY,
"Capacity exceeds max buffer capacity: %s",
MAX_BUFFER_CAPACITY);
ByteBuffer newBuffer = ByteBuffer.allocate(newcapacity);
this.buffer.flip();
newBuffer.put(this.buffer);
this.buffer = newBuffer;
}
public BytesBuffer write(byte val) {
require(BYTE_LEN);
this.buffer.put(val);
return this;
}
public BytesBuffer write(byte[] val) {
require(BYTE_LEN * val.length);
this.buffer.put(val);
return this;
}
public BytesBuffer writeBoolean(boolean val) {
return this.write((byte) (val ? 1 : 0));
}
public BytesBuffer writeChar(char val) {
require(CHAR_LEN);
this.buffer.putChar(val);
return this;
}
public BytesBuffer writeShort(short val) {
require(SHORT_LEN);
this.buffer.putShort(val);
return this;
}
public BytesBuffer writeInt(int val) {
require(INT_LEN);
this.buffer.putInt(val);
return this;
}
public BytesBuffer writeLong(long val) {
require(LONG_LEN);
this.buffer.putLong(val);
return this;
}
public BytesBuffer writeFloat(float val) {
require(FLOAT_LEN);
this.buffer.putFloat(val);
return this;
}
public BytesBuffer writeDouble(double val) {
require(DOUBLE_LEN);
this.buffer.putDouble(val);
return this;
}
public BytesBuffer writeBytes(byte[] bytes) {
require(SHORT_LEN + bytes.length);
this.writeUInt16(bytes.length);
this.write(bytes);
return this;
}
public BytesBuffer writeString(String val) {
byte[] bytes = StringEncoding.encode(val);
this.writeBytes(bytes);
return this;
}
public byte read() {
return this.buffer.get();
}
public byte[] read(int length) {
byte[] bytes = new byte[length];
this.buffer.get(bytes);
return bytes;
}
public boolean readBoolean() {
return this.buffer.get() == 0 ? false : true;
}
public char readChar() {
return this.buffer.getChar();
}
public short readShort() {
return this.buffer.getShort();
}
public int readInt() {
return this.buffer.getInt();
}
public long readLong() {
return this.buffer.getLong();
}
public float readFloat() {
return this.buffer.getFloat();
}
public double readDouble() {
return this.buffer.getDouble();
}
public byte[] readBytes() {
int length = this.readUInt16();
byte[] bytes = this.read(length);
return bytes;
}
public String readString() {
return StringEncoding.decode(this.readBytes());
}
public BytesBuffer writeUInt8(int val) {
assert val <= UINT8_MAX;
this.write((byte) val);
return this;
}
public int readUInt8() {
return this.read() & 0x000000ff;
}
public BytesBuffer writeUInt16(int val) {
assert val <= UINT16_MAX;
this.writeShort((short) val);
return this;
}
public int readUInt16() {
return this.readShort() & 0x0000ffff;
}
public BytesBuffer writeUInt32(long val) {
assert val <= UINT32_MAX;
this.writeInt((int) val);
return this;
}
public long readUInt32() {
return this.readInt() & 0xffffffff;
}
public BytesBuffer writeStringToRemaining(String value) {
byte[] bytes = StringEncoding.encode(value);
this.write(bytes);
return this;
}
public String readStringFromRemaining() {
byte[] bytes = new byte[this.buffer.remaining()];
this.buffer.get(bytes);
return StringEncoding.decode(bytes);
}
public BytesBuffer writeId(Id id) {
boolean number = id.number();
if (number) {
long value = id.asLong();
this.writeNumber(value);
} else {
byte[] bytes = id.asBytes();
int len = bytes.length;
E.checkArgument(len < 128,
"Id max length is 127, but got {%s}", id);
len |= 0x80;
this.write((byte) len);
this.write(bytes);
}
return this;
}
public Id readId() {
int b = this.read();
int len = b & 0x7f;
boolean number = (b & 0x80) == 0;
if (number) {
return IdGenerator.of(this.readNumber(len));
} else {
byte[] id = this.read(len);
return IdGenerator.of(StringEncoding.decode(id));
}
}
public BinaryId asId() {
int start = this.buffer.position();
Id id = this.readId();
int end = this.buffer.position();
int len = end - start;
byte[] bytes = new byte[len];
System.arraycopy(this.array(), start, bytes, 0, len);
return new BinaryId(bytes, id);
}
private void writeNumber(long val) {
if (Byte.MIN_VALUE <= val && val <= Byte.MAX_VALUE) {
this.write((byte) 1);
this.write((byte) val);
} else if (Short.MIN_VALUE <= val && val <= Short.MAX_VALUE) {
this.write((byte) 2);
this.writeShort((short) val);
} else if (Integer.MIN_VALUE <= val && val <= Integer.MAX_VALUE) {
this.write((byte) 4);
this.writeInt((int) val);
} else {
this.write((byte) 8);
this.writeLong(val);
}
}
private long readNumber(int len) {
if (len <= 1) {
return this.read();
} else if (len <= 2) {
return this.readShort();
} else if (len <= 4) {
return this.readInt();
} else {
assert len == 8 : len;
return this.readLong();
}
}
}

82
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextBackendEntry.java
View File

@ -41,22 +41,21 @@ public class TextBackendEntry implements BackendEntry {
public static final String VALUE_SPLITOR = "\u0002";
public static final String IDS_SPLITOR = "\u0003";
private Id id;
private final HugeType type;
private final Id id;
private Id subId;
private HugeType type;
private Map<String, String> columns;
public TextBackendEntry(HugeType type, Id id) {
this.id = id;
this.type = type;
this.id = id;
this.subId = null;
this.columns = new ConcurrentHashMap<>();
}
public TextBackendEntry(Id id) {
this.id = id;
this.subId = null;
this.type = null;
this.columns = new ConcurrentHashMap<>();
@Override
public HugeType type() {
return this.type;
}
@Override
@ -64,25 +63,15 @@ public class TextBackendEntry implements BackendEntry {
return this.id;
}
@Override
public void id(Id id) {
this.id = id;
}
@Override
public Id subId() {
return this.subId;
}
@Override
public void subId(Id subId) {
this.subId = subId;
}
public HugeType type() {
return this.type;
}
public Set<String> columnNames() {
return this.columns.keySet();
}
@ -131,8 +120,8 @@ public class TextBackendEntry implements BackendEntry {
if (c.startsWith(column)) {
String v = this.columns.get(c);
BackendColumn bytesColumn = new BackendColumn();
bytesColumn.name = StringEncoding.encodeString(c);
bytesColumn.value = StringEncoding.encodeString(v);
bytesColumn.name = StringEncoding.encode(c);
bytesColumn.value = StringEncoding.encode(v);
list.add(bytesColumn);
}
}
@ -144,23 +133,23 @@ public class TextBackendEntry implements BackendEntry {
String newValue = col.getValue();
String oldValue = this.column(col.getKey());
// Not changed
if (newValue.equals(oldValue)) {
continue;
}
// TODO: use more general method
if (col.getKey().startsWith(HugeType.PROPERTY.name())) {
this.columns.put(col.getKey(), col.getValue());
continue;
}
// TODO: use more general method
if (!col.getKey().endsWith(HugeKeys.ELEMENT_IDS.string())) {
continue;
}
// TODO: ensure the old value is a list and json format (for index)
List<String> values = new ArrayList<>();
values.addAll(Arrays.asList(JsonUtil.fromJson(oldValue,
String[].class)));
values.addAll(Arrays.asList(JsonUtil.fromJson(newValue,
String[].class)));
List<Object> values = new ArrayList<>();
Object[] oldValues = JsonUtil.fromJson(oldValue, Object[].class);
Object[] newValues = JsonUtil.fromJson(newValue, Object[].class);
values.addAll(Arrays.asList(oldValues));
values.addAll(Arrays.asList(newValues));
// Update the old value
this.column(col.getKey(), JsonUtil.toJson(values));
}
@ -184,11 +173,11 @@ public class TextBackendEntry implements BackendEntry {
}
// TODO: ensure the old value is a list and json format (for index)
List<String> values = new ArrayList<>();
values.addAll(Arrays.asList(JsonUtil.fromJson(oldValue,
String[].class)));
values.removeAll(Arrays.asList(JsonUtil.fromJson(newValue,
String[].class)));
List<Object> values = new ArrayList<>();
Object[] oldValues = JsonUtil.fromJson(oldValue, Object[].class);
Object[] newValues = JsonUtil.fromJson(newValue, Object[].class);
values.addAll(Arrays.asList(oldValues));
values.removeAll(Arrays.asList(newValues));
// Update the old value
this.column(col.getKey(), JsonUtil.toJson(values));
}
@ -204,8 +193,8 @@ public class TextBackendEntry implements BackendEntry {
List<BackendColumn> list = new ArrayList<>(this.columns.size());
for (Entry<String, String> column : this.columns.entrySet()) {
BackendColumn bytesColumn = new BackendColumn();
bytesColumn.name = StringEncoding.encodeString(column.getKey());
bytesColumn.value = StringEncoding.encodeString(column.getValue());
bytesColumn.name = StringEncoding.encode(column.getKey());
bytesColumn.value = StringEncoding.encode(column.getValue());
list.add(bytesColumn);
}
return list;
@ -213,11 +202,17 @@ public class TextBackendEntry implements BackendEntry {
@Override
public void columns(Collection<BackendColumn> bytesColumns) {
this.columns.clear();
for (BackendColumn column : bytesColumns) {
this.columns.put(StringEncoding.decodeString(column.name),
StringEncoding.decodeString(column.value));
this.columns.put(StringEncoding.decode(column.name),
StringEncoding.decode(column.value));
}
}
@Override
public void columns(BackendColumn... bytesColumns) {
for (BackendColumn column : bytesColumns) {
this.columns.put(StringEncoding.decode(column.name),
StringEncoding.decode(column.value));
}
}
@ -227,6 +222,11 @@ public class TextBackendEntry implements BackendEntry {
this.columns.putAll(text.columns);
}
@Override
public void clear() {
this.columns.clear();
}
@Override
public boolean equals(Object obj) {
if (!(obj instanceof TextBackendEntry)) {

95
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TextSerializer.java
View File

@ -67,13 +67,22 @@ public class TextSerializer extends AbstractSerializer {
private static final String COLUME_SPLITOR = TextBackendEntry.COLUME_SPLITOR;
private static final String VALUE_SPLITOR = TextBackendEntry.VALUE_SPLITOR;
@Override
public TextBackendEntry newBackendEntry(HugeType type, Id id) {
return new TextBackendEntry(type, id);
}
@Override
public BackendEntry newBackendEntry(Id id) {
return new TextBackendEntry(null, id);
private TextBackendEntry newBackendEntry(HugeElement elem) {
return new TextBackendEntry(elem.type(), elem.id());
}
private TextBackendEntry newBackendEntry(HugeIndex index) {
return new TextBackendEntry(index.type(), index.id());
}
private TextBackendEntry newBackendEntry(SchemaElement elem) {
Id id = IdGenerator.of(elem);
return new TextBackendEntry(elem.type(), id);
}
@Override
@ -86,7 +95,8 @@ public class TextSerializer extends AbstractSerializer {
}
protected String formatSyspropName(String name) {
return String.format("%s%s%s", HugeType.SYS_PROPERTY.name(),
return String.format("%s%s%s",
HugeType.SYS_PROPERTY.name(),
COLUME_SPLITOR, name);
}
@ -95,12 +105,14 @@ public class TextSerializer extends AbstractSerializer {
}
protected Object formatPropertyName(Object key) {
return String.format("%s%s%s", HugeType.PROPERTY.name(),
return String.format("%s%s%s",
HugeType.PROPERTY.name(),
COLUME_SPLITOR, key);
}
protected String formatPropertyName(HugeProperty<?> prop) {
return String.format("%s%s%s", prop.type().name(),
return String.format("%s%s%s",
prop.type().name(),
COLUME_SPLITOR, prop.key());
}
@ -227,7 +239,7 @@ public class TextSerializer extends AbstractSerializer {
@Override
public BackendEntry writeVertex(HugeVertex vertex) {
TextBackendEntry entry = newBackendEntry(HugeType.VERTEX, vertex.id());
TextBackendEntry entry = newBackendEntry(vertex);
// Write label (NOTE: maybe just with edges if label is null)
if (vertex.vertexLabel() != null) {
@ -247,7 +259,7 @@ public class TextSerializer extends AbstractSerializer {
@Override
public BackendEntry writeVertexProperty(HugeVertexProperty<?> prop) {
HugeVertex vertex = prop.element();
TextBackendEntry entry = newBackendEntry(HugeType.VERTEX, vertex.id());
TextBackendEntry entry = newBackendEntry(vertex);
entry.subId(IdGenerator.of(prop.key()));
// Write label (NOTE: maybe just with edges if label is null)
@ -299,7 +311,8 @@ public class TextSerializer extends AbstractSerializer {
@Override
public BackendEntry writeEdgeProperty(HugeEdgeProperty<?> prop) {
HugeEdge edge = prop.element();
TextBackendEntry entry = newBackendEntry(edge.type(), edge.id());
TextBackendEntry entry = newBackendEntry(HugeType.EDGE,
edge.idWithDirection());
entry.subId(IdGenerator.of(prop.key()));
entry.column(this.formatEdgeName(edge), this.formatEdgeValue(edge));
return entry;
@ -309,12 +322,12 @@ public class TextSerializer extends AbstractSerializer {
public HugeEdge readEdge(BackendEntry backendEntry, HugeGraph graph) {
E.checkNotNull(graph, "serializer graph");
// TODO: implement
throw new NotImplementedException("Unsupport readEdge()");
throw new NotImplementedException("Unsupported readEdge()");
}
@Override
public BackendEntry writeIndex(HugeIndex index) {
TextBackendEntry entry = newBackendEntry(index.type(), index.id());
TextBackendEntry entry = newBackendEntry(index);
/*
* When field-values is null and elementIds size is 0, it is
* meaningful for deletion of index data in secondary/search index.
@ -374,8 +387,8 @@ public class TextSerializer extends AbstractSerializer {
@Override
public Query writeQuery(Query query) {
/*
* Serialize edge query by id/conditions to query by src-vertex +
* edge-name.
* Serialize edge query by id/conditions to query by
* src-vertex + edge-name.
*/
if (query.resultType() == HugeType.EDGE && query instanceof IdQuery) {
return this.writeEdgeQuery((IdQuery) query);
@ -400,7 +413,7 @@ public class TextSerializer extends AbstractSerializer {
}
protected IdQuery writeEdgeQuery(IdQuery query) {
IdQuery result = query.clone();
IdQuery result = query.copy();
result.resetIds();
if (!query.conditions().isEmpty() && !query.ids().isEmpty()) {
@ -472,9 +485,8 @@ public class TextSerializer extends AbstractSerializer {
@Override
public BackendEntry writeVertexLabel(VertexLabel vertexLabel) {
Id id = IdGenerator.of(vertexLabel);
TextBackendEntry entry = newBackendEntry(vertexLabel);
TextBackendEntry entry = this.writeId(vertexLabel.type(), id);
entry.column(HugeKeys.NAME, JsonUtil.toJson(vertexLabel.name()));
entry.column(HugeKeys.ID_STRATEGY,
JsonUtil.toJson(vertexLabel.idStrategy()));
@ -484,15 +496,14 @@ public class TextSerializer extends AbstractSerializer {
JsonUtil.toJson(vertexLabel.nullableKeys().toArray()));
entry.column(HugeKeys.INDEX_NAMES,
JsonUtil.toJson(vertexLabel.indexNames().toArray()));
writeProperties(vertexLabel, entry);
writeSchemaProperties(vertexLabel, entry);
return entry;
}
@Override
public BackendEntry writeEdgeLabel(EdgeLabel edgeLabel) {
Id id = IdGenerator.of(edgeLabel);
TextBackendEntry entry = newBackendEntry(edgeLabel);
TextBackendEntry entry = this.writeId(edgeLabel.type(), id);
entry.column(HugeKeys.NAME, JsonUtil.toJson(edgeLabel.name()));
entry.column(HugeKeys.SOURCE_LABEL,
JsonUtil.toJson(edgeLabel.sourceLabel()));
@ -506,26 +517,42 @@ public class TextSerializer extends AbstractSerializer {
JsonUtil.toJson(edgeLabel.nullableKeys().toArray()));
entry.column(HugeKeys.INDEX_NAMES,
JsonUtil.toJson(edgeLabel.indexNames().toArray()));
writeProperties(edgeLabel, entry);
writeSchemaProperties(edgeLabel, entry);
return entry;
}
@Override
public BackendEntry writePropertyKey(PropertyKey propertyKey) {
Id id = IdGenerator.of(propertyKey);
TextBackendEntry entry = newBackendEntry(propertyKey);
TextBackendEntry entry = this.writeId(propertyKey.type(), id);
entry.column(HugeKeys.NAME, JsonUtil.toJson(propertyKey.name()));
entry.column(HugeKeys.DATA_TYPE,
JsonUtil.toJson(propertyKey.dataType()));
entry.column(HugeKeys.CARDINALITY,
JsonUtil.toJson(propertyKey.cardinality()));
writeProperties(propertyKey, entry);
writeSchemaProperties(propertyKey, entry);
return entry;
}
public void writeProperties(SchemaElement schemaElement,
TextBackendEntry entry) {
@Override
public BackendEntry writeIndexLabel(IndexLabel indexLabel) {
TextBackendEntry entry = newBackendEntry(indexLabel);
entry.column(HugeKeys.NAME,
JsonUtil.toJson(indexLabel.name()));
entry.column(HugeKeys.BASE_TYPE,
JsonUtil.toJson(indexLabel.baseType()));
entry.column(HugeKeys.BASE_VALUE,
JsonUtil.toJson(indexLabel.baseValue()));
entry.column(HugeKeys.INDEX_TYPE,
JsonUtil.toJson(indexLabel.indexType()));
entry.column(HugeKeys.FIELDS,
JsonUtil.toJson(indexLabel.indexFields().toArray()));
return entry;
}
private static void writeSchemaProperties(SchemaElement schemaElement,
TextBackendEntry entry) {
Set<String> properties = schemaElement.properties();
if (properties == null) {
entry.column(HugeKeys.PROPERTIES, "[]");
@ -620,24 +647,6 @@ public class TextSerializer extends AbstractSerializer {
return propertyKey;
}
@Override
public BackendEntry writeIndexLabel(IndexLabel indexLabel) {
Id id = IdGenerator.of(indexLabel);
TextBackendEntry entry = this.writeId(indexLabel.type(), id);
entry.column(HugeKeys.NAME,
JsonUtil.toJson(indexLabel.name()));
entry.column(HugeKeys.BASE_TYPE,
JsonUtil.toJson(indexLabel.baseType()));
entry.column(HugeKeys.BASE_VALUE,
JsonUtil.toJson(indexLabel.baseValue()));
entry.column(HugeKeys.INDEX_TYPE,
JsonUtil.toJson(indexLabel.indexType()));
entry.column(HugeKeys.FIELDS,
JsonUtil.toJson(indexLabel.indexFields().toArray()));
return entry;
}
@Override
public IndexLabel readIndexLabel(BackendEntry backendEntry) {
if (backendEntry == null) {

4
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java
View File

@ -66,10 +66,6 @@ public abstract class AbstractBackendStoreProvider
@Override
public void close() throws BackendException {
this.checkOpened();
for (BackendStore store : this.stores.values()) {
// TODO: catch exceptions here
store.close();
}
this.storeEventHub.notify(Events.STORE_CLOSE, this);
}

18
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendEntry.java
View File

@ -22,6 +22,8 @@ package com.baidu.hugegraph.backend.store;
import java.util.Collection;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.util.Bytes;
import com.baidu.hugegraph.util.StringEncoding;
public interface BackendEntry {
@ -33,19 +35,27 @@ public interface BackendEntry {
@Override
public String toString() {
return String.format("%s=%s",
StringEncoding.decodeString(name),
StringEncoding.decodeString(value));
StringEncoding.decode(name),
StringEncoding.decode(value));
}
}
public HugeType type();
public Id id();
public void id(Id id);
public Id subId();
public void subId(Id subId);
public Collection<BackendColumn> columns();
public void columns(Collection<BackendColumn> columns);
public void columns(BackendColumn... columns);
public void merge(BackendEntry other);
public void clear();
public default boolean belongToMe(BackendColumn column) {
return Bytes.prefixWith(column.name, id().asBytes());
}
}

125
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendEntryIterator.java Normal file
View File

@ -0,0 +1,125 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.backend.store;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.function.Function;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.store.BackendEntry.BackendColumn;
import com.baidu.hugegraph.util.E;
public class BackendEntryIterator implements Iterator<BackendEntry> {
private final Iterator<BackendColumn> columns;
private final Query query;
private Function<BackendColumn, BackendEntry> entryCreater;
private BackendEntry current;
private BackendEntry next;
private long count;
public BackendEntryIterator(Iterator<BackendColumn> columns, Query query,
Function<BackendColumn, BackendEntry> entry) {
E.checkNotNull(columns, "columns");
E.checkNotNull(entry, "entry");
this.columns = columns;
this.entryCreater = entry;
this.query = query;
this.count = 0L;
this.current = null;
this.next = null;
}
@Override
public boolean hasNext() {
if (this.reachLimit()) {
return false;
}
if (this.current != null) {
return true;
}
assert this.current == null;
if (this.next != null) {
this.current = this.next;
this.next = null;
}
while (this.columns.hasNext()) {
BackendColumn col = this.columns.next();
if (this.current == null) {
// The first time to read
this.current = this.entryCreater.apply(col);
assert this.current != null;
this.current.columns(col);
} else if (this.current.belongToMe(col)) {
// Does the column belongs to the current entry
this.current.columns(col);
} else {
// New entry
assert this.next == null;
this.next = this.entryCreater.apply(col);
assert this.next != null;
this.next.columns(col);
return true;
}
}
return this.current != null;
}
@Override
public BackendEntry next() {
if (this.reachLimit()) {
throw new NoSuchElementException();
}
if (this.current == null) {
this.hasNext();
}
BackendEntry current = this.current;
if (current == null) {
throw new NoSuchElementException();
}
this.current = null;
this.count++;
return current;
}
private boolean reachLimit() {
if (this.query.limit() != Query.NO_LIMIT &&
this.count >= this.query.limit()) {
/*
* NOTE: if the query is separated with multi sub-queries(like query
* id in [id1, id2, ...]), then each BackendEntryIterator is only
* result(s) of one sub-query, so the query limit is inaccurate.
*/
return true;
}
return false;
}
}

22
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendFeatures.java
View File

@ -21,19 +21,29 @@ package com.baidu.hugegraph.backend.store;
public interface BackendFeatures {
public boolean supportsScan();
public boolean supportsScanToken();
public boolean supportsTransaction();
public boolean supportsScanKeyPrefix();
public boolean supportsQueryByContains();
public boolean supportsScanKeyRange();
public boolean supportsQueryByContainsKey();
public boolean supportsQuerySchemaByName();
public boolean supportsDeleteEdgeByLabel();
public boolean supportsQueryByLabel();
public boolean supportsQueryWithSearchCondition();
public boolean supportsQueryWithContains();
public boolean supportsQueryWithContainsKey();
public boolean supportsQueryWithOrderBy();
public boolean supportsDeleteEdgeByLabel();
public boolean supportsUpdateEdgeProperty();
public boolean supportsOrderByQuery();
public boolean supportsTransaction();
public boolean supportsNumberType();
}

151
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendSessionPool.java Normal file
View File

@ -0,0 +1,151 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.backend.store;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import com.baidu.hugegraph.backend.store.BackendStore.TxState;
import com.baidu.hugegraph.util.Log;
public abstract class BackendSessionPool {
private static final Logger LOG = Log.logger(BackendSessionPool.class);
private ThreadLocal<Session> threadLocalSession;
private AtomicInteger sessionCount;
public BackendSessionPool() {
this.threadLocalSession = new ThreadLocal<>();
this.sessionCount = new AtomicInteger(0);
}
public final Session getOrNewSession() {
Session session = this.threadLocalSession.get();
if (session == null) {
session = this.newSession();
assert session != null;
this.threadLocalSession.set(session);
this.sessionCount.incrementAndGet();
LOG.debug("Now(after connect({})) session count is: {}",
this, this.sessionCount.get());
}
return session;
}
public Session useSession() {
Session session = this.threadLocalSession.get();
if (session != null) {
session.attach();
} else {
session = this.getOrNewSession();
}
return session;
}
public int closeSession() {
Session session = this.threadLocalSession.get();
if (session == null) {
LOG.warn("Current session has ever been closed");
return -1;
}
int ref = session.detach();
assert ref >= 0 : ref;
if (ref == 0) {
try {
session.close();
} catch (Throwable e) {
session.attach();
throw e;
}
this.threadLocalSession.remove();
this.sessionCount.decrementAndGet();
}
return ref;
}
public void close() {
int ref = -1;
try {
ref = this.closeSession();
} finally {
if (this.sessionCount.get() == 0) {
this.doClose();
}
}
LOG.debug("Now(after close({})) session count is: {}, " +
"current session reference is: {}",
this, this.sessionCount.get(), ref);
}
@Override
public String toString() {
return String.format("%s@%08X",
this.getClass().getSimpleName(),
this.hashCode());
}
protected abstract Session newSession();
protected abstract void doClose();
/**
* interface Session for backend store
*/
public static abstract class Session {
private int refs;
private TxState txState;
public Session() {
this.refs = 1;
this.txState = TxState.CLEAN;
}
public abstract void close();
public abstract boolean closed();
public abstract void clear();
public abstract Object commit();
protected int attach() {
return ++this.refs;
}
protected int detach() {
return --this.refs;
}
public boolean closeable() {
return this.refs <= 0;
}
public TxState txState() {
return this.txState;
}
public void txState(TxState state) {
this.txState = state;
}
}
}

4
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStore.java
View File

@ -19,6 +19,8 @@
package com.baidu.hugegraph.backend.store;
import java.util.Iterator;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.config.HugeConfig;
import com.baidu.hugegraph.type.HugeType;
@ -43,7 +45,7 @@ public interface BackendStore {
public void mutate(BackendMutation mutation);
// Query data
public Iterable<BackendEntry> query(Query query);
public Iterator<BackendEntry> query(Query query);
// Transaction
public void beginTx();

76
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBStore.java
View File

@ -20,6 +20,7 @@
package com.baidu.hugegraph.backend.store.memory;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -79,17 +80,17 @@ public class InMemoryDBStore implements BackendStore {
assert type != null;
InMemoryDBTable table = this.tables.get(type);
if (table == null) {
throw new BackendException("Unsupported type: %s", type.name());
throw new BackendException("Unsupported table type: %s", type);
}
return table;
}
@Override
public Iterable<BackendEntry> query(final Query query) {
public Iterator<BackendEntry> query(final Query query) {
InMemoryDBTable table = this.table(query.resultType());
Iterable<BackendEntry> rs = table.query(query);
LOG.debug("[store {}] return {} for query: {}",
this.name, rs, query);
Iterator<BackendEntry> rs = table.query(query);
LOG.debug("[store {}] has result({}) for query: {}",
this.name, rs.hasNext(), query);
return rs;
}
@ -152,7 +153,7 @@ public class InMemoryDBStore implements BackendStore {
@Override
public void init() {
// TODO Auto-generated method stub
// pass
}
@Override
@ -231,43 +232,70 @@ public class InMemoryDBStore implements BackendStore {
private static final BackendFeatures FEATURES = new BackendFeatures() {
@Override
public boolean supportsDeleteEdgeByLabel() {
public boolean supportsScanToken() {
return false;
}
@Override
public boolean supportsScanKeyPrefix() {
return false;
}
@Override
public boolean supportsScanKeyRange() {
return false;
}
@Override
public boolean supportsQuerySchemaByName() {
// Traversal all data in memory
return true;
}
@Override
public boolean supportsQueryByLabel() {
// Traversal all data in memory
return true;
}
@Override
public boolean supportsQueryWithSearchCondition() {
return false;
}
@Override
public boolean supportsQueryWithOrderBy() {
return false;
}
@Override
public boolean supportsQueryWithContains() {
return true;
}
@Override
public boolean supportsQueryWithContainsKey() {
return true;
}
@Override
public boolean supportsDeleteEdgeByLabel() {
return false;
}
@Override
public boolean supportsUpdateEdgeProperty() {
return false;
}
@Override
public boolean supportsOrderByQuery() {
return false;
}
@Override
public boolean supportsScan() {
return false;
}
@Override
public boolean supportsTransaction() {
return false;
}
@Override
public boolean supportsQueryByContains() {
return true;
}
@Override
public boolean supportsQueryByContainsKey() {
return true;
public boolean supportsNumberType() {
return false;
}
};
}

5
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBTable.java
View File

@ -21,6 +21,7 @@ package com.baidu.hugegraph.backend.store.memory;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentSkipListMap;
@ -98,7 +99,7 @@ public class InMemoryDBTable {
}
}
public Iterable<BackendEntry> query(final Query query) {
public Iterator<BackendEntry> query(final Query query) {
Map<Id, BackendEntry> rs = this.store;
// Query by id(s)
@ -125,7 +126,7 @@ public class InMemoryDBTable {
}
}
return rs.values();
return rs.values().iterator();
}
protected Map<Id, BackendEntry> queryById(Set<Id> ids,

9
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/AbstractTransaction.java
View File

@ -86,7 +86,7 @@ public abstract class AbstractTransaction implements Transaction {
}
@Watched(prefix = "tx")
public Iterable<BackendEntry> query(Query query) {
public Iterator<BackendEntry> query(Query query) {
LOG.debug("Transaction query: {}", query);
/*
* NOTE: it's dangerous if an IdQuery/ConditionQuery is empty
@ -99,7 +99,7 @@ public abstract class AbstractTransaction implements Transaction {
query = this.serializer.writeQuery(query);
this.beforeRead();
Iterable<BackendEntry> result = this.store.query(query);
Iterator<BackendEntry> result = this.store.query(query);
this.afterRead();
return result;
@ -108,7 +108,7 @@ public abstract class AbstractTransaction implements Transaction {
@Watched(prefix = "tx")
public BackendEntry query(HugeType type, Id id) {
IdQuery q = new IdQuery(type, id);
Iterator<BackendEntry> results = this.query(q).iterator();
Iterator<BackendEntry> results = this.query(q);
if (results.hasNext()) {
BackendEntry entry = results.next();
assert !results.hasNext();
@ -204,6 +204,9 @@ public abstract class AbstractTransaction implements Transaction {
if (this.hasUpdates()) {
throw new BackendException("There are still changes to commit");
}
if (this.closed) {
return;
}
this.closed = true;
this.autoCommit = true; /* Let call after close() fail to commit */
this.store().close();

206
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/IndexTransaction.java → hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java
View File

@ -22,6 +22,7 @@ package com.baidu.hugegraph.backend.tx;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
@ -60,12 +61,12 @@ import com.baidu.hugegraph.util.LockUtil;
import com.baidu.hugegraph.util.NumericUtil;
import com.google.common.collect.ImmutableSet;
public class IndexTransaction extends AbstractTransaction {
public class GraphIndexTransaction extends AbstractTransaction {
private static final String INDEX_EMPTY_SYM = "\u0000";
private static final Query EMPTY_QUERY = new ConditionQuery(null);
public IndexTransaction(HugeGraph graph, BackendStore store) {
public GraphIndexTransaction(HugeGraph graph, BackendStore store) {
super(graph, store);
}
@ -101,7 +102,9 @@ public class IndexTransaction extends AbstractTransaction {
continue;
}
// Search and delete index equals element id
for (BackendEntry entry : super.query(q)) {
for (Iterator<BackendEntry> itor = super.query(q);
itor.hasNext();) {
BackendEntry entry = itor.next();
HugeIndex index = this.serializer.readIndex(entry, graph());
if (index.elementIds().contains(element.id())) {
index.resetElementIds();
@ -130,49 +133,40 @@ public class IndexTransaction extends AbstractTransaction {
this.removeSecondaryIndexLeft(element, elem, propKeys);
}
private Set<IndexLabel> relatedIndexLabels(HugeElement element) {
Set<IndexLabel> indexLabels = new HashSet<>();
Set<String> indexNames = element instanceof HugeVertex ?
((HugeVertex) element).vertexLabel().indexNames():
((HugeEdge) element).edgeLabel().indexNames();
for (String indexName : indexNames) {
SchemaTransaction schema = this.graph().schemaTransaction();
IndexLabel indexLabel = schema.getIndexLabel(indexName);
indexLabels.add(indexLabel);
}
return indexLabels;
}
private void removeSecondaryIndexLeft(HugeElement correctElem,
HugeElement incorrectElem,
Set<String> propKeys) {
for (IndexLabel indexLabel : this.relatedIndexLabels(incorrectElem)) {
if (CollectionUtils.containsAny(indexLabel.indexFields(),
propKeys)) {
this.updateIndex(indexLabel.name(), incorrectElem, true);
this.updateIndex(indexLabel.name(), correctElem, false);
for (IndexLabel il : relatedIndexLabels(incorrectElem)) {
if (CollectionUtils.containsAny(il.indexFields(), propKeys)) {
this.updateIndex(il.name(), incorrectElem, true);
this.updateIndex(il.name(), correctElem, false);
}
}
}
private Set<ConditionQuery> query2IndexQuery(ConditionQuery query,
HugeElement element) {
Set<ConditionQuery> indexQueries = new HashSet<>();
for (IndexLabel indexLabel : relatedIndexLabels(element)) {
ConditionQuery indexQuery = matchIndexLabel(query, indexLabel);
if (indexQuery != null) {
indexQueries.add(indexQuery);
}
@Watched(prefix = "index")
public void updateLabelIndex(HugeElement element, boolean removed) {
if (!this.needIndexForLabel()) {
return;
}
// Update label index if backend store not supports label-query
HugeIndex index = new HugeIndex(IndexLabel.label(element.type()));
index.fieldValues(element.label());
index.elementIds(element.id());
if (removed) {
this.eliminateEntry(this.serializer.writeIndex(index));
} else {
this.appendEntry(this.serializer.writeIndex(index));
}
return indexQueries;
}
@Watched(prefix = "index")
public void updateVertexIndex(HugeVertex vertex, boolean removed) {
// Update index(only property, no edge) of a vertex
for (String indexName : vertex.vertexLabel().indexNames()) {
updateIndex(indexName, vertex, removed);
this.updateIndex(indexName, vertex, removed);
}
}
@ -180,21 +174,14 @@ public class IndexTransaction extends AbstractTransaction {
public void updateEdgeIndex(HugeEdge edge, boolean removed) {
// Update index of an edge
for (String indexName : edge.edgeLabel().indexNames()) {
updateIndex(indexName, edge, removed);
this.updateIndex(indexName, edge, removed);
}
}
private static boolean hasNullableProp(HugeElement element, String key) {
Set<String> nullableKeys;
if (element instanceof HugeVertex) {
nullableKeys = ((HugeVertex) element).vertexLabel().nullableKeys();
} else {
assert element instanceof HugeEdge;
nullableKeys = ((HugeEdge) element).edgeLabel().nullableKeys();
}
return nullableKeys.contains(key);
}
/**
* Update index of (user properties in) vertex or edge
*/
@Watched(prefix = "index")
protected void updateIndex(String indexName,
HugeElement element,
boolean removed) {
@ -234,17 +221,17 @@ public class IndexTransaction extends AbstractTransaction {
assert indexLabel.indexType() == IndexType.SEARCH;
E.checkState(subPropValues.size() == 1,
"Expect searching by only one property");
propValue = NumericUtil.convert2Number(subPropValues.get(0));
propValue = NumericUtil.convertToNumber(subPropValues.get(0));
}
HugeIndex index = new HugeIndex(indexLabel);
index.fieldValues(propValue);
index.elementIds(element.id());
if (!removed) {
this.appendEntry(this.serializer.writeIndex(index));
} else {
if (removed) {
this.eliminateEntry(this.serializer.writeIndex(index));
} else {
this.appendEntry(this.serializer.writeIndex(index));
}
}
}
@ -257,15 +244,6 @@ public class IndexTransaction extends AbstractTransaction {
"there are changes in transaction");
}
SchemaTransaction schema = graph().schemaTransaction();
// Get user applied label or collect all qualified labels
List<IndexLabel> indexLabels = schema.getIndexLabels();
Set<String> labels = collectQueryLabels(query, indexLabels);
if (labels.isEmpty()) {
throw noIndexException(query, "<any label>");
}
// Can't query by index and by non-label sysprop at the same time
List<Condition> conds = query.syspropConditions();
if (conds.size() > 1 ||
@ -273,6 +251,52 @@ public class IndexTransaction extends AbstractTransaction {
throw new BackendException("Can't do index query with %s", conds);
}
// Query by index
Iterator<BackendEntry> entries;
if (query.allSysprop() && conds.size() == 1 &&
query.containsCondition(HugeKeys.LABEL)) {
// Query only by label
entries = this.queryByLabel(query);
} else {
// Query by userprops (or userprops + label)
entries = this.queryByUserprop(query);
}
if (!entries.hasNext()) {
return EMPTY_QUERY;
}
// Entry => Id
IdQuery ids = new IdQuery(query.resultType(), query);
while (entries.hasNext()) {
BackendEntry entry = entries.next();
HugeIndex index = this.serializer.readIndex(entry, graph());
ids.query(index.elementIds());
}
return ids;
}
@Watched(prefix = "index")
private Iterator<BackendEntry> queryByLabel(ConditionQuery query) {
IndexLabel il = IndexLabel.label(query.resultType());
String label = (String) query.condition(HugeKeys.LABEL);
assert label != null;
ConditionQuery indexQuery;
indexQuery = new ConditionQuery(HugeType.SECONDARY_INDEX, query);
indexQuery.eq(HugeKeys.INDEX_LABEL_NAME, il.name());
indexQuery.eq(HugeKeys.FIELD_VALUES, label);
return super.query(indexQuery);
}
@Watched(prefix = "index")
private Iterator<BackendEntry> queryByUserprop(ConditionQuery query) {
// Get user applied label or collect all qualified labels.
Set<String> labels = this.collectQueryLabels(query);
if (labels.isEmpty()) {
throw noIndexException(query, "<any label>");
}
// Do index query
ExtendableIterator<BackendEntry> entries = new ExtendableIterator<>();
for (String label : labels) {
@ -284,23 +308,20 @@ public class IndexTransaction extends AbstractTransaction {
ConditionQuery indexQuery = this.makeIndexQuery(query, label);
// Value type of Condition not matched
if (!this.validQueryConditionValues(query)) {
return EMPTY_QUERY;
assert !entries.hasNext();
break;
}
// Query index from backend store
entries.extend(super.query(indexQuery).iterator());
entries.extend(super.query(indexQuery));
} finally {
locks.unlock();
}
}
return entries;
}
// Entry => Id
IdQuery ids = new IdQuery(query.resultType(), query);
while (entries.hasNext()) {
BackendEntry entry = entries.next();
HugeIndex index = this.serializer.readIndex(entry, graph());
ids.query(index.elementIds());
}
return ids;
private boolean needIndexForLabel() {
return !this.store().features().supportsQueryByLabel();
}
private boolean validQueryConditionValues(ConditionQuery query) {
@ -324,15 +345,19 @@ public class IndexTransaction extends AbstractTransaction {
return true;
}
private Set<String> collectQueryLabels(ConditionQuery query,
List<IndexLabel> indexLabels) {
private Set<String> collectQueryLabels(ConditionQuery query) {
Set<String> labels = new HashSet<>();
String label = (String) query.condition(HugeKeys.LABEL);
if (label != null) {
labels.add(label);
} else {
// TODO: improve that get from cache
SchemaTransaction schema = graph().schemaTransaction();
List<IndexLabel> indexLabels = schema.getIndexLabels();
Set<String> queryKeys = query.userpropKeys();
assert queryKeys.size() > 0;
for (IndexLabel indexLabel : indexLabels) {
List<String> indexFields = indexLabel.indexFields();
if (query.resultType() == indexLabel.queryType() &&
@ -380,6 +405,18 @@ public class IndexTransaction extends AbstractTransaction {
return indexQuery;
}
private Set<ConditionQuery> query2IndexQuery(ConditionQuery query,
HugeElement element) {
Set<ConditionQuery> indexQueries = new HashSet<>();
for (IndexLabel indexLabel : relatedIndexLabels(element)) {
ConditionQuery indexQuery = matchIndexLabel(query, indexLabel);
if (indexQuery != null) {
indexQueries.add(indexQuery);
}
}
return indexQueries;
}
private static ConditionQuery matchIndexLabel(ConditionQuery query,
IndexLabel indexLabel) {
boolean requireSearch = query.hasSearchCondition();
@ -421,11 +458,11 @@ public class IndexTransaction extends AbstractTransaction {
// Replace the query key with PROPERTY_VALUES, and set number value
Condition condition = query.userpropConditions().get(0).copy();
for (Condition.Relation r : condition.relations()) {
Condition.Relation sys = new Condition.SyspropRelation(
Condition.Relation sr = new Condition.SyspropRelation(
HugeKeys.FIELD_VALUES,
r.relation(),
NumericUtil.convert2Number(r.value()));
condition = condition.replace(r, sys);
NumericUtil.convertToNumber(r.value()));
condition = condition.replace(r, sr);
}
indexQuery = new ConditionQuery(HugeType.SEARCH_INDEX, query);
@ -456,6 +493,31 @@ public class IndexTransaction extends AbstractTransaction {
query.userpropKeys(), label);
}
private static boolean hasNullableProp(HugeElement element, String key) {
Set<String> nullableKeys;
if (element instanceof HugeVertex) {
nullableKeys = ((HugeVertex) element).vertexLabel().nullableKeys();
} else {
assert element instanceof HugeEdge;
nullableKeys = ((HugeEdge) element).edgeLabel().nullableKeys();
}
return nullableKeys.contains(key);
}
private static Set<IndexLabel> relatedIndexLabels(HugeElement element) {
Set<IndexLabel> indexLabels = new HashSet<>();
Set<String> indexNames = element instanceof HugeVertex ?
((HugeVertex) element).vertexLabel().indexNames() :
((HugeEdge) element).edgeLabel().indexNames();
SchemaTransaction schema = element.graph().schemaTransaction();
for (String indexName : indexNames) {
IndexLabel indexLabel = schema.getIndexLabel(indexName);
indexLabels.add(indexLabel);
}
return indexLabels;
}
public void removeIndex(IndexLabel indexLabel) {
HugeIndex index = new HugeIndex(indexLabel);
this.removeEntry(this.serializer.writeIndex(index));

101
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java
View File

@ -20,7 +20,6 @@
package com.baidu.hugegraph.backend.tx;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
@ -80,7 +79,7 @@ import com.google.common.collect.ImmutableList;
public class GraphTransaction extends AbstractTransaction {
private final IndexTransaction indexTx;
private final GraphIndexTransaction indexTx;
private Map<Id, HugeVertex> addedVertexes;
private Map<Id, HugeVertex> removedVertexes;
@ -102,7 +101,7 @@ public class GraphTransaction extends AbstractTransaction {
public GraphTransaction(HugeGraph graph, BackendStore store) {
super(graph, store);
this.indexTx = new IndexTransaction(graph, store);
this.indexTx = new GraphIndexTransaction(graph, store);
assert !this.indexTx.autoCommit();
final HugeConfig conf = graph.configuration();
@ -190,8 +189,9 @@ public class GraphTransaction extends AbstractTransaction {
v.committed();
// Add vertex entry
this.addEntry(this.serializer.writeVertex(v));
// Update index of vertex(only include props)
// Update index of vertex(only include props, without edges)
this.indexTx.updateVertexIndex(v, false);
this.indexTx.updateLabelIndex(v, false);
}
// Do edge update
@ -203,6 +203,7 @@ public class GraphTransaction extends AbstractTransaction {
this.addEntry(this.serializer.writeEdge(e.switchOwner()));
// Update index of edge
this.indexTx.updateEdgeIndex(e, false);
this.indexTx.updateLabelIndex(e, false);
}
// Clear updates
@ -242,16 +243,18 @@ public class GraphTransaction extends AbstractTransaction {
*/
this.removeEntry(this.serializer.writeVertex(v.prepareRemoved()));
this.indexTx.updateVertexIndex(v, true);
this.indexTx.updateLabelIndex(v, true);
}
// Remove edges
for (HugeEdge edge : edges.values()) {
for (HugeEdge e : edges.values()) {
// Update edge index
this.indexTx.updateEdgeIndex(edge, true);
this.indexTx.updateEdgeIndex(e, true);
this.indexTx.updateLabelIndex(e, true);
// Remove edge of OUT and IN
edge = edge.prepareRemoved();
this.removeEntry(this.serializer.writeEdge(edge));
this.removeEntry(this.serializer.writeEdge(edge.switchOwner()));
e = e.prepareRemoved();
this.removeEntry(this.serializer.writeEdge(e));
this.removeEntry(this.serializer.writeEdge(e.switchOwner()));
}
}
@ -291,7 +294,7 @@ public class GraphTransaction extends AbstractTransaction {
}
@Override
public Iterable<BackendEntry> query(Query query) {
public Iterator<BackendEntry> query(Query query) {
if (query instanceof ConditionQuery) {
query = this.optimizeQuery((ConditionQuery) query);
/*
@ -301,7 +304,7 @@ public class GraphTransaction extends AbstractTransaction {
*/
if (query.empty()) {
// Return empty if there is no result after index-query
return ImmutableList.of();
return ImmutableList.<BackendEntry>of().iterator();
}
}
return super.query(query);
@ -433,11 +436,10 @@ public class GraphTransaction extends AbstractTransaction {
}
public Iterable<Vertex> queryVertices(Query query) {
assert Arrays.asList(HugeType.VERTEX, HugeType.EDGE)
.contains(query.resultType());
assert query.resultType() == HugeType.VERTEX;
Set<HugeVertex> results = this.newSetWithInsertionOrder();
Iterator<BackendEntry> entries = this.query(query).iterator();
Iterator<BackendEntry> entries = this.query(query);
while (entries.hasNext()) {
BackendEntry entry = entries.next();
HugeVertex vertex = this.serializer.readVertex(entry, graph());
@ -549,11 +551,13 @@ public class GraphTransaction extends AbstractTransaction {
public Iterable<Edge> queryEdges(Query query) {
assert query.resultType() == HugeType.EDGE;
Iterator<Vertex> vertices = this.queryVertices(query).iterator();
Map<Id, HugeEdge> results = this.newMapWithInsertionOrder();
while (vertices.hasNext()) {
HugeVertex vertex = (HugeVertex) vertices.next();
Iterator<BackendEntry> entries = this.query(query);
while (entries.hasNext()) {
BackendEntry entry = entries.next();
// Edges are in a vertex
HugeVertex vertex = this.serializer.readVertex(entry, graph());
for (HugeEdge edge : vertex.getEdges()) {
// Filter hidden results
if (!query.showHidden() &&
@ -615,6 +619,7 @@ public class GraphTransaction extends AbstractTransaction {
E.checkArgument(!primaryKeys.contains(prop.key()),
"Can't update primary key: '%s'", prop.key());
// Do property update
Set<String> lockNames = relatedIndexNames(prop.name(),
vertex.vertexLabel());
this.lockForUpdateProperty(lockNames, (locks) -> {
@ -658,6 +663,7 @@ public class GraphTransaction extends AbstractTransaction {
"Can't remove property '%s' for removing-state vertex",
prop.key());
// Do property update
Set<String> lockNames = relatedIndexNames(prop.name(),
vertex.vertexLabel());
this.lockForUpdateProperty(lockNames, (locks) -> {
@ -687,7 +693,8 @@ public class GraphTransaction extends AbstractTransaction {
return;
}
// Check is updating property of added/removed edge
E.checkArgument(!this.addedEdges.containsKey(edge.id()),
E.checkArgument(!this.addedEdges.containsKey(edge.id()) ||
this.updatedEdges.containsKey(edge.id()),
"Can't update property '%s' for adding-state edge",
prop.key());
E.checkArgument(!edge.removed() &&
@ -698,6 +705,7 @@ public class GraphTransaction extends AbstractTransaction {
E.checkArgument(!edge.edgeLabel().sortKeys().contains(prop.key()),
"Can't update sort key '%s'", prop.key());
// Do property update
Set<String> lockNames = relatedIndexNames(prop.name(),
edge.edgeLabel());
this.lockForUpdateProperty(lockNames, (locks) -> {
@ -708,10 +716,15 @@ public class GraphTransaction extends AbstractTransaction {
this.propertyUpdated(edge, edge.setProperty(prop));
this.indexTx.updateEdgeIndex(edge, false);
// Append new property(OUT and IN owner edge)
this.appendEntry(this.serializer.writeEdgeProperty(prop));
this.appendEntry(this.serializer.writeEdgeProperty(
prop.switchEdgeOwner()));
if (this.store().features().supportsUpdateEdgeProperty()) {
// Append new property(OUT and IN owner edge)
this.appendEntry(this.serializer.writeEdgeProperty(prop));
this.appendEntry(this.serializer.writeEdgeProperty(
prop.switchEdgeOwner()));
} else {
// Override edge(the edge will be in addedEdges & updatedEdges)
this.addEdge(edge);
}
});
}
@ -734,13 +747,15 @@ public class GraphTransaction extends AbstractTransaction {
return;
}
// Check is updating property of added/removed edge
E.checkArgument(!this.addedEdges.containsKey(edge.id()),
E.checkArgument(!this.addedEdges.containsKey(edge.id()) ||
this.updatedEdges.containsKey(edge.id()),
"Can't remove property '%s' for adding-state edge",
prop.key());
E.checkArgument(!this.removedEdges.containsKey(edge.id()),
"Can't remove property '%s' for removing-state edge",
prop.key());
// Do property update
Set<String> lockNames = relatedIndexNames(prop.name(),
edge.edgeLabel());
this.lockForUpdateProperty(lockNames, (locks) -> {
@ -751,10 +766,15 @@ public class GraphTransaction extends AbstractTransaction {
this.propertyUpdated(edge, edge.removeProperty(prop.key()));
this.indexTx.updateEdgeIndex(edge, false);
// Eliminate the property(OUT and IN owner edge)
this.eliminateEntry(this.serializer.writeEdgeProperty(prop));
this.eliminateEntry(this.serializer.writeEdgeProperty(
prop.switchEdgeOwner()));
if (this.store().features().supportsUpdateEdgeProperty()) {
// Eliminate the property(OUT and IN owner edge)
this.eliminateEntry(this.serializer.writeEdgeProperty(prop));
this.eliminateEntry(this.serializer.writeEdgeProperty(
prop.switchEdgeOwner()));
} else {
// Override edge(the edge will be in addedEdges & updatedEdges)
this.addEdge(edge);
}
});
}
@ -892,7 +912,10 @@ public class GraphTransaction extends AbstractTransaction {
if (query.resultType() == HugeType.EDGE) {
verifyEdgesConditionQuery(query);
}
return query;
if (this.store().features().supportsQueryByLabel() ||
!(label != null && query.conditions().size() == 1)) {
return query;
}
}
/*
@ -970,9 +993,6 @@ public class GraphTransaction extends AbstractTransaction {
private Iterable<?> joinTxVertices(Query query,
Collection<HugeVertex> vertices) {
if (query.resultType() != HugeType.VERTEX) {
return vertices;
}
assert query.resultType() == HugeType.VERTEX;
return this.joinTxRecords(query, vertices, (q, v) -> q.test(v),
this.addedVertexes, this.removedVertexes,
@ -1108,6 +1128,10 @@ public class GraphTransaction extends AbstractTransaction {
}
public void removeVertices(VertexLabel vertexLabel) {
if (this.hasUpdates()) {
throw new BackendException("There are still changes to commit");
}
boolean autoCommit = this.autoCommit();
this.autoCommit(false);
try {
@ -1117,9 +1141,9 @@ public class GraphTransaction extends AbstractTransaction {
if (vertexLabel.hidden()) {
query.showHidden(true);
}
Iterator<Vertex> vertices = this.queryVertices(query).iterator();
while (vertices.hasNext()) {
for (Iterator<Vertex> vertices = queryVertices(query).iterator();
vertices.hasNext();) {
this.removeVertex((HugeVertex) vertices.next());
}
this.commit();
@ -1132,6 +1156,10 @@ public class GraphTransaction extends AbstractTransaction {
}
public void removeEdges(EdgeLabel edgeLabel) {
if (this.hasUpdates()) {
throw new BackendException("There are still changes to commit");
}
boolean autoCommit = this.autoCommit();
this.autoCommit(false);
try {
@ -1146,9 +1174,8 @@ public class GraphTransaction extends AbstractTransaction {
if (edgeLabel.hidden()) {
query.showHidden(true);
}
Iterator<Edge> edges = this.queryEdges(query).iterator();
while (edges.hasNext()) {
for (Iterator<Edge> edges = queryEdges(query).iterator();
edges.hasNext();) {
this.removeEdge((HugeEdge) edges.next());
}
}

17
hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java
View File

@ -20,6 +20,7 @@
package com.baidu.hugegraph.backend.tx;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
@ -64,8 +65,8 @@ public class SchemaTransaction extends AbstractTransaction {
public List<PropertyKey> getPropertyKeys() {
List<PropertyKey> propertyKeys = new ArrayList<>();
Query q = new Query(HugeType.PROPERTY_KEY);
Iterable<BackendEntry> entries = this.query(q);
entries.forEach(entry -> {
Iterator<BackendEntry> entries = this.query(q);
entries.forEachRemaining(entry -> {
propertyKeys.add(this.serializer.readPropertyKey(entry));
});
return propertyKeys;
@ -74,8 +75,8 @@ public class SchemaTransaction extends AbstractTransaction {
public List<VertexLabel> getVertexLabels() {
List<VertexLabel> vertexLabels = new ArrayList<>();
Query q = new Query(HugeType.VERTEX_LABEL);
Iterable<BackendEntry> entries = this.query(q);
entries.forEach(entry -> {
Iterator<BackendEntry> entries = this.query(q);
entries.forEachRemaining(entry -> {
vertexLabels.add(this.serializer.readVertexLabel(entry));
});
return vertexLabels;
@ -84,8 +85,8 @@ public class SchemaTransaction extends AbstractTransaction {
public List<EdgeLabel> getEdgeLabels() {
List<EdgeLabel> edgeLabels = new ArrayList<>();
Query q = new Query(HugeType.EDGE_LABEL);
Iterable<BackendEntry> entries = this.query(q);
entries.forEach(entry -> {
Iterator<BackendEntry> entries = this.query(q);
entries.forEachRemaining(entry -> {
edgeLabels.add(this.serializer.readEdgeLabel(entry));
});
return edgeLabels;
@ -94,8 +95,8 @@ public class SchemaTransaction extends AbstractTransaction {
public List<IndexLabel> getIndexLabels() {
List<IndexLabel> indexLabels = new ArrayList<>();
Query q = new Query(HugeType.INDEX_LABEL);
Iterable<BackendEntry> entries = this.query(q);
entries.forEach(entry -> {
Iterator<BackendEntry> entries = this.query(q);
entries.forEachRemaining(entry -> {
indexLabels.add(this.serializer.readIndexLabel(entry));
});
return indexLabels;

15
hugegraph-core/src/main/java/com/baidu/hugegraph/io/HugeGraphIoRegistry.java
View File

@ -102,8 +102,10 @@ public class HugeGraphIoRegistry extends AbstractIoRegistry {
private static void writeEntry(Output output, BackendEntry entry) {
/* Write id */
output.writeInt(entry.id().asBytes().length);
output.writeBytes(entry.id().asBytes());
byte[] id = entry.id().asBytes();
output.writeBoolean(entry.id().number());
output.writeShort(id.length);
output.writeBytes(id);
/* Write columns size and data */
output.writeInt(entry.columns().size());
@ -117,21 +119,22 @@ public class HugeGraphIoRegistry extends AbstractIoRegistry {
private static BackendEntry readEntry(Input input) {
/* Read id */
int idLen = input.readInt();
Id id = IdGenerator.of(input.readBytes(idLen));
boolean number = input.readBoolean();
int idLen = input.readShortUnsigned();
Id id = IdGenerator.of(input.readBytes(idLen), number);
/* Read columns size and data */
Collection<BackendEntry.BackendColumn> columns = new ArrayList<>();
int columnSize = input.readInt();
for (int i = 0; i < columnSize; i++) {
BackendEntry.BackendColumn backendColumn =
new BackendEntry.BackendColumn();
new BackendEntry.BackendColumn();
backendColumn.name = input.readBytes(input.readInt());
backendColumn.value = input.readBytes(input.readInt());
columns.add(backendColumn);
}
BackendEntry backendEntry = new TextBackendEntry(id);
BackendEntry backendEntry = new TextBackendEntry(null, id);
backendEntry.columns(columns);
return backendEntry;
}

39
hugegraph-core/src/main/java/com/baidu/hugegraph/schema/IndexLabel.java
View File

@ -26,6 +26,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.baidu.hugegraph.HugeGraph;
import com.baidu.hugegraph.backend.query.ConditionQuery;
import com.baidu.hugegraph.backend.tx.SchemaTransaction;
import com.baidu.hugegraph.config.CoreOptions;
@ -153,6 +154,44 @@ public class IndexLabel extends SchemaElement {
return String.format(".by(%s)", sb.substring(0, endIdx));
}
static class PrimitiveIndexLabel extends IndexLabel {
public PrimitiveIndexLabel(String name) {
super(name);
// TODO: add indexFields and id(from -1)
}
@Override
public boolean primitive() {
return true;
}
}
public static final IndexLabel VL_IL = new PrimitiveIndexLabel("~vli");
public static final IndexLabel EL_IL = new PrimitiveIndexLabel("~eli");
public static IndexLabel label(HugeType type) {
if (type == HugeType.VERTEX) {
return VL_IL;
} else if (type == HugeType.EDGE || // TODO: just EDGE when separate e-p
type == HugeType.EDGE_OUT || type == HugeType.EDGE_IN) {
return EL_IL;
}
throw new AssertionError("No index label for " + type);
}
public static IndexLabel indexLabel(HugeGraph graph, String il) {
// Primitive IndexLabel first
if (VL_IL.name().equals(il)) {
return VL_IL;
}
if (EL_IL.name().equals(il)) {
return EL_IL;
}
return graph.indexLabel(il);
}
public static class Builder implements IndexLabelBuilder {
private IndexLabel indexLabel;

2
hugegraph-core/src/main/java/com/baidu/hugegraph/schema/PropertyKey.java
View File

@ -219,7 +219,7 @@ public class PropertyKey extends SchemaElement {
@Override
public Builder asTimestamp() {
this.propertyKey.dataType(DataType.TIMESTAMP);
this.propertyKey.dataType(DataType.DATE);
return this;
}

4
hugegraph-core/src/main/java/com/baidu/hugegraph/schema/SchemaElement.java
View File

@ -65,6 +65,10 @@ public abstract class SchemaElement
return Graph.Hidden.isHidden(this.name());
}
public boolean primitive() {
return false;
}
protected String propertiesSchema() {
StringBuilder sb = new StringBuilder();
for (String propertyName : this.properties) {

87
hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeIndex.java
View File

@ -19,17 +19,24 @@
package com.baidu.hugegraph.structure;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import com.baidu.hugegraph.HugeGraph;
import com.baidu.hugegraph.backend.BackendException;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.id.IdGenerator;
import com.baidu.hugegraph.backend.id.SplicingIdGenerator;
import com.baidu.hugegraph.schema.IndexLabel;
import com.baidu.hugegraph.schema.PropertyKey;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.type.define.IndexType;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.NumericUtil;
public class HugeIndex implements GraphType {
@ -60,14 +67,7 @@ public class HugeIndex implements GraphType {
}
public Id id() {
String propValues = fieldValues() == null ?
"<?>" : fieldValues().toString();
if (type() == HugeType.SECONDARY_INDEX) {
return SplicingIdGenerator.splicing(propValues, indexLabelName());
} else {
assert type() == HugeType.SEARCH_INDEX;
return SplicingIdGenerator.splicing(indexLabelName(), propValues);
}
return formatIndexId(type(), indexLabelName(), fieldValues());
}
public Object fieldValues() {
@ -122,4 +122,75 @@ public class HugeIndex implements GraphType {
this.label.name(), this.label.indexType().string(),
this.fieldValues, this.elementIds);
}
public static HugeIndex parseIndexId(HugeGraph graph,
HugeType type, Id id) {
String label;
Object values;
IndexLabel indexLabel;
if (type == HugeType.SECONDARY_INDEX) {
String[] parts = SplicingIdGenerator.parse(id);
E.checkState(parts.length == 2, "Invalid SECONDARY_INDEX id");
values = parts[0];
label = parts[1];
indexLabel = IndexLabel.indexLabel(graph, label);
} else {
assert type == HugeType.SEARCH_INDEX;
// TODO: parse from bytes id
String str = id.asString();
E.checkState(str.length() > 8, "Invalid SEARCH_INDEX id");
int offset = str.length() - 8;
label = str.substring(0, offset);
indexLabel = IndexLabel.indexLabel(graph, label);
List<String> fields = indexLabel.indexFields();
E.checkState(fields.size() == 1, "Invalid SEARCH_INDEX fields");
PropertyKey pk = graph.propertyKey(fields.get(0));
E.checkState(pk.dataType().isNumberType(),
"Invalid SEARCH_INDEX field type");
values = string2number(str.substring(offset),
pk.dataType().clazz());
}
HugeIndex index = new HugeIndex(indexLabel);
index.fieldValues(values);
return index;
}
public static Id formatIndexId(HugeType type, String indexName,
Object fieldValues) {
if (type == HugeType.SECONDARY_INDEX) {
String value = fieldValues == null ? "<?>" : fieldValues.toString();
return SplicingIdGenerator.splicing(value, indexName);
} else {
assert type == HugeType.SEARCH_INDEX;
String value = "";
if (fieldValues != null) {
E.checkState(fieldValues instanceof Number,
"Field value search index must be number type: %s",
fieldValues.getClass().getSimpleName());
value = number2string((Number) fieldValues);
}
// TODO: use bytes id
return IdGenerator.of(indexName + value);
}
}
public static String number2string(Number number) {
byte[] bytes = NumericUtil.numberToSortableBytes(number);
try {
// TODO: use bytes id
return new String(bytes, "ISO-8859-1");
} catch (UnsupportedEncodingException e) {
throw new BackendException(e);
}
}
public static Number string2number(String value, Class<?> clazz) {
try {
byte[] bytes = value.getBytes("ISO-8859-1");
return NumericUtil.sortableBytesToNumber(bytes, clazz);
} catch (UnsupportedEncodingException e) {
throw new BackendException(e);
}
}
}

1
hugegraph-core/src/main/java/com/baidu/hugegraph/structure/HugeVertex.java
View File

@ -69,6 +69,7 @@ public class HugeVertex extends HugeElement implements Vertex, Cloneable {
this(tx.graph(), id, label);
this.tx = tx;
this.fresh = true;
E.checkNotNull(label, "label");
}
public HugeVertex(final HugeGraph graph, Id id, VertexLabel label) {

9
hugegraph-core/src/main/java/com/baidu/hugegraph/type/HugeType.java
View File

@ -35,14 +35,17 @@ public enum HugeType {
SYS_PROPERTY(102),
// Property
PROPERTY(103),
// Edge
EDGE(120),
// Edge's direction is OUT for the specified vertex
EDGE_OUT(120),
EDGE_OUT(130),
// Edge's direction is IN for the specified vertex
EDGE_IN(121),
EDGE_IN(140),
SECONDARY_INDEX(150),
SEARCH_INDEX(151),
SEARCH_INDEX(160),
COUNTERS(250),
MAX_TYPE(255);

9
hugegraph-core/src/main/java/com/baidu/hugegraph/type/define/DataType.java
View File

@ -19,20 +19,17 @@
package com.baidu.hugegraph.type.define;
import java.sql.Timestamp;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import org.apache.commons.lang.StringUtils;
import com.baidu.hugegraph.structure.HugeProperty;
public enum DataType {
// This property has sub properties
OBJECT(1, "object", HugeProperty.class),
OBJECT(1, "object", Object.class),
BOOLEAN(2, "boolean", Boolean.class),
BYTE(3, "byte", Byte.class),
BLOB(4, "blob", byte[].class),
@ -41,7 +38,7 @@ public enum DataType {
INT(7, "int", Integer.class),
LONG(8, "long", Long.class),
TEXT(9, "text", String.class),
TIMESTAMP(10, "timestamp", Timestamp.class),
DATE(10, "date", Date.class),
UUID(11, "uuid", UUID.class);
private byte code = 0;

4
hugegraph-core/src/main/java/com/baidu/hugegraph/util/JsonUtil.java
View File

@ -45,8 +45,8 @@ public class JsonUtil {
* this method used to cast element in collection to original number type
*/
public static Object castNumber(Object object, Class<?> clazz) {
if (object instanceof Double) {
Double number = (Double) object;
if (object instanceof Number) {
Number number = (Number) object;
if (clazz == Byte.class) {
object = number.byteValue();
} else if (clazz == Integer.class) {

4
hugegraph-core/src/main/java/com/baidu/hugegraph/util/StringEncoding.java
View File

@ -63,7 +63,7 @@ public class StringEncoding {
return attribute.isEmpty() ? 1 : attribute.length();
}
public static byte[] encodeString(String value) {
public static byte[] encode(String value) {
try {
return value.getBytes("UTF-8");
} catch (UnsupportedEncodingException e) {
@ -71,7 +71,7 @@ public class StringEncoding {
}
}
public static String decodeString(byte[] bytes) {
public static String decode(byte[] bytes) {
try {
return new String(bytes, "UTF-8");
} catch (UnsupportedEncodingException e) {

17
hugegraph-dist/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>hugegraph</artifactId>
<groupId>com.baidu.hugegraph</groupId>
<version>0.4.1-SNAPSHOT</version>
<version>0.4.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<name>hugegraph-dist: Tar and Distribute Archives</name>
@ -32,6 +32,16 @@
<serializer>scylladb</serializer>
</properties>
</profile>
<profile>
<id>rocksdb</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<backend>rocksdb</backend>
<serializer>binary</serializer>
</properties>
</profile>
</profiles>
<properties>
@ -62,6 +72,11 @@
<artifactId>hugegraph-scylladb</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-rocksdb</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>airline</artifactId>

11
hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java vendored
View File

@ -27,6 +27,7 @@ import org.apache.commons.configuration.ConfigurationException;
import com.baidu.hugegraph.HugeException;
import com.baidu.hugegraph.backend.serializer.SerializerFactory;
import com.baidu.hugegraph.backend.store.BackendProviderFactory;
import com.baidu.hugegraph.backend.store.rocksdb.RocksDBOptions;
import com.baidu.hugegraph.config.CassandraOptions;
import com.baidu.hugegraph.config.CoreOptions;
import com.baidu.hugegraph.config.HugeConfig;
@ -64,6 +65,9 @@ public class RegisterUtil {
case "hbase":
registerHBase();
break;
case "rocksdb":
registerRocksDB();
break;
default:
throw new HugeException("Unsupported backend type '%s'", backend);
}
@ -92,7 +96,14 @@ public class RegisterUtil {
}
public static void registerHBase() {
}
public static void registerRocksDB() {
// Register config
OptionSpace.register(RocksDBOptions.Instance());
// Register backend
BackendProviderFactory.register("rocksdb",
"com.baidu.hugegraph.backend.store.rocksdb.RocksDBStoreProvider");
}
public static void registerServer() {

2
hugegraph-dist/src/main/resources/backend.properties
View File

@ -1 +1 @@
backends=[cassandra, scylladb]
backends=[cassandra, scylladb, rocksdb]

2
hugegraph-example/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>hugegraph</artifactId>
<groupId>com.baidu.hugegraph</groupId>
<version>0.4.1-SNAPSHOT</version>
<version>0.4.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

8
hugegraph-example/src/main/java/com/baidu/hugegraph/example/Example1.java
View File

@ -257,6 +257,12 @@ public class Example1 {
assert size == 12;
System.out.println(">>>> query all vertices: size=" + size);
// query by label
vertexes = graph.traversal().V().hasLabel("person");
size = vertexes.toList().size();
assert size == 5;
System.out.println(">>>> query all persons: size=" + size);
// query vertex by primary-values
vertexes = graph.traversal().V().hasLabel("author").has("id", "1");
List<Vertex> vertexList = vertexes.toList();
@ -310,7 +316,7 @@ public class Example1 {
assert edgeList.size() == 1;
System.out.println(">>>> query edge by id: " + edgeList);
Edge edge = graph.traversal().E(id).toList().get(0);
Edge edge = edgeList.get(0);
edges = graph.traversal().E(edge.id());
edgeList = edges.toList();
assert edgeList.size() == 1;

2
hugegraph-example/src/main/java/com/baidu/hugegraph/example/ExampleUtil.java
View File

@ -36,6 +36,8 @@ public class ExampleUtil {
RegisterUtil.registerCassandra();
RegisterUtil.registerScyllaDB();
RegisterUtil.registerHBase();
RegisterUtil.registerRocksDB();
}
public static HugeGraph loadGraph() {

67
hugegraph-example/src/main/java/com/baidu/hugegraph/example/GraphOfTheMoviesExample.java
View File

@ -26,11 +26,11 @@ import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.slf4j.Logger;
import com.baidu.hugegraph.util.Log;
import com.baidu.hugegraph.HugeGraph;
import com.baidu.hugegraph.perf.PerfUtil;
import com.baidu.hugegraph.schema.SchemaManager;
import com.baidu.hugegraph.util.Log;
public class GraphOfTheMoviesExample {
@ -63,8 +63,11 @@ public class GraphOfTheMoviesExample {
GraphTraversal<Edge, Edge> edges = graph.traversal().E();
System.out.println(">>>> query all edges: size=" + edges.toList().size());
List<Edge> tomhanksMovies =
graph.traversal().V().hasLabel("person").has("name", "Tom Hanks").outE("ACTED_IN").toList();
// query edges by condition
List<Edge> tomhanksMovies = graph.traversal().V()
.hasLabel("person")
.has("name", "Tom Hanks")
.outE("ACTED_IN").toList();
System.out.println(">>>> Tom Hanks ACTED_IN: " + tomhanksMovies);
}
@ -129,8 +132,7 @@ public class GraphOfTheMoviesExample {
Vertex emil = graph.addVertex(T.label, "person", "name", "emil Eifrem", "born", 1978);
emil.addEdge("ACTED_IN", theMatrix, "roles", "emil");
Vertex theMatrixReloaded =
graph.addVertex(T.label, "movie", "title", "The Matrix Reloaded", "released", 2003);
Vertex theMatrixReloaded = graph.addVertex(T.label, "movie", "title", "The Matrix Reloaded", "released", 2003);
keanu.addEdge("ACTED_IN", theMatrixReloaded, "roles", "Neo");
carrie.addEdge("ACTED_IN", theMatrixReloaded, "roles", "Trinity");
@ -140,8 +142,7 @@ public class GraphOfTheMoviesExample {
lanaW.addEdge("DIRECTED", theMatrix, "score", 10);
joelS.addEdge("PRODUCED", theMatrixReloaded, "score", 10);
Vertex theMatrixRevolutions =
graph.addVertex(T.label, "movie", "title", "The Matrix Revolutions", "released", 2003);
Vertex theMatrixRevolutions = graph.addVertex(T.label, "movie", "title", "The Matrix Revolutions", "released", 2003);
keanu.addEdge("ACTED_IN", theMatrixRevolutions, "roles", "Neo");
carrie.addEdge("ACTED_IN", theMatrixRevolutions, "roles", "Trinity");
@ -151,8 +152,7 @@ public class GraphOfTheMoviesExample {
lanaW.addEdge("DIRECTED", theMatrixRevolutions, "score", 10);
joelS.addEdge("PRODUCED", theMatrixRevolutions, "score", 10);
Vertex theDevilsadvocate =
graph.addVertex(T.label, "movie", "title", "The Devil's advocate", "released", 1997);
Vertex theDevilsadvocate = graph.addVertex(T.label, "movie", "title", "The Devil's advocate", "released", 1997);
Vertex charlize = graph.addVertex(T.label, "person", "name", "charlize Theron", "born", 1975);
Vertex al = graph.addVertex(T.label, "person", "name", "al Pacino", "born", 1940);
@ -254,8 +254,7 @@ public class GraphOfTheMoviesExample {
marshallB.addEdge("ACTED_IN", standByMe, "roles", "Mr. Lachance");
robR.addEdge("DIRECTED", standByMe, "score", 10);
Vertex asGoodasItGets =
graph.addVertex(T.label, "movie", "title", "as Good as It Gets", "released", 1997);
Vertex asGoodasItGets = graph.addVertex(T.label, "movie", "title", "as Good as It Gets", "released", 1997);
Vertex helenH = graph.addVertex(T.label, "person", "name", "Helen Hunt", "born", 1963);
Vertex gregK = graph.addVertex(T.label, "person", "name", "Greg Kinnear", "born", 1963);
@ -267,8 +266,7 @@ public class GraphOfTheMoviesExample {
cubaG.addEdge("ACTED_IN", asGoodasItGets, "roles", "Frank Sachs");
jamesB.addEdge("DIRECTED", asGoodasItGets, "score", 10);
Vertex whatDreamsMayCome =
graph.addVertex(T.label, "movie", "title", "What Dreams May Come", "released", 1998);
Vertex whatDreamsMayCome = graph.addVertex(T.label, "movie", "title", "What Dreams May Come", "released", 1998);
Vertex annabellaS = graph.addVertex(T.label, "person", "name", "annabella Sciorra", "born", 1960);
Vertex maxS = graph.addVertex(T.label, "person", "name", "Max von Sydow", "born", 1929);
@ -283,8 +281,7 @@ public class GraphOfTheMoviesExample {
wernerH.addEdge("ACTED_IN", whatDreamsMayCome, "roles", "The Face");
vincentW.addEdge("DIRECTED", whatDreamsMayCome, "score", 10);
Vertex snowFallingonCedars =
graph.addVertex(T.label, "movie", "title", "Snow Falling on Cedars", "released", 1999);
Vertex snowFallingonCedars = graph.addVertex(T.label, "movie", "title", "Snow Falling on Cedars", "released", 1999);
Vertex ethanH = graph.addVertex(T.label, "person", "name", "Ethan Hawke", "born", 1970);
Vertex rickY = graph.addVertex(T.label, "person", "name", "Rick Yune", "born", 1971);
@ -313,8 +310,7 @@ public class GraphOfTheMoviesExample {
steveZ.addEdge("ACTED_IN", youveGotMail, "roles", "George Pappas");
noraE.addEdge("DIRECTED", youveGotMail, "score", 10);
Vertex sleeplessInSeattle =
graph.addVertex(T.label, "movie", "title", "Sleepless in Seattle", "released", 1993);
Vertex sleeplessInSeattle = graph.addVertex(T.label, "movie", "title", "Sleepless in Seattle", "released", 1993);
Vertex ritaW = graph.addVertex(T.label, "person", "name", "Rita Wilson", "born", 1956);
Vertex billPull = graph.addVertex(T.label, "person", "name", "Bill Pullman", "born", 1953);
@ -329,8 +325,7 @@ public class GraphOfTheMoviesExample {
rosieO.addEdge("ACTED_IN", sleeplessInSeattle, "roles", "Becky");
noraE.addEdge("DIRECTED", sleeplessInSeattle, "score", 10);
Vertex joeVersustheVolcano =
graph.addVertex(T.label, "movie", "title", "Joe Versus the Volcano", "released", 1990);
Vertex joeVersustheVolcano = graph.addVertex(T.label, "movie", "title", "Joe Versus the Volcano", "released", 1990);
Vertex johnS = graph.addVertex(T.label, "person", "name", "John Patrick Stanley", "born", 1950);
Vertex nathan = graph.addVertex(T.label, "person", "name", "nathan Lane", "born", 1956);
@ -340,8 +335,7 @@ public class GraphOfTheMoviesExample {
nathan.addEdge("ACTED_IN", joeVersustheVolcano, "roles", "Baw");
johnS.addEdge("DIRECTED", joeVersustheVolcano, "score", 10);
Vertex whenHarryMetSally =
graph.addVertex(T.label, "movie", "title", "When Harry Met Sally", "released", 1998);
Vertex whenHarryMetSally = graph.addVertex(T.label, "movie", "title", "When Harry Met Sally", "released", 1998);
Vertex billyC = graph.addVertex(T.label, "person", "name", "Billy Crystal", "born", 1948);
Vertex carrieF = graph.addVertex(T.label, "person", "name", "carrie Fisher", "born", 1956);
@ -356,8 +350,7 @@ public class GraphOfTheMoviesExample {
noraE.addEdge("PRODUCED", whenHarryMetSally, "score", 10);
noraE.addEdge("WROTE", whenHarryMetSally, "score", 10);
Vertex thatThingYouDo =
graph.addVertex(T.label, "movie", "title", "That Thing You Do", "released", 1996);
Vertex thatThingYouDo = graph.addVertex(T.label, "movie", "title", "That Thing You Do", "released", 1996);
Vertex livT = graph.addVertex(T.label, "person", "name", "Liv Tyler", "born", 1977);
@ -366,8 +359,7 @@ public class GraphOfTheMoviesExample {
charlize.addEdge("ACTED_IN", thatThingYouDo, "roles", "Tina");
tomH.addEdge("DIRECTED", thatThingYouDo, "score", 10);
Vertex theReplacements =
graph.addVertex(T.label, "movie", "title", "The Replacements", "released", 2000);
Vertex theReplacements = graph.addVertex(T.label, "movie", "title", "The Replacements", "released", 2000);
Vertex brooke = graph.addVertex(T.label, "person", "name", "brooke Langton", "born", 1970);
Vertex gene = graph.addVertex(T.label, "person", "name", "gene Hackman", "born", 1930);
@ -433,7 +425,7 @@ public class GraphOfTheMoviesExample {
tomH.addEdge("ACTED_IN", cloudatlas, "roles", "Zachry, Dr. Henry Goose, Isaac Sachs, Dermot Hoggins");
hugo.addEdge("ACTED_IN", cloudatlas, "roles", "Bill Smoke, Haskell Moore, Tadeusz Kesselring, Nurse Noakes,"
+ " Boardman Mephi, Old Georgie");
+ " Boardman Mephi, Old Georgie");
halleB.addEdge("ACTED_IN", cloudatlas, "roles", "Luisa Rey, Jocasta ayrs, Ovid, Meronym");
jimB.addEdge("ACTED_IN", cloudatlas, "roles", "Vyvyan ayrs, Captain Molyneux, Timothy Cavendish");
tomT.addEdge("DIRECTED", cloudatlas, "score", 10);
@ -442,8 +434,7 @@ public class GraphOfTheMoviesExample {
davidMitchell.addEdge("WROTE", cloudatlas, "score", 10);
stefanarndt.addEdge("PRODUCED", cloudatlas, "score", 10);
Vertex theDaVinciCode =
graph.addVertex(T.label, "movie", "title", "The Da Vinci Code", "released", 2006);
Vertex theDaVinciCode = graph.addVertex(T.label, "movie", "title", "The Da Vinci Code", "released", 2006);
Vertex ianM = graph.addVertex(T.label, "person", "name", "Ian McKellen", "born", 1939);
Vertex audreyT = graph.addVertex(T.label, "person", "name", "audrey Tautou", "born", 1976);
@ -584,8 +575,7 @@ public class GraphOfTheMoviesExample {
helenH.addEdge("ACTED_IN", castaway, "roles", "Kelly Frears");
robertZ.addEdge("DIRECTED", castaway, "score", 10);
Vertex oneFlewOvertheCuckoosNest =
graph.addVertex(T.label, "movie", "title", "One Flew Over the Cuckoo's Nest", "released", 1975);
Vertex oneFlewOvertheCuckoosNest = graph.addVertex(T.label, "movie", "title", "One Flew Over the Cuckoo's Nest", "released", 1975);
Vertex milosF = graph.addVertex(T.label, "person", "name", "Milos Forman", "born", 1932);
@ -593,8 +583,7 @@ public class GraphOfTheMoviesExample {
dannyD.addEdge("ACTED_IN", oneFlewOvertheCuckoosNest, "roles", "Martini");
milosF.addEdge("DIRECTED", oneFlewOvertheCuckoosNest, "score", 10);
Vertex somethingsGottaGive =
graph.addVertex(T.label, "movie", "title", "Something's Gotta Give", "released", 2003);
Vertex somethingsGottaGive = graph.addVertex(T.label, "movie", "title", "Something's Gotta Give", "released", 2003);
Vertex dianeK = graph.addVertex(T.label, "person", "name", "Diane Keaton", "born", 1946);
Vertex nancyM = graph.addVertex(T.label, "person", "name", "Nancy Meyers", "born", 1949);
@ -606,8 +595,7 @@ public class GraphOfTheMoviesExample {
nancyM.addEdge("PRODUCED", somethingsGottaGive, "score", 10);
nancyM.addEdge("WROTE", somethingsGottaGive, "score", 10);
Vertex bicentennialMan =
graph.addVertex(T.label, "movie", "title", "Bicentennial Man", "released", 2000);
Vertex bicentennialMan = graph.addVertex(T.label, "movie", "title", "Bicentennial Man", "released", 2000);
Vertex chrisC = graph.addVertex(T.label, "person", "name", "Chris Columbus", "born", 1958);
@ -615,8 +603,7 @@ public class GraphOfTheMoviesExample {
oliverP.addEdge("ACTED_IN", bicentennialMan, "roles", "Rupert Burns");
chrisC.addEdge("DIRECTED", bicentennialMan, "score", 10);
Vertex charlieWilsonsWar =
graph.addVertex(T.label, "movie", "title", "Charlie Wilson's War", "released", 2007);
Vertex charlieWilsonsWar = graph.addVertex(T.label, "movie", "title", "Charlie Wilson's War", "released", 2007);
Vertex juliaR = graph.addVertex(T.label, "person", "name", "Julia Roberts", "born", 1967);
@ -625,15 +612,13 @@ public class GraphOfTheMoviesExample {
philipH.addEdge("ACTED_IN", charlieWilsonsWar, "roles", "Gust avrakotos");
mikeN.addEdge("DIRECTED", charlieWilsonsWar, "score", 10);
Vertex thePolarExpress =
graph.addVertex(T.label, "movie", "title", "The Polar Express", "released", 2004);
Vertex thePolarExpress = graph.addVertex(T.label, "movie", "title", "The Polar Express", "released", 2004);
tomH.addEdge("ACTED_IN", thePolarExpress, "roles", "Hero Boy");
robertZ.addEdge("DIRECTED", thePolarExpress, "score", 10);
Vertex aLeagueofTheirOwn =
graph.addVertex(T.label, "movie", "title", "a League of Their Own", "released", 1992);
Vertex aLeagueofTheirOwn = graph.addVertex(T.label, "movie", "title", "a League of Their Own", "released", 1992);
Vertex madonna = graph.addVertex(T.label, "person", "name", "madonna", "born", 1954);
Vertex geenaD = graph.addVertex(T.label, "person", "name", "Geena Davis", "born", 1956);
@ -651,5 +636,3 @@ public class GraphOfTheMoviesExample {
graph.tx().commit();
}
}

157
hugegraph-example/src/main/java/com/baidu/hugegraph/example/PerfExample1.java
View File

@ -24,104 +24,25 @@ import java.util.List;
import java.util.Random;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Transaction;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.slf4j.Logger;
import com.baidu.hugegraph.HugeGraph;
import com.baidu.hugegraph.backend.BackendException;
import com.baidu.hugegraph.backend.cache.Cache;
import com.baidu.hugegraph.backend.cache.CacheManager;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.event.EventHub;
import com.baidu.hugegraph.perf.PerfUtil;
import com.baidu.hugegraph.schema.SchemaManager;
import com.baidu.hugegraph.structure.HugeVertex;
import com.baidu.hugegraph.util.Log;
import com.datastax.driver.core.exceptions.NoHostAvailableException;
public class PerfExample1 {
public static final int PERSON_NUM = 70;
public static final int SOFTWARE_NUM = 30;
public static final int EDGE_NUM = 100;
private static final Logger LOG = Log.logger(PerfExample1.class);
public class PerfExample1 extends PerfExampleBase {
public static void main(String[] args) throws InterruptedException {
if (args.length != 3) {
System.out.println("Usage: threadCount times multiple");
return;
}
int threadCount = Integer.parseInt(args[0]);
int times = Integer.parseInt(args[1]);
int multiple = Integer.parseInt(args[2]);
// NOTE: this test with HugeGraph is for local, change it into
// client if test with restful server from remote
HugeGraph hugegraph = ExampleUtil.loadGraph(true);
GraphManager graph = new GraphManager(hugegraph);
initSchema(hugegraph.schema());
testInsertPerf(graph, threadCount, times, multiple);
hugegraph.close();
PerfExample1 tester = new PerfExample1();
tester.test(args);
// Stop event hub before main thread exits
EventHub.destroy(30);
}
/**
* Multi-threaded and multi-commits and batch insertion test
* @param graph
* @param threadCount
* The count of threads that perform the insert operation at the
* same time
* @param times
* The transaction commit times for each thread
* @param multiple
* The coefficient to multiple number of vertices(100) and edges(100)
* for each transaction commit
* @throws InterruptedException
*/
public static void testInsertPerf(GraphManager graph,
int threadCount,
int times,
int multiple)
throws InterruptedException {
List<Thread> threads = new ArrayList<>(threadCount);
for (int i = 0; i < threadCount; i++) {
Thread t = new Thread(() -> {
testInsertPerf(graph, times, multiple);
graph.close();
LOG.info("option = {}", PerfUtil.instance().toECharts());
});
threads.add(t);
}
long beginTime = System.currentTimeMillis();
for (Thread t : threads) {
t.start();
}
for (Thread t : threads) {
t.join();
}
long endTime = System.currentTimeMillis();
// Total edges
long edges = EDGE_NUM * threadCount * times * multiple;
long cost = endTime - beginTime;
LOG.info("Total edges: {}, cost times: {}", edges, cost);
LOG.info("Rate with threads: {} edges/s", edges * 1000 / cost);
}
public static void initSchema(SchemaManager schema) {
@Override
protected void initSchema(SchemaManager schema) {
schema.propertyKey("name").asText().create();
schema.propertyKey("age").asInt().create();
schema.propertyKey("lang").asText().create();
@ -129,33 +50,34 @@ public class PerfExample1 {
schema.propertyKey("price").asInt().create();
schema.vertexLabel("person")
.properties("name", "age")
.primaryKeys("name")
.ifNotExist()
.create();
.properties("name", "age")
.primaryKeys("name")
.ifNotExist()
.create();
schema.vertexLabel("software")
.properties("name", "lang", "price")
.primaryKeys("name")
.ifNotExist()
.create();
.properties("name", "lang", "price")
.primaryKeys("name")
.ifNotExist()
.create();
schema.edgeLabel("knows")
.sourceLabel("person").targetLabel("person")
.properties("date")
.nullableKeys("date")
.ifNotExist()
.create();
.sourceLabel("person").targetLabel("person")
.properties("date")
.nullableKeys("date")
.ifNotExist()
.create();
schema.edgeLabel("created")
.sourceLabel("person").targetLabel("software")
.properties("date")
.nullableKeys("date")
.ifNotExist()
.create();
.sourceLabel("person").targetLabel("software")
.properties("date")
.nullableKeys("date")
.ifNotExist()
.create();
}
public static void testInsertPerf(GraphManager graph, int times, int multiple) {
@Override
protected void testInsertPerf(GraphManager graph, int times, int multiple) {
List<Object> personIds = new ArrayList<>(PERSON_NUM * multiple);
List<Object> softwareIds = new ArrayList<>(SOFTWARE_NUM * multiple);
@ -211,33 +133,4 @@ public class PerfExample1 {
softwareIds.clear();
}
}
static class GraphManager {
private HugeGraph hugegraph;
private Cache cache = CacheManager.instance().cache("perf-test");
public GraphManager(HugeGraph hugegraph) {
this.hugegraph = hugegraph;
}
public Transaction tx() {
return this.hugegraph.tx();
}
public void close() {
this.hugegraph.close();
}
public Vertex addVertex(Object... keyValues) {
HugeVertex v = (HugeVertex) this.hugegraph.addVertex(keyValues);
this.cache.update(v.id(), v.resetTx());
return v;
}
public Vertex getVertex(Object id) {
return ((Vertex) this.cache.getOrFetch((Id) id, k -> {
return this.hugegraph.vertices(k).next();
}));
}
}
}

119
hugegraph-example/src/main/java/com/baidu/hugegraph/example/PerfExample2.java Normal file
View File

@ -0,0 +1,119 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.example;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import com.baidu.hugegraph.backend.BackendException;
import com.baidu.hugegraph.event.EventHub;
import com.baidu.hugegraph.schema.SchemaManager;
import com.datastax.driver.core.exceptions.NoHostAvailableException;
public class PerfExample2 extends PerfExampleBase {
public static void main(String[] args) throws InterruptedException {
PerfExample2 tester = new PerfExample2();
tester.test(args);
// Stop event hub before main thread exits
EventHub.destroy(30);
}
@Override
protected void initSchema(SchemaManager schema) {
schema.propertyKey("name").asText().create();
schema.vertexLabel("person")
.useAutomaticId()
.ifNotExist()
.create();
schema.vertexLabel("software")
.useAutomaticId()
.ifNotExist()
.create();
schema.edgeLabel("knows")
.sourceLabel("person")
.targetLabel("person")
.ifNotExist()
.create();
schema.edgeLabel("created")
.sourceLabel("person")
.targetLabel("software")
.ifNotExist()
.create();
}
@Override
protected void testInsertPerf(GraphManager graph, int times, int multiple) {
List<Object> personIds = new ArrayList<>(PERSON_NUM * multiple);
List<Object> softwareIds = new ArrayList<>(SOFTWARE_NUM * multiple);
for (int time = 0; time < times; time++) {
LOG.debug("============== random person vertex ===============");
for (int i = 0; i < PERSON_NUM * multiple; i++) {
Vertex vetex = graph.addVertex(T.label, "person");
personIds.add(vetex.id());
LOG.debug("Add person: {}", vetex);
}
LOG.debug("============== random software vertex ============");
for (int i = 0; i < SOFTWARE_NUM * multiple; i++) {
Vertex vetex = graph.addVertex(T.label, "software");
softwareIds.add(vetex.id());
LOG.debug("Add software: {}", vetex);
}
LOG.debug("========== random knows & created edges ==========");
for (int i = 0; i < EDGE_NUM / 2 * multiple; i++) {
Random random = new Random();
// Add edge: person --knows-> person
Object p1 = personIds.get(random.nextInt(PERSON_NUM));
Object p2 = personIds.get(random.nextInt(PERSON_NUM));
graph.getVertex(p1).addEdge("knows", graph.getVertex(p2));
// Add edge: person --created-> software
Object p3 = personIds.get(random.nextInt(PERSON_NUM));
Object s1 = softwareIds.get(random.nextInt(SOFTWARE_NUM));
graph.getVertex(p3).addEdge("created", graph.getVertex(s1));
}
try {
graph.tx().commit();
} catch (BackendException e) {
if (e.getCause() instanceof NoHostAvailableException) {
LOG.warn("Failed to commit tx: {}", e.getMessage());
} else {
throw e;
}
}
personIds.clear();
softwareIds.clear();
}
}
}

152
hugegraph-example/src/main/java/com/baidu/hugegraph/example/PerfExampleBase.java Normal file
View File

@ -0,0 +1,152 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.example;
import java.util.ArrayList;
import java.util.List;
import org.apache.tinkerpop.gremlin.structure.Transaction;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.slf4j.Logger;
import com.baidu.hugegraph.HugeGraph;
import com.baidu.hugegraph.backend.cache.Cache;
import com.baidu.hugegraph.backend.cache.CacheManager;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.perf.PerfUtil;
import com.baidu.hugegraph.schema.SchemaManager;
import com.baidu.hugegraph.structure.HugeVertex;
import com.baidu.hugegraph.util.Log;
public abstract class PerfExampleBase {
public static final int PERSON_NUM = 70;
public static final int SOFTWARE_NUM = 30;
public static final int EDGE_NUM = 100;
protected static final Logger LOG = Log.logger(PerfExampleBase.class);
public int test(String[] args) throws InterruptedException {
if (args.length != 3) {
System.out.println("Usage: threadCount times multiple");
return -1;
}
int threadCount = Integer.parseInt(args[0]);
int times = Integer.parseInt(args[1]);
int multiple = Integer.parseInt(args[2]);
// NOTE: this test with HugeGraph is for local, change it into
// client if test with restful server from remote
HugeGraph hugegraph = ExampleUtil.loadGraph(true);
GraphManager graph = new GraphManager(hugegraph);
initSchema(hugegraph.schema());
testInsertPerf(graph, threadCount, times, multiple);
hugegraph.close();
return 0;
}
/**
* Multi-threaded and multi-commits and batch insertion test
* @param graph
* @param threadCount
* The count of threads that perform the insert operation at the
* same time
* @param times
* The transaction commit times for each thread
* @param multiple
* The coefficient to multiple number of vertices(100) and edges(100)
* for each transaction commit
* @throws InterruptedException
*/
public void testInsertPerf(GraphManager graph,
int threadCount,
int times,
int multiple)
throws InterruptedException {
List<Thread> threads = new ArrayList<>(threadCount);
for (int i = 0; i < threadCount; i++) {
Thread t = new Thread(() -> {
testInsertPerf(graph, times, multiple);
graph.close();
LOG.info("option = {}", PerfUtil.instance().toECharts());
});
threads.add(t);
}
long beginTime = System.currentTimeMillis();
for (Thread t : threads) {
t.start();
}
for (Thread t : threads) {
t.join();
}
long endTime = System.currentTimeMillis();
// Total edges
long edges = EDGE_NUM * threadCount * times * multiple;
long cost = endTime - beginTime;
LOG.info("Total edges: {}, cost times: {}", edges, cost);
LOG.info("Rate with threads: {} edges/s", edges * 1000 / cost);
}
protected abstract void initSchema(SchemaManager schema);
protected abstract void testInsertPerf(GraphManager graph,
int times,
int multiple);
protected static class GraphManager {
private HugeGraph hugegraph;
private Cache cache = CacheManager.instance().cache("perf-test");
public GraphManager(HugeGraph hugegraph) {
this.hugegraph = hugegraph;
}
public Transaction tx() {
return this.hugegraph.tx();
}
public void close() {
this.hugegraph.close();
}
public Vertex addVertex(Object... keyValues) {
HugeVertex v = (HugeVertex) this.hugegraph.addVertex(keyValues);
this.cache.update(v.id(), v.resetTx());
return v;
}
public Vertex getVertex(Object id) {
return ((Vertex) this.cache.getOrFetch((Id) id, k -> {
return this.hugegraph.vertices(k).next();
}));
}
}
}

8
hugegraph-example/src/main/resources/hugegraph.properties
View File

@ -3,10 +3,12 @@ gremlin.graph=com.baidu.hugegraph.HugeFactory
backend=cassandra
serializer=cassandra
#backend=rocksdb
#serializer=binary
#rocksdb.data_path=.
#rocksdb.wal_path=.
store=hugegraph
store.schema=huge_schema
store.graph=huge_graph
store.index=huge_index
# cassandra backend config
cassandra.host=localhost

27
hugegraph-rocksdb/pom.xml Normal file
View File

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>hugegraph</artifactId>
<groupId>com.baidu.hugegraph</groupId>
<version>0.4.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hugegraph-rocksdb</artifactId>
<dependencies>
<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
<version>5.8.6</version>
</dependency>
</dependencies>
</project>

97
hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBFeatures.java Normal file
View File

@ -0,0 +1,97 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.backend.store.rocksdb;
import com.baidu.hugegraph.backend.store.BackendFeatures;
public class RocksDBFeatures implements BackendFeatures {
@Override
public boolean supportsScanToken() {
return false;
}
@Override
public boolean supportsScanKeyPrefix() {
return true;
}
@Override
public boolean supportsScanKeyRange() {
return true;
}
@Override
public boolean supportsQuerySchemaByName() {
// No index in RocksDB
return false;
}
@Override
public boolean supportsQueryByLabel() {
// No index in RocksDB
return false;
}
@Override
public boolean supportsQueryWithSearchCondition() {
return true;
}
@Override
public boolean supportsQueryWithOrderBy() {
return true;
}
@Override
public boolean supportsQueryWithContains() {
// TODO: Need to traversal all items
return false;
}
@Override
public boolean supportsQueryWithContainsKey() {
// TODO: Need to traversal all items
return false;
}
@Override
public boolean supportsDeleteEdgeByLabel() {
// No index in RocksDB
return false;
}
@Override
public boolean supportsUpdateEdgeProperty() {
// Edge properties are stored in a cell(column value)
return false;
}
@Override
public boolean supportsTransaction() {
// Supports tx with WriteBatch
return true;
}
@Override
public boolean supportsNumberType() {
return false;
}
}

62
hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBOptions.java Normal file
View File

@ -0,0 +1,62 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.backend.store.rocksdb;
import static com.baidu.hugegraph.config.OptionChecker.disallowEmpty;
import com.baidu.hugegraph.config.ConfigOption;
import com.baidu.hugegraph.config.OptionHolder;
public class RocksDBOptions extends OptionHolder {
private RocksDBOptions() {
super();
}
private static volatile RocksDBOptions instance;
public static RocksDBOptions Instance() {
if (instance == null) {
synchronized (RocksDBOptions.class) {
if (instance == null) {
instance = new RocksDBOptions();
instance.registerOptions();
}
}
}
return instance;
}
public static final ConfigOption<String> ROCKS_DATA_PATH = new ConfigOption<>(
"rocksdb.data_path",
"rocksdbdata",
true,
"The path for storing data of RocksDB.",
disallowEmpty(String.class)
);
public static final ConfigOption<String> ROCKS_WAL_PATH = new ConfigOption<>(
"rocksdb.wal_path",
"rocksdbwal",
true,
"The path for storing WAL of RocksDB.",
disallowEmpty(String.class)
);
}

425
hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBSessions.java Normal file
View File

@ -0,0 +1,425 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.backend.store.rocksdb;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.ColumnFamilyOptionsInterface;
import org.rocksdb.DBOptions;
import org.rocksdb.DBOptionsInterface;
import org.rocksdb.Options;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.rocksdb.RocksIterator;
import org.rocksdb.WriteBatch;
import org.rocksdb.WriteOptions;
import com.baidu.hugegraph.backend.BackendException;
import com.baidu.hugegraph.backend.store.BackendEntry.BackendColumn;
import com.baidu.hugegraph.backend.store.BackendSessionPool;
import com.baidu.hugegraph.util.Bytes;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.StringEncoding;
import com.google.common.primitives.UnsignedBytes;
public class RocksDBSessions extends BackendSessionPool {
private final RocksDB rocksdb;
private final Map<String, ColumnFamilyHandle> cfs;
public RocksDBSessions(String path, String walPath)
throws RocksDBException {
this.cfs = new HashMap<>();
Options options = new Options();
this.initOptions(options, options);
options.setWalDir(walPath);
this.rocksdb = RocksDB.open(options, path);
}
public RocksDBSessions(String path, String walPath, List<String> cfNames)
throws RocksDBException {
this.cfs = new HashMap<>();
// Old CFs should always be opened
List<String> cfs = this.mergeOldCFs(path, cfNames);
List<ColumnFamilyDescriptor> cfds = new ArrayList<>(cfs.size());
for (String cf : cfs) {
ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(cf));
this.initOptions(null, cfd.columnFamilyOptions());
cfds.add(cfd);
}
List<ColumnFamilyHandle> cfhs = new ArrayList<>();
DBOptions options = new DBOptions();
this.initOptions(options, null);
options.setWalDir(walPath);
this.rocksdb = RocksDB.open(options, path, cfds, cfhs);
E.checkState(cfhs.size() == cfs.size(),
"Excepct same size of cf-handles and cf-names");
for (int i = 0; i < cfs.size(); i++) {
this.cfs.put(cfs.get(i), cfhs.get(i));
}
}
public Set<String> openedTables() {
return this.cfs.keySet();
}
public void createTable(String table) throws RocksDBException {
// Should we use options.setCreateMissingColumnFamilies() to create CF
ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(table));
this.initOptions(null, cfd.columnFamilyOptions());
this.cfs.put(table, this.rocksdb.createColumnFamily(cfd));
}
public void dropTable(String table) throws RocksDBException {
ColumnFamilyHandle cfh = cf(table);
this.rocksdb.dropColumnFamily(cfh);
cfh.close();
this.cfs.remove(table);
}
public final synchronized Session session() {
return (Session) super.getOrNewSession();
}
@Override
protected final synchronized Session newSession() {
E.checkState(this.rocksdb != null,
"RocksDB has not been initialized");
return new Session();
}
@Override
protected synchronized void doClose() {
for (ColumnFamilyHandle cf : this.cfs.values()) {
cf.close();
}
this.cfs.clear();
this.rocksdb.close();
}
private RocksDB rocksdb() {
return this.rocksdb;
}
private ColumnFamilyHandle cf(String cf) {
ColumnFamilyHandle cfh = this.cfs.get(cf);
if (cfh == null) {
throw new BackendException("Table '%s' is not opened", cf);
}
return cfh;
}
private List<String> mergeOldCFs(String path, List<String> cfNames)
throws RocksDBException {
List<String> cfs = new ArrayList<>(cfNames);
List<byte[]> oldCFs = RocksDB.listColumnFamilies(new Options(), path);
if (oldCFs.isEmpty()) {
cfs.add("default");
} else {
for (byte[] oldCF : oldCFs) {
String old = decode(oldCF);
if (!cfNames.contains(old)) {
cfs.add(old);
}
}
}
return cfs;
}
private void initOptions(DBOptionsInterface<?> db,
ColumnFamilyOptionsInterface<?> cf) {
if (db != null) {
db.setCreateIfMissing(true);
// Optimize RocksDB
//db.setIncreaseParallelism(4);
}
if (cf != null) {
// https://github.com/facebook/rocksdb/tree/master/utilities/merge_operators
cf.setMergeOperatorName("uint64add"); // uint64add/stringappend
}
}
public static final byte[] encode(String string) {
return StringEncoding.encode(string);
}
public static final String decode(byte[] bytes) {
return StringEncoding.decode(bytes);
}
public static final byte[] increase(byte[] bytes) {
assert bytes[bytes.length - 1] != 0xff;
bytes[bytes.length - 1] += 0x01; // FIXME: maybe overflow
return bytes;
}
/**
* Session for RocksDB
*/
public final class Session extends BackendSessionPool.Session {
private boolean closed;
private WriteBatch batch;
private WriteOptions writeOptions;
public Session() {
this.closed = false;
this.batch = new WriteBatch();
this.writeOptions = new WriteOptions();
//this.writeOptions.setDisableWAL(true);
//this.writeOptions.setSync(false);
}
@Override
public void close() {
assert this.closeable();
this.closed = true;
}
@Override
public boolean closed() {
return this.closed;
}
/**
* Clear updates not committed in the session
*/
@Override
public void clear() {
this.batch.clear();
}
/**
* Commit all updates(put/delete) to DB
*/
@Override
public Object commit() {
try {
rocksdb().write(this.writeOptions, this.batch);
} catch (RocksDBException e) {
//this.batch.rollbackToSavePoint();
throw new BackendException(e);
} finally {
this.batch.clear();
}
return null;
}
/**
* Add a KV record to a table
*/
public void put(String table, byte[] key, byte[] value) {
this.batch.put(cf(table), key, value);
}
/**
* Merge a record to an existing key to a table
* For more details about merge-operator:
* https://github.com/facebook/rocksdb/wiki/merge-operator
*/
public void merge(String table, byte[] key, byte[] value) {
this.batch.merge(cf(table), key, value);
}
/**
* Delete a record by key from a table
*/
public void remove(String table, byte[] key) {
this.batch.remove(cf(table), key);
}
/**
* Delete a record by key(or prefix with key) from a table
*/
public void delete(String table, byte[] key) {
byte[] keyFrom = key;
byte[] keyTo = Arrays.copyOf(key, key.length);
keyTo = increase(keyTo);
this.batch.deleteRange(cf(table), keyFrom, keyTo);
}
/**
* Delete a range of keys from a table
*/
public void delete(String table, byte[] keyFrom, byte[] keyTo) {
this.batch.deleteRange(cf(table), keyFrom, keyTo);
}
/**
* Get a record by key from a table
*/
public byte[] get(String table, byte[] key) {
try {
return rocksdb().get(cf(table), key);
} catch (RocksDBException e) {
throw new BackendException(e);
}
}
/**
* Scan all records from a table
*/
public Iterator<BackendColumn> scan(String table) {
return scan(table, null, null, false);
}
/**
* Scan records by key prefix from a table
*/
public Iterator<BackendColumn> scan(String table, byte[] key) {
return scan(table, key, null, true);
}
/**
* Scan records by key range from a table
*/
public Iterator<BackendColumn> scan(String table,
byte[] keyFrom,
byte[] keyTo) {
return scan(table, keyFrom, keyTo, false);
}
/**
* Scan records by key prefix or key range from a table
*/
public Iterator<BackendColumn> scan(String table,
byte[] keyFrom,
byte[] keyTo,
boolean matchPrefix) {
RocksIterator itor = rocksdb().newIterator(cf(table));
return new ColumnIterator(table, itor, keyFrom, keyTo, matchPrefix);
}
}
/**
* A wrapper for RocksIterator that convert RocksDB results to std Iterator
*/
private static class ColumnIterator implements Iterator<BackendColumn> {
private final String table;
private final RocksIterator itor;
private byte[] keyBegin;
private byte[] keyEnd;
private boolean matchPrefix;
// Don't use BytewiseComparator
private static final Comparator<byte[]> C =
UnsignedBytes.lexicographicalComparator();
public ColumnIterator(String table,
RocksIterator itor,
byte[] keyBegin,
byte[] keyEnd,
boolean matchPrefix) {
E.checkNotNull(itor, "itor");
this.table = table;
this.itor = itor;
this.keyBegin = keyBegin;
this.keyEnd = keyEnd;
this.matchPrefix = matchPrefix;
if (matchPrefix && keyEnd != null) {
throw new IllegalArgumentException(
"Param keyEnd must be null when matchPrefix=true");
}
//this.dump();
if (keyBegin != null) {
this.itor.seek(keyBegin);
} else {
this.itor.seekToFirst();
}
}
/**
* Just for debug
*/
@SuppressWarnings("unused")
private void dump() {
this.itor.seekToFirst();
System.out.println(">>>> seek from " + this.table + ": " +
(this.keyBegin == null ? "*" :
StringEncoding.decode(this.keyBegin)) +
(this.itor.isValid() ? "" : " - No data"));
for (; this.itor.isValid(); this.itor.next()) {
System.out.println(StringEncoding.decode(this.itor.key()) +
": " +
StringEncoding.decode(this.itor.value()));
}
}
@Override
public boolean hasNext() {
boolean matched = false;
if (this.itor.isOwningHandle() && this.itor.isValid()) {
if (this.matchPrefix) {
// Prefix match? TODO: use prefix_extractor instead
matched = Bytes.prefixWith(this.itor.key(), this.keyBegin);
} else if (this.keyEnd != null) {
// Range match?
matched = C.compare(this.itor.key(), this.keyEnd) < 0;
} else {
// Any match
matched = true;
}
}
if (!matched) {
// Free if finished
this.itor.close();
}
return matched;
}
@Override
public BackendColumn next() {
if (!this.itor.isOwningHandle() || !this.itor.isValid()) {
throw new NoSuchElementException();
}
BackendColumn entry = new BackendColumn();
entry.name = this.itor.key();
entry.value = this.itor.value();
this.itor.next();
return entry;
}
}
}

305
hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java Normal file
View File

@ -0,0 +1,305 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.backend.store.rocksdb;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.rocksdb.RocksDBException;
import org.slf4j.Logger;
import com.baidu.hugegraph.backend.BackendException;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.backend.store.BackendFeatures;
import com.baidu.hugegraph.backend.store.BackendMutation;
import com.baidu.hugegraph.backend.store.BackendStore;
import com.baidu.hugegraph.backend.store.BackendStoreProvider;
import com.baidu.hugegraph.backend.store.MutateItem;
import com.baidu.hugegraph.config.HugeConfig;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.Log;
public class RocksDBStore implements BackendStore {
private static final Logger LOG = Log.logger(RocksDBStore.class);
private static final BackendFeatures FEATURES = new RocksDBFeatures();
private final String name;
private final String database;
private final BackendStoreProvider provider;
private final Map<HugeType, RocksDBTable> tables;
private HugeConfig conf;
private RocksDBSessions sessions;
public RocksDBStore(final BackendStoreProvider provider,
final String database, final String name) {
this.tables = new HashMap<>();
this.provider = provider;
this.database = database;
this.name = name;
this.conf = null;
this.sessions = null;
}
protected void registerTableManager(HugeType type, RocksDBTable table) {
this.tables.put(type, table);
}
protected final RocksDBTable table(HugeType type) {
assert type != null;
RocksDBTable table = this.tables.get(type);
if (table == null) {
throw new BackendException("Unsupported table type: %s", type);
}
return table;
}
protected final List<String> tableNames() {
return this.tables.values().stream().map(t -> t.table())
.collect(Collectors.toList());
}
public String database() {
return this.database;
}
@Override
public String name() {
return this.name;
}
@Override
public BackendStoreProvider provider() {
return this.provider;
}
@Override
public BackendFeatures features() {
return FEATURES;
}
@Override
public void open(HugeConfig config) {
E.checkNotNull(config, "config");
this.conf = config;
if (this.sessions != null) {
LOG.debug("Store {} has been opened before", this.name);
this.sessions.useSession();
return;
}
String dataPath = this.conf.get(RocksDBOptions.ROCKS_DATA_PATH);
dataPath = Paths.get(dataPath, this.name).toString();
String walPath = this.conf.get(RocksDBOptions.ROCKS_WAL_PATH);
walPath = Paths.get(walPath, this.name).toString();
try {
this.sessions = new RocksDBSessions(dataPath, walPath,
this.tableNames());
} catch (RocksDBException e) {
if (!e.getMessage().contains("Column family not found")) {
LOG.error("Failed to open RocksDB '{}'", this.name, e);
throw new BackendException("Failed to open RocksDB '%s'",
e, this.name);
}
LOG.info("Failed to open RocksDB '{}' with database '{}', " +
"try to init CF later", this.name, this.database);
try {
this.sessions = new RocksDBSessions(dataPath, walPath);
} catch (RocksDBException e1) {
LOG.error("Failed to open RocksDB with default CF", e);
}
}
LOG.debug("Store opened: {}", this.name);
}
@Override
public void close() {
this.checkOpened();
this.sessions.close();
LOG.debug("Store closed: {}", this.name);
}
@Override
public void mutate(BackendMutation mutation) {
LOG.debug("Store {} mutation: {}", this.name, mutation);
this.checkOpened();
RocksDBSessions.Session session = this.sessions.session();
for (List<MutateItem> items : mutation.mutation().values()) {
for (MutateItem item : items) {
this.mutate(session, item);
}
}
}
private void mutate(RocksDBSessions.Session session, MutateItem item) {
BackendEntry entry = item.entry();
RocksDBTable table = this.table(entry.type());
switch (item.action()) {
case INSERT:
table.insert(session, entry);
break;
case DELETE:
table.delete(session, entry);
break;
case APPEND:
table.append(session, entry);
break;
case ELIMINATE:
table.eliminate(session, entry);
break;
default:
throw new AssertionError(String.format(
"Unsupported mutate type: %s", item.action()));
}
}
@Override
public Iterator<BackendEntry> query(Query query) {
this.checkOpened();
RocksDBSessions.Session session = this.sessions.session();
RocksDBTable table = this.table(query.resultType());
return table.query(session, query);
}
@Override
public void init() {
this.checkOpened();
for (String table : this.tableNames()) {
try {
this.sessions.createTable(table);
} catch (RocksDBException e) {
throw new BackendException("Failed to create '%s' for '%s'",
e, table, this.name);
}
}
}
@Override
public void clear() {
this.checkOpened();
for (String table : this.tableNames()) {
try {
this.sessions.dropTable(table);
} catch (BackendException e) {
if (e.getMessage().contains("is not opened")) {
continue;
}
throw e;
} catch (RocksDBException e) {
throw new BackendException("Failed to drop '%s' for '%s'",
e, table, this.name);
}
}
}
@Override
public void beginTx() {
// pass
}
@Override
public void commitTx() {
this.checkOpened();
RocksDBSessions.Session session = this.sessions.session();
try {
session.commit();
} finally {
session.clear();
}
}
@Override
public void rollbackTx() {
// pass
}
@Override
public Object metadata(HugeType type, String meta, Object[] args) {
throw new UnsupportedOperationException("RocksDBStore.metadata()");
}
private void checkOpened() {
E.checkState(this.sessions != null,
"RocksDB store has not been initialized");
}
/***************************** Store defines *****************************/
public static class RocksDBSchemaStore extends RocksDBStore {
public RocksDBSchemaStore(BackendStoreProvider provider,
String database, String name) {
super(provider, database, name);
registerTableManager(HugeType.VERTEX_LABEL,
new RocksDBTables.VertexLabel(database));
registerTableManager(HugeType.EDGE_LABEL,
new RocksDBTables.EdgeLabel(database));
registerTableManager(HugeType.PROPERTY_KEY,
new RocksDBTables.PropertyKey(database));
registerTableManager(HugeType.INDEX_LABEL,
new RocksDBTables.IndexLabel(database));
registerTableManager(HugeType.COUNTERS,
new RocksDBTables.Counters(database));
}
}
public static class RocksDBGraphStore extends RocksDBStore {
public RocksDBGraphStore(BackendStoreProvider provider,
String database, String name) {
super(provider, database, name);
registerTableManager(HugeType.VERTEX,
new RocksDBTables.Vertex(database));
registerTableManager(HugeType.EDGE,
new RocksDBTables.Edge(database));
registerTableManager(HugeType.SECONDARY_INDEX,
new RocksDBTables.SecondaryIndex(database));
registerTableManager(HugeType.SEARCH_INDEX,
new RocksDBTables.SearchIndex(database));
}
}
}

77
hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java Normal file
View File

@ -0,0 +1,77 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.backend.store.rocksdb;
import org.slf4j.Logger;
import com.baidu.hugegraph.backend.store.AbstractBackendStoreProvider;
import com.baidu.hugegraph.backend.store.BackendStore;
import com.baidu.hugegraph.backend.store.rocksdb.RocksDBStore.RocksDBGraphStore;
import com.baidu.hugegraph.backend.store.rocksdb.RocksDBStore.RocksDBSchemaStore;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.Log;
public class RocksDBStoreProvider extends AbstractBackendStoreProvider {
private static final Logger LOG = Log.logger(RocksDBStore.class);
protected String database() {
return this.name().toLowerCase();
}
@Override
public BackendStore loadSchemaStore(final String name) {
LOG.debug("RocksDBStoreProvider load SchemaStore '{}'", name);
this.checkOpened();
if (!this.stores.containsKey(name)) {
BackendStore s = new RocksDBSchemaStore(this, database(), name);
this.stores.putIfAbsent(name, s);
}
BackendStore store = this.stores.get(name);
E.checkNotNull(store, "store");
E.checkState(store instanceof RocksDBSchemaStore,
"SchemaStore must be an instance of RocksDBSchemaStore");
return store;
}
@Override
public BackendStore loadGraphStore(String name) {
LOG.debug("RocksDBStoreProvider load GraphStore '{}'", name);
this.checkOpened();
if (!this.stores.containsKey(name)) {
BackendStore s = new RocksDBGraphStore(this, database(), name);
this.stores.putIfAbsent(name, s);
}
BackendStore store = this.stores.get(name);
E.checkNotNull(store, "store");
E.checkState(store instanceof RocksDBGraphStore,
"GraphStore must be an instance of RocksDBGraphStore");
return store;
}
@Override
public String type() {
return "rocksdb";
}
}

132
hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java Normal file
View File

@ -0,0 +1,132 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.backend.store.rocksdb;
import java.util.Iterator;
import org.slf4j.Logger;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.query.ConditionQuery;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry;
import com.baidu.hugegraph.backend.serializer.BinarySerializer;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.backend.store.BackendEntry.BackendColumn;
import com.baidu.hugegraph.backend.store.BackendEntryIterator;
import com.baidu.hugegraph.backend.store.rocksdb.RocksDBSessions.Session;
import com.baidu.hugegraph.exception.NotSupportException;
import com.baidu.hugegraph.type.ExtendableIterator;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.util.Log;
import com.google.common.collect.ImmutableList;
public class RocksDBTable {
private static final Logger LOG = Log.logger(RocksDBStore.class);
private final String table;
public RocksDBTable(String database, String table) {
this.table = String.format("%s/%s", database, table);
}
public final String table() {
return this.table;
}
public void insert(Session session, BackendEntry entry) {
assert !entry.columns().isEmpty();
for (BackendColumn col : entry.columns()) {
assert entry.belongToMe(col) : entry;
session.put(this.table, col.name, col.value);
}
}
public void delete(Session session, BackendEntry entry) {
if (entry.columns().isEmpty()) {
session.delete(this.table, entry.id().asBytes());
} else {
for (BackendColumn col : entry.columns()) {
assert entry.belongToMe(col) : entry;
session.remove(this.table, col.name);
}
}
}
public void append(Session session, BackendEntry entry) {
assert entry.columns().size() == 1;
this.insert(session, entry);
}
public void eliminate(Session session, BackendEntry entry) {
assert entry.columns().size() == 1;
this.delete(session, entry);
}
public Iterator<BackendEntry> query(Session session, Query query) {
if (query.limit() == 0 && query.limit() != Query.NO_LIMIT) {
LOG.debug("Return empty result(limit=0) for query {}", query);
return ImmutableList.<BackendEntry>of().iterator();
}
// Query all
if (query.empty()) {
return newEntryIterator(session.scan(this.table), query);
}
// Query by id
if (query.conditions().isEmpty()) {
assert !query.ids().isEmpty();
ExtendableIterator<BackendEntry> rs = new ExtendableIterator<>();
for (Id id : query.ids()) {
rs.extend(newEntryIterator(this.queryById(session, id), query));
}
return rs;
}
// Query by condition (or condition + id)
return this.queryByCond(session, (ConditionQuery) query);
}
protected Iterator<BackendColumn> queryById(Session session, Id id) {
return session.scan(this.table, id.asBytes());
}
protected Iterator<BackendColumn> queryByRange(Session session,
Id begin, Id end) {
return session.scan(this.table, begin.asBytes(), end.asBytes());
}
protected Iterator<BackendEntry> queryByCond(Session session,
ConditionQuery query) {
throw new NotSupportException("query: %s", query);
}
protected static BackendEntryIterator newEntryIterator(
Iterator<BackendColumn> cols,
Query query) {
HugeType t = query.resultType();
return new BackendEntryIterator(cols, query, c ->
// NOTE: only support BinarySerializer currently
new BinaryBackendEntry(t, BinarySerializer.splitIdKey(t, c.name))
);
}
}

292
hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java Normal file
View File

@ -0,0 +1,292 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.backend.store.rocksdb;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Iterator;
import java.util.List;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.id.IdGenerator;
import com.baidu.hugegraph.backend.query.Condition;
import com.baidu.hugegraph.backend.query.Condition.Relation;
import com.baidu.hugegraph.backend.query.ConditionQuery;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.backend.store.BackendEntry.BackendColumn;
import com.baidu.hugegraph.backend.store.rocksdb.RocksDBSessions.Session;
import com.baidu.hugegraph.structure.HugeIndex;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.type.define.HugeKeys;
import com.baidu.hugegraph.util.E;
public class RocksDBTables {
public static class Counters extends RocksDBTable {
public static final String TABLE = "c";
public static final int MAX_TIMES = 1000;
public static final byte[] ONE = b(1L);
public Counters(String database) {
super(database, TABLE);
}
public synchronized Id nextId(Session session, HugeType type) {
byte[] key = new byte[]{type.code()};
// Do get-increase-get-compare operation
long counter = 0L;
long expect = -1L;
for (int i = 0; i < MAX_TIMES; i++) {
// Get the latest value
byte[] value = session.get(this.table(), key);
if (value != null) {
counter = l(value);
}
if (counter == expect) {
break;
}
// Increase local counter
expect = counter + 1;
// Increase 1, the default value of counter is 0 in RocksDB
session.merge(this.table(), key, ONE);
session.commit();
}
E.checkState(counter != 0, "Please check whether RocksDB is OK");
E.checkState(counter == expect, "RocksDB is busy please try again");
return IdGenerator.of(counter);
}
private static byte[] b(long value) {
return ByteBuffer.allocate(Long.BYTES)
.order(ByteOrder.nativeOrder())
.putLong(value).array();
}
private static long l(byte[] bytes) {
assert bytes.length == Long.BYTES;
return ByteBuffer.wrap(bytes)
.order(ByteOrder.nativeOrder())
.getLong();
}
}
public static class VertexLabel extends RocksDBTable {
public static final String TABLE = "vl";
public VertexLabel(String database) {
super(database, TABLE);
}
}
public static class EdgeLabel extends RocksDBTable {
public static final String TABLE = "el";
public EdgeLabel(String database) {
super(database, TABLE);
}
}
public static class PropertyKey extends RocksDBTable {
public static final String TABLE = "pk";
public PropertyKey(String database) {
super(database, TABLE);
}
}
public static class IndexLabel extends RocksDBTable {
public static final String TABLE = "il";
public IndexLabel(String database) {
super(database, TABLE);
}
}
public static class Vertex extends RocksDBTable {
public static final String TABLE = "v";
public Vertex(String database) {
super(database, TABLE);
}
}
public static class Edge extends RocksDBTable {
public static final String TABLE = "e";
public Edge(String database) {
super(database, TABLE);
}
}
public static class SecondaryIndex extends RocksDBTable {
public static final String TABLE = "si";
public SecondaryIndex(String database) {
super(database, TABLE);
}
@Override
protected Iterator<BackendEntry> queryByCond(Session session,
ConditionQuery query) {
E.checkArgument(query.allSysprop() &&
query.conditions().size() == 2,
"There should be two conditions: " +
"INDEX_LABEL_NAME and FIELD_VALUES" +
"in secondary index query");
String index = (String) query.condition(HugeKeys.INDEX_LABEL_NAME);
Object key = query.condition(HugeKeys.FIELD_VALUES);
E.checkArgument(index != null, "Please specify the index label");
E.checkArgument(key != null, "Please specify the index key");
Id id = HugeIndex.formatIndexId(query.resultType(), index, key);
return newEntryIterator(this.queryById(session, id), query);
}
}
// TODO: change to RangeIndex
public static class SearchIndex extends RocksDBTable {
public static final String TABLE = "ri";
public SearchIndex(String database) {
super(database, TABLE);
}
@Override
protected Iterator<BackendEntry> queryByCond(Session session,
ConditionQuery query) {
assert !query.conditions().isEmpty();
String index = (String) query.condition(HugeKeys.INDEX_LABEL_NAME);
Object key = query.condition(HugeKeys.FIELD_VALUES);
E.checkArgument(index != null, "Please specify the index label");
List<? extends Relation> relations = null;
if (key != null) {
final String msg = "Expect one relation in search query";
E.checkArgument(query.conditions().size() == 2, msg);
for (Condition c : query.conditions()) {
if (c.isRelation()) {
key = ((Condition.Relation) c).key();
if (key.equals(HugeKeys.FIELD_VALUES)) {
relations = c.relations();
break;
}
}
}
} else {
// TODO: query by range, like: 18 < age and age < 20
final String msg = "Expect one AND condition in search query";
E.checkArgument(query.conditions().size() == 2, msg);
Condition.And and = null;
for (Condition c : query.conditions()) {
if (c instanceof Condition.And) {
and = (Condition.And) c;
break;
}
}
E.checkArgument(and != null, msg);
E.checkArgument(and.left().isRelation() &&
and.right().isRelation(),
"Expect relations in AND condition");
relations = and.relations();
E.checkArgument(relations.size() == 2,
"Expect 2 relations in AND condition");
}
E.checkArgument(relations != null,
"Expect relations in search query");
Object keyEq = null;
Object keyMin = null;
boolean keyMinEq = false;
Object keyMax = null;
boolean keyMaxEq = false;
for (Relation r : relations) {
E.checkArgument(r.key() == HugeKeys.FIELD_VALUES,
"Expect FIELD_VALUES in AND condition");
switch (r.relation()) {
case EQ:
keyEq = r.value();
break;
case GTE:
keyMinEq = true;
case GT:
keyMin = r.value();
break;
case LTE:
keyMaxEq = true;
case LT:
keyMax = r.value();
break;
default:
E.checkArgument(false, "Unsupported relation '%s'",
r.relation());
}
}
HugeType type = query.resultType();
Iterator<BackendColumn> itor;
if (keyEq != null) {
Id id = HugeIndex.formatIndexId(type, index, keyEq);
itor = queryById(session, id);
} else {
if (keyMin == null) {
keyMin = 0L;
keyMinEq = true;
}
Id min = HugeIndex.formatIndexId(type, index, keyMin);
byte[] begin = min.asBytes();
if (!keyMinEq) {
begin = RocksDBSessions.increase(begin);
}
if (keyMax == null) {
itor = session.scan(table(), begin, null);
} else {
Id max = HugeIndex.formatIndexId(type, index, keyMax);
byte[] end = max.asBytes();
if (keyMaxEq) {
end = RocksDBSessions.increase(end);
}
itor = session.scan(table(), begin, end);
}
}
return newEntryIterator(itor, query);
}
}
}

2
hugegraph-scylladb/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>hugegraph</artifactId>
<groupId>com.baidu.hugegraph</groupId>
<version>0.4.1-SNAPSHOT</version>
<version>0.4.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

4
hugegraph-scylladb/src/main/java/com/baidu/hugegraph/backend/store/scylladb/ScyllaDBFeatures.java
View File

@ -24,12 +24,12 @@ import com.baidu.hugegraph.backend.store.cassandra.CassandraFeatures;
public class ScyllaDBFeatures extends CassandraFeatures {
@Override
public boolean supportsQueryByContains() {
public boolean supportsQueryWithContains() {
return false;
}
@Override
public boolean supportsQueryByContainsKey() {
public boolean supportsQueryWithContainsKey() {
return false;
}
}

11
hugegraph-scylladb/src/main/java/com/baidu/hugegraph/backend/store/scylladb/ScyllaDBTables.java
View File

@ -186,11 +186,12 @@ public class ScyllaDBTables {
}
@Override
public Iterable<BackendEntry> query(
CassandraSessionPool.Session session, Query query) {
public Iterator<BackendEntry> query(
CassandraSessionPool.Session session,
Query query) {
query = queryByLabelIndex(session, LABEL_INDEX_TABLE, query);
if (query == null) {
return ImmutableList.of();
return ImmutableList.<BackendEntry>of().iterator();
}
return super.query(session, query);
}
@ -266,11 +267,11 @@ public class ScyllaDBTables {
}
@Override
public Iterable<BackendEntry> query(
public Iterator<BackendEntry> query(
CassandraSessionPool.Session session, Query query) {
query = queryByLabelIndex(session, LABEL_INDEX_TABLE, query);
if (query == null) {
return ImmutableList.of();
return ImmutableList.<BackendEntry>of().iterator();
}
return super.query(session, query);
}

2
hugegraph-test/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>hugegraph</artifactId>
<groupId>com.baidu.hugegraph</groupId>
<version>0.4.1-SNAPSHOT</version>
<version>0.4.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

44
hugegraph-test/src/main/java/com/baidu/hugegraph/core/BaseCoreTest.java
View File

@ -57,8 +57,16 @@ public class BaseCoreTest {
if (graph == null) {
return;
}
graph.clearBackend();
graph.close();
try {
graph.clearBackend();
} finally {
try {
graph.close();
} catch (Throwable e) {
LOG.error("Error when close()", e);
}
}
}
public HugeGraph graph() {
@ -79,30 +87,18 @@ public class BaseCoreTest {
protected void clearData() {
HugeGraph graph = graph();
try {
// Clear edge
graph().traversal().E().toStream().forEach(e -> {
e.remove();
});
// Clear edge
graph().traversal().E().toStream().forEach(e -> {
e.remove();
});
// Clear vertex
graph().traversal().V().toStream().forEach(v -> {
v.remove();
});
// Clear vertex
graph().traversal().V().toStream().forEach(v -> {
v.remove();
});
// Commit changes
graph.tx().commit();
} finally {
try {
graph.tx().close();
} catch (Throwable e) {
/*
* Ignore exception when close() due to we can't throw
* a new exception which may override the origin one
*/
LOG.error("Error when close tx", e);
}
}
// Commit changes
graph.tx().commit();
}
private void clearSchema() {

27
hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java
View File

@ -612,7 +612,7 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testQueryEdgesWithLimitAndOrderBy() {
Assume.assumeTrue("Not support order by",
storeFeatures().supportsOrderByQuery());
storeFeatures().supportsQueryWithOrderBy());
HugeGraph graph = graph();
init18Edges();
@ -945,7 +945,8 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testScanEdge() {
HugeGraph graph = graph();
Assume.assumeTrue("Not support scan", storeFeatures().supportsScan());
Assume.assumeTrue("Not support scan",
storeFeatures().supportsScanToken());
init18Edges();
Set<Edge> edges = new HashSet<>();
@ -1162,8 +1163,6 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testAddEdgeProperty() {
Assume.assumeTrue("Not support append/eliminate edge property",
storeFeatures().supportsUpdateEdgeProperty());
HugeGraph graph = graph();
Vertex louise = graph.addVertex(T.label, "person", "name", "Louise",
@ -1228,8 +1227,6 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testUpdateEdgeProperty() {
Assume.assumeTrue("Not support append/eliminate edge property",
storeFeatures().supportsUpdateEdgeProperty());
HugeGraph graph = graph();
Edge edge = initEdgeTransfer();
@ -1247,8 +1244,6 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testUpdateEdgePropertyWithRemoveAndSet() {
Assume.assumeTrue("Not support append/eliminate edge property",
storeFeatures().supportsUpdateEdgeProperty());
HugeGraph graph = graph();
Vertex louise = graph.addVertex(T.label, "person", "name", "Louise",
@ -1274,8 +1269,6 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testUpdateEdgePropertyTwice() {
Assume.assumeTrue("Not support append/eliminate edge property",
storeFeatures().supportsUpdateEdgeProperty());
HugeGraph graph = graph();
Edge edge = initEdgeTransfer();
@ -1374,8 +1367,6 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testRemoveEdgeProperty() {
Assume.assumeTrue("Not support append/eliminate edge property",
storeFeatures().supportsUpdateEdgeProperty());
HugeGraph graph = graph();
Edge edge = initEdgeTransfer();
@ -1393,8 +1384,6 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testRemoveEdgePropertyTwice() {
Assume.assumeTrue("Not support append/eliminate edge property",
storeFeatures().supportsUpdateEdgeProperty());
HugeGraph graph = graph();
Edge edge = initEdgeTransfer();
@ -1423,8 +1412,6 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testRemoveEdgePropertyNullableWithIndex() {
Assume.assumeTrue("Not support append/eliminate edge property",
storeFeatures().supportsUpdateEdgeProperty());
HugeGraph graph = graph();
Vertex louise = graph.addVertex(T.label, "person", "name", "Louise",
"city", "Beijing", "age", 21);
@ -1463,8 +1450,6 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testRemoveEdgePropertyNullableWithoutIndex() {
Assume.assumeTrue("Not support append/eliminate edge property",
storeFeatures().supportsUpdateEdgeProperty());
HugeGraph graph = graph();
Vertex louise = graph.addVertex(T.label, "person", "name", "Louise",
"city", "Beijing", "age", 21);
@ -1522,8 +1507,6 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testQueryEdgeBeforeAfterUpdateMultiPropertyWithIndex() {
Assume.assumeTrue("Not support append/eliminate edge property",
storeFeatures().supportsUpdateEdgeProperty());
HugeGraph graph = graph();
Vertex louise = graph.addVertex(T.label, "person", "name", "Louise",
"city", "Beijing", "age", 21);
@ -1563,8 +1546,6 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testQueryEdgeBeforeAfterUpdatePropertyWithSecondaryIndex() {
Assume.assumeTrue("Not support append/eliminate edge property",
storeFeatures().supportsUpdateEdgeProperty());
HugeGraph graph = graph();
Vertex louise = graph.addVertex(T.label, "person", "name", "Louise",
"city", "Beijing", "age", 21);
@ -1594,8 +1575,6 @@ public class EdgeCoreTest extends BaseCoreTest {
@Test
public void testQueryEdgeBeforeAfterUpdatePropertyWithSearchIndex() {
Assume.assumeTrue("Not support append/eliminate edge property",
storeFeatures().supportsUpdateEdgeProperty());
HugeGraph graph = graph();
Vertex louise = graph.addVertex(T.label, "person", "name", "Louise",
"city", "Beijing", "age", 21);

26
hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeLabelCoreTest.java
View File

@ -680,9 +680,9 @@ public class EdgeLabelCoreTest extends SchemaCoreTest {
graph().tx().commit();
List<Edge> edge = graph().traversal().E().hasLabel("write").toList();
Assert.assertNotNull(edge);
Assert.assertEquals(2, edge.size());
List<Edge> edges = graph().traversal().E().hasLabel("write").toList();
Assert.assertNotNull(edges);
Assert.assertEquals(2, edges.size());
schema.edgeLabel("write").remove();
@ -696,7 +696,7 @@ public class EdgeLabelCoreTest extends SchemaCoreTest {
}
@Test
public void testRemoveEdgeLabelWithEdgeWithSearchIndex() {
public void testRemoveEdgeLabelWithEdgeAndSearchIndex() {
super.initPropertyKeys();
SchemaManager schema = graph().schema();
@ -731,10 +731,10 @@ public class EdgeLabelCoreTest extends SchemaCoreTest {
marko.addEdge("write", hadoop, "time", "2014-2-28",
"weight", 0.5);
List<Edge> edge = graph().traversal().E().hasLabel("write")
.has("weight", 0.5).toList();
Assert.assertNotNull(edge);
Assert.assertEquals(1, edge.size());
List<Edge> edges = graph().traversal().E().hasLabel("write")
.has("weight", 0.5).toList();
Assert.assertNotNull(edges);
Assert.assertEquals(1, edges.size());
schema.edgeLabel("write").remove();
@ -752,7 +752,7 @@ public class EdgeLabelCoreTest extends SchemaCoreTest {
}
@Test
public void testRemoveEdgeLabelWithEdgeWithSecondaryIndex() {
public void testRemoveEdgeLabelWithEdgeAndSecondaryIndex() {
super.initPropertyKeys();
SchemaManager schema = graph().schema();
@ -785,10 +785,10 @@ public class EdgeLabelCoreTest extends SchemaCoreTest {
marko.addEdge("write", java, "time", "2016-12-12", "weight", 0.3);
marko.addEdge("write", hadoop, "time", "2014-2-28", "weight", 0.5);
List<Edge> edge = graph().traversal().E().hasLabel("write")
.has("time", "2016-12-12").toList();
Assert.assertNotNull(edge);
Assert.assertEquals(1, edge.size());
List<Edge> edges = graph().traversal().E().hasLabel("write")
.has("time", "2016-12-12").toList();
Assert.assertNotNull(edges);
Assert.assertEquals(1, edges.size());
schema.edgeLabel("write").remove();

2
hugegraph-test/src/main/java/com/baidu/hugegraph/core/IndexLabelCoreTest.java
View File

@ -196,7 +196,7 @@ public class IndexLabelCoreTest extends SchemaCoreTest {
.secondary().by("contribution").create();
Edge edge = graph().traversal().E().hasLabel("authored")
.has("contribution", "test").next();
.has("contribution", "test").next();
Assert.assertNotNull(edge);
}

31
hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java
View File

@ -633,7 +633,7 @@ public class VertexCoreTest extends BaseCoreTest {
HugeGraph graph = graph();
BackendFeatures features = graph.graphTransaction().store().features();
Assume.assumeTrue("Not support CONTAINS_KEY query",
features.supportsQueryByContainsKey());
features.supportsQueryWithContainsKey());
init10Vertices();
// Query vertex by condition (does contain the property name?)
@ -652,7 +652,7 @@ public class VertexCoreTest extends BaseCoreTest {
HugeGraph graph = graph();
BackendFeatures features = graph.graphTransaction().store().features();
Assume.assumeTrue("Not support CONTAINS_KEY query",
features.supportsQueryByContainsKey());
features.supportsQueryWithContainsKey());
init10Vertices();
List<Vertex> vertexes = graph.traversal().V()
@ -691,7 +691,7 @@ public class VertexCoreTest extends BaseCoreTest {
HugeGraph graph = graph();
BackendFeatures features = graph.graphTransaction().store().features();
Assume.assumeTrue("Not support CONTAINS query",
features.supportsQueryByContains());
features.supportsQueryWithContains());
init10Vertices();
List<Vertex> vertexes = graph.traversal().V()
@ -1191,10 +1191,14 @@ public class VertexCoreTest extends BaseCoreTest {
AtomicInteger size = new AtomicInteger(-1);
Thread t = new Thread(() -> {
List<Vertex> vertices = graph.traversal()
.V("person:marko")
.toList();
size.set(vertices.size());
try {
List<Vertex> vertices = graph.traversal()
.V("person:marko")
.toList();
size.set(vertices.size());
} finally {
graph.close();
}
});
t.start();
t.join();
@ -1985,7 +1989,9 @@ public class VertexCoreTest extends BaseCoreTest {
@Test
public void testScanVertex() {
HugeGraph graph = graph();
Assume.assumeTrue("Not support scan", storeFeatures().supportsScan());
// TODO: also support test scan by range
Assume.assumeTrue("Not support scan",
storeFeatures().supportsScanToken());
init10Vertices();
List<Vertex> vertexes = new LinkedList<>();
@ -2005,7 +2011,8 @@ public class VertexCoreTest extends BaseCoreTest {
@Test
public void testScanVertexWithSplitSizeLt1MB() {
HugeGraph graph = graph();
Assume.assumeTrue("Not support scan", storeFeatures().supportsScan());
Assume.assumeTrue("Not support scan",
storeFeatures().supportsScanToken());
init10Vertices();
long splitSize = 1 * 1024 * 1024 - 1;
@ -2018,7 +2025,8 @@ public class VertexCoreTest extends BaseCoreTest {
@Test
public void testScanVertexWithSplitSizeTypeError() {
HugeGraph graph = graph();
Assume.assumeTrue("Not support scan", storeFeatures().supportsScan());
Assume.assumeTrue("Not support scan",
storeFeatures().supportsScanToken());
init10Vertices();
String splitSize = "123456";
@ -2031,7 +2039,8 @@ public class VertexCoreTest extends BaseCoreTest {
@Test
public void testScanVertexWithoutSplitSize() {
HugeGraph graph = graph();
Assume.assumeTrue("Not support scan", storeFeatures().supportsScan());
Assume.assumeTrue("Not support scan",
storeFeatures().supportsScanToken());
init10Vertices();
Assert.assertThrows(IllegalArgumentException.class, () -> {

3
hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexLabelCoreTest.java
View File

@ -630,6 +630,7 @@ public class VertexLabelCoreTest extends SchemaCoreTest {
graph().addVertex(T.label, "person", "name", "marko", "age", 22);
graph().addVertex(T.label, "person", "name", "jerry", "age", 5);
graph().addVertex(T.label, "person", "name", "tom", "age", 8);
graph().tx().commit();
List<Vertex> vertex = graph().traversal().V().hasLabel("person")
.toList();
@ -666,6 +667,7 @@ public class VertexLabelCoreTest extends SchemaCoreTest {
graph().addVertex(T.label, "person", "name", "marko", "age", 22);
graph().addVertex(T.label, "person", "name", "jerry", "age", 5);
graph().addVertex(T.label, "person", "name", "tom", "age", 8);
graph().tx().commit();
List<Vertex> vertex = graph().traversal().V().hasLabel("person")
.has("age", P.inside(4, 10)).toList();
@ -707,6 +709,7 @@ public class VertexLabelCoreTest extends SchemaCoreTest {
"city", "Beijing");
graph().addVertex(T.label, "person", "name", "tom",
"city", "HongKong");
graph().tx().commit();
List<Vertex> vertex = graph().traversal().V().hasLabel("person")
.has("city", "Beijing").toList();

14
hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java
View File

@ -22,13 +22,25 @@ package com.baidu.hugegraph.unit;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import com.baidu.hugegraph.unit.common.CacheManagerTest;
import com.baidu.hugegraph.unit.common.EventHubTest;
import com.baidu.hugegraph.unit.common.RamCacheTest;
import com.baidu.hugegraph.unit.core.BackendMutationTest;
import com.baidu.hugegraph.unit.core.ConditionQueryFlattenTest;
import com.baidu.hugegraph.unit.rocksdb.RocksDBCountersTest;
import com.baidu.hugegraph.unit.rocksdb.RocksDBSessionsTest;
@RunWith(Suite.class)
@Suite.SuiteClasses({
RamCacheTest.class,
CacheManagerTest.class,
EventHubTest.class,
BackendMutationTest.class,
ConditionQueryFlattenTest.class
ConditionQueryFlattenTest.class,
RocksDBSessionsTest.class,
RocksDBCountersTest.class
})
public class UnitTestSuite {
}

3
hugegraph-test/src/main/java/com/baidu/hugegraph/unit/CacheManagerTest.java → hugegraph-test/src/main/java/com/baidu/hugegraph/unit/common/CacheManagerTest.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package com.baidu.hugegraph.unit;
package com.baidu.hugegraph.unit.common;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@ -33,6 +33,7 @@ import com.baidu.hugegraph.backend.cache.RamCache;
import com.baidu.hugegraph.backend.id.IdGenerator;
import com.baidu.hugegraph.testutil.Assert;
import com.baidu.hugegraph.testutil.Whitebox;
import com.baidu.hugegraph.unit.BaseUnitTest;
import com.google.common.collect.ImmutableMap;
public class CacheManagerTest extends BaseUnitTest {

3
hugegraph-test/src/main/java/com/baidu/hugegraph/unit/EventHubTest.java → hugegraph-test/src/main/java/com/baidu/hugegraph/unit/common/EventHubTest.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package com.baidu.hugegraph.unit;
package com.baidu.hugegraph.unit.common;
import java.util.concurrent.atomic.AtomicInteger;
@ -31,6 +31,7 @@ import com.baidu.hugegraph.event.Event;
import com.baidu.hugegraph.event.EventHub;
import com.baidu.hugegraph.event.EventListener;
import com.baidu.hugegraph.testutil.Assert;
import com.baidu.hugegraph.unit.BaseUnitTest;
import com.google.common.collect.ImmutableList;
public class EventHubTest extends BaseUnitTest {

3
hugegraph-test/src/main/java/com/baidu/hugegraph/unit/RamCacheTest.java → hugegraph-test/src/main/java/com/baidu/hugegraph/unit/common/RamCacheTest.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package com.baidu.hugegraph.unit;
package com.baidu.hugegraph.unit.common;
import java.util.HashMap;
import java.util.Map;
@ -30,6 +30,7 @@ import org.junit.Test;
import com.baidu.hugegraph.backend.cache.RamCache;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.id.IdGenerator;
import com.baidu.hugegraph.unit.BaseUnitTest;
public class RamCacheTest extends BaseUnitTest {

5
hugegraph-test/src/main/java/com/baidu/hugegraph/unit/BackendMutationTest.java → hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/BackendMutationTest.java
View File

@ -1,4 +1,4 @@
package com.baidu.hugegraph.unit;
package com.baidu.hugegraph.unit.core;
import java.util.List;
@ -15,6 +15,7 @@ import com.baidu.hugegraph.backend.store.BackendMutation;
import com.baidu.hugegraph.backend.store.MutateAction;
import com.baidu.hugegraph.backend.store.MutateItem;
import com.baidu.hugegraph.testutil.Assert;
import com.baidu.hugegraph.unit.BaseUnitTest;
public class BackendMutationTest extends BaseUnitTest {
@ -319,7 +320,7 @@ public class BackendMutationTest extends BaseUnitTest {
private static BackendEntry constructBackendEntry(String id,
String... columns) {
assert (columns.length == 0 || columns.length == 2);
TextBackendEntry entry = new TextBackendEntry(IdGenerator.of(id));
TextBackendEntry entry = new TextBackendEntry(null, IdGenerator.of(id));
if (columns.length == 2) {
entry.subId(SplicingIdGenerator.concat(id, columns[0]));
}

3
hugegraph-test/src/main/java/com/baidu/hugegraph/unit/ConditionQueryFlattenTest.java → hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/ConditionQueryFlattenTest.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package com.baidu.hugegraph.unit;
package com.baidu.hugegraph.unit.core;
import java.util.HashSet;
import java.util.List;
@ -31,6 +31,7 @@ import com.baidu.hugegraph.backend.query.Condition;
import com.baidu.hugegraph.backend.query.ConditionQuery;
import com.baidu.hugegraph.testutil.Assert;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.unit.BaseUnitTest;
import com.google.common.collect.ImmutableSet;
public class ConditionQueryFlattenTest extends BaseUnitTest {

117
hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.unit.rocksdb;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.rocksdb.RocksDBException;
import com.baidu.hugegraph.backend.store.rocksdb.RocksDBSessions;
import com.baidu.hugegraph.unit.BaseUnitTest;
public class BaseRocksDBUnitTest extends BaseUnitTest {
private static final String TMP_DIR = System.getProperty("java.io.tmpdir");
private static final String DB_PATH = TMP_DIR + "/" + "rocksdb";
protected static final String TABLE = "test-table";
protected RocksDBSessions rocks;
@AfterClass
public static void clear() throws IOException {
FileUtils.forceDelete(DB_PATH);
}
@Before
public void setup() throws RocksDBException {
this.rocks = open(TABLE);
this.rocks.session();
}
@After
public void teardown() throws RocksDBException {
this.clearData();
close(this.rocks);
}
protected void put(String key, String value) {
this.rocks.session().put(TABLE, b(key), b(value));
this.commit();
}
protected String get(String key) throws RocksDBException {
return s(this.rocks.session().get(TABLE, b(key)));
}
protected void clearData() throws RocksDBException {
for (String table : new ArrayList<>(this.rocks.openedTables())) {
this.rocks.session().delete(table, new byte[]{0}, new byte[]{-1});
}
this.commit();
}
protected void commit() {
try {
this.rocks.session().commit();
} finally {
this.rocks.session().clear();
}
}
protected static byte[] b(String str) {
return str.getBytes();
}
protected static String s(byte[] bytes) {
return bytes == null ? null : new String(bytes);
}
protected static byte[] b(long val) {
ByteBuffer buf = ByteBuffer.allocate(8).order(ByteOrder.nativeOrder());
buf.putLong(val);
return buf.array();
}
protected static long l(byte[] bytes) {
ByteBuffer buf = ByteBuffer.wrap(bytes).order(ByteOrder.nativeOrder());
return buf.getLong();
}
private static RocksDBSessions open(String table) throws RocksDBException {
RocksDBSessions rocks = new RocksDBSessions(DB_PATH, DB_PATH);
rocks.createTable(table);
return rocks;
}
private static void close(RocksDBSessions rocks) throws RocksDBException {
for (String table : new ArrayList<>(rocks.openedTables())) {
rocks.dropTable(table);
}
rocks.close();
}
}

99
hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/RocksDBCountersTest.java Normal file
View File

@ -0,0 +1,99 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.unit.rocksdb;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.Before;
import org.junit.Test;
import org.rocksdb.RocksDBException;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.store.rocksdb.RocksDBSessions.Session;
import com.baidu.hugegraph.backend.store.rocksdb.RocksDBTables;
import com.baidu.hugegraph.testutil.Assert;
import com.baidu.hugegraph.type.HugeType;
public class RocksDBCountersTest extends BaseRocksDBUnitTest {
private static final String DATABASE = "test-db";
private static final int THREADS_NUM = 8;
private RocksDBTables.Counters counters;
@Override
@Before
public void setup() throws RocksDBException {
super.setup();
this.counters = new RocksDBTables.Counters(DATABASE);
this.rocks.createTable(this.counters.table());
}
@Test
public void testCounter() throws RocksDBException {
Session session = this.rocks.session();
for (int i = 1; i < 10000; i++) {
Id id = this.counters.nextId(session, HugeType.PROPERTY_KEY);
Assert.assertEquals(i, id.asLong());
}
}
@Test
public void testCounterWithMultiTypes() throws RocksDBException {
Session session = this.rocks.session();
for (int i = 1; i < 1000; i++) {
Id id = this.counters.nextId(session, HugeType.PROPERTY_KEY);
Assert.assertEquals(i, id.asLong());
id = this.counters.nextId(session, HugeType.VERTEX_LABEL);
Assert.assertEquals(i, id.asLong());
id = this.counters.nextId(session, HugeType.EDGE_LABEL);
Assert.assertEquals(i, id.asLong());
id = this.counters.nextId(session, HugeType.INDEX_LABEL);
Assert.assertEquals(i, id.asLong());
}
}
@Test
public void testCounterWithMutiThreads() {
final int TIMES = 1000;
AtomicLong times = new AtomicLong(0);
Map<Id, Boolean> ids = new ConcurrentHashMap<>();
runWithThreads(THREADS_NUM, () -> {
Session session = this.rocks.session();
for (int i = 0; i < TIMES; i++) {
Id id = this.counters.nextId(session, HugeType.PROPERTY_KEY);
Assert.assertFalse(ids.containsKey(id));
ids.put(id, true);
times.incrementAndGet();
}
this.rocks.close();
});
Assert.assertEquals(THREADS_NUM * TIMES, times.get());
}
}

118
hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/RocksDBPerfTest.java Normal file
View File

@ -0,0 +1,118 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.unit.rocksdb;
import java.util.Iterator;
import org.junit.Test;
import org.rocksdb.RocksDBException;
import com.baidu.hugegraph.backend.store.BackendEntry.BackendColumn;
import com.baidu.hugegraph.backend.store.rocksdb.RocksDBSessions.Session;
public class RocksDBPerfTest extends BaseRocksDBUnitTest {
private static final int TIMES = 10000 * 1000;
@Test
public void testPut() throws RocksDBException {
for (int i = 0; i < TIMES; i++) {
put("person-" + i, "value-" + i);
}
}
@Test
public void testGet3Keys() throws RocksDBException {
put("person:1gname", "James");
put("person:1gage", "19");
put("person:1gcity", "Beijing");
put("person:2gname", "Lisa");
put("person:2gage", "20");
put("person:2gcity", "Beijing");
Session session = this.rocks.session();
for (int i = 0; i < TIMES; i++) {
s(session.get(TABLE, b("person:1gname")));
s(session.get(TABLE, b("person:1gage")));
s(session.get(TABLE, b("person:1gcity")));
}
}
@Test
public void testGet1KeyWithMultiValues() throws RocksDBException {
put("person:1gname", "James");
put("person:1gage", "19");
put("person:1gcity", "Beijing");
put("person:2gname", "Lisa");
put("person:2gage", "20");
put("person:2gcity", "Beijing");
put("person:2all", "name=Lisa,age=20,city=Beijing");
Session session = this.rocks.session();
for (int i = 0; i < TIMES; i++) {
s(session.get(TABLE, b("person:2all")));
}
}
@Test
public void testScanByPrefix() throws RocksDBException {
put("person:1gname", "James");
put("person:1gage", "19");
put("person:1gcity", "Beijing");
put("person:2gname", "Lisa");
put("person:2gage", "20");
put("person:2gcity", "Beijing");
Session session = this.rocks.session();
for (int i = 0; i < TIMES; i++) {
Iterator<BackendColumn> itor = session.scan(TABLE, b("person:1"));
while (itor.hasNext()) {
BackendColumn col = itor.next();
s(col.name);
s(col.value);
}
}
}
@Test
public void testGet3KeysWithData() throws RocksDBException {
testPut();
testGet3Keys();
}
@Test
public void testGet1KeyWithData() throws RocksDBException {
testPut();
testGet1KeyWithMultiValues();
}
@Test
public void testScanByPrefixWithData() throws RocksDBException {
testPut();
testScanByPrefix();
}
}

458
hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/RocksDBSessionsTest.java Normal file
View File

@ -0,0 +1,458 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.unit.rocksdb;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import org.junit.Assume;
import org.junit.Test;
import org.rocksdb.RocksDBException;
import com.baidu.hugegraph.backend.store.BackendEntry.BackendColumn;
import com.baidu.hugegraph.backend.store.rocksdb.RocksDBSessions.Session;
import com.baidu.hugegraph.testutil.Assert;
public class RocksDBSessionsTest extends BaseRocksDBUnitTest {
@Test
public void testPutGet() throws RocksDBException {
String value = s(this.rocks.session().get(TABLE, b("person:1gname")));
Assert.assertEquals(null, value);
this.rocks.session().put(TABLE, b("person:1gname"), b("James"));
this.rocks.session().put(TABLE, b("person:1gage"), b("19"));
this.rocks.session().put(TABLE, b("person:1gcity"), b("Beijing"));
this.commit();
value = s(this.rocks.session().get(TABLE, b("person:1gname")));
Assert.assertEquals("James", value);
value = s(this.rocks.session().get(TABLE, b("person:1gage")));
Assert.assertEquals("19", value);
value = s(this.rocks.session().get(TABLE, b("person:1gcity")));
Assert.assertEquals("Beijing", value);
}
@Test
public void testPutGetWithMultiTables() throws RocksDBException {
final String TABLE2 = "test-table2";
this.rocks.createTable(TABLE2);
this.rocks.session().put(TABLE, b("person:1gname"), b("James"));
this.rocks.session().put(TABLE2, b("person:1gname"), b("James2"));
this.commit();
String value = s(this.rocks.session().get(TABLE, b("person:1gname")));
Assert.assertEquals("James", value);
String value2 = s(this.rocks.session().get(TABLE2, b("person:1gname")));
Assert.assertEquals("James2", value2);
}
@Test
public void testMergeWithCounter() throws RocksDBException {
this.rocks.session().put(TABLE, b("person:1gage"), b(19));
this.commit();
this.rocks.session().merge(TABLE, b("person:1gage"), b(1));
this.commit();
byte[] value = this.rocks.session().get(TABLE, b("person:1gage"));
Assert.assertEquals(20L, l(value));
this.rocks.session().merge(TABLE, b("person:1gage"), b(123456789000L));
this.commit();
value = this.rocks.session().get(TABLE, b("person:1gage"));
Assert.assertEquals(123456789020L, l(value));
this.rocks.session().put(TABLE, b("person:1gage"), b(250));
this.commit();
this.rocks.session().merge(TABLE, b("person:1gage"), b(10));
this.commit();
value = this.rocks.session().get(TABLE, b("person:1gage"));
Assert.assertEquals(260L, l(value));
}
@Test
public void testMergeWithStringList() throws RocksDBException {
Assume.assumeTrue("Not support string append now", false);
this.rocks.session().put(TABLE, b("person:1gphoneno"), b("12306"));
this.commit();
this.rocks.session().merge(TABLE, b("person:1gphoneno"), b("12315"));
this.commit();
Assert.assertEquals("12306,12315", get("person:1gphoneno"));
}
@Test
public void testScanByPrefix() throws RocksDBException {
put("person:1gname", "James");
put("person:1gage", "19");
put("person:1gcity", "Beijing");
put("person:2gname", "Lisa");
put("person:2gage", "20");
put("person:2gcity", "Beijing");
Map<String, String> results = new HashMap<>();
Session session = this.rocks.session();
Iterator<BackendColumn> itor = session.scan(TABLE, b("person:1"));
while (itor.hasNext()) {
BackendColumn col = itor.next();
results.put(s(col.name), s(col.value));
}
Assert.assertEquals(3, results.size());
Assert.assertEquals("James", results.get("person:1gname"));
Assert.assertEquals("19", results.get("person:1gage"));
Assert.assertEquals("Beijing", results.get("person:1gcity"));
Assert.assertEquals("Lisa", get("person:2gname"));
}
@Test
public void testScanByRange() throws RocksDBException {
put("person:1gname", "James");
put("person:1gage", "19");
put("person:1gcity", "Beijing");
put("person:2gname", "Lisa");
put("person:2gage", "20");
put("person:2gcity", "Beijing");
put("person:3gname", "Hebe");
put("person:3gage", "21");
put("person:3gcity", "Taipei");
Map<String, String> results = new HashMap<>();
Session session = this.rocks.session();
Iterator<BackendColumn> itor = session.scan(TABLE,
b("person:1"),
b("person:3"));
while (itor.hasNext()) {
BackendColumn col = itor.next();
results.put(s(col.name), s(col.value));
}
Assert.assertEquals(6, results.size());
Assert.assertEquals("James", results.get("person:1gname"));
Assert.assertEquals("19", results.get("person:1gage"));
Assert.assertEquals("Beijing", results.get("person:1gcity"));
Assert.assertEquals("Lisa", results.get("person:2gname"));
Assert.assertEquals("20", results.get("person:2gage"));
Assert.assertEquals("Beijing", results.get("person:2gcity"));
Assert.assertEquals("Hebe", get("person:3gname"));
}
@Test
public void testScanByRangeWithBytes() throws RocksDBException {
Session session = this.rocks.session();
byte[] key11 = new byte[]{1, 1};
byte[] value11 = b("value-1-1");
session.put(TABLE, key11, value11);
byte[] key12 = new byte[]{1, 2};
byte[] value12 = b("value-1-2");
session.put(TABLE, key12, value12);
byte[] key21 = new byte[]{2, 1};
byte[] value21 = b("value-2-1");
session.put(TABLE, key21, value21);
this.commit();
Map<ByteBuffer, byte[]> results = new HashMap<>();
Iterator<BackendColumn> itor = session.scan(TABLE,
new byte[]{1, 0},
new byte[]{1, 3});
while (itor.hasNext()) {
BackendColumn col = itor.next();
results.put(ByteBuffer.wrap(col.name), col.value);
}
Assert.assertEquals(2, results.size());
Assert.assertArrayEquals(value11, results.get(ByteBuffer.wrap(key11)));
Assert.assertArrayEquals(value12, results.get(ByteBuffer.wrap(key12)));
Assert.assertArrayEquals(value21, session.get(TABLE, key21));
}
@Test
public void testScanByRangeWithSignedBytes() throws RocksDBException {
Session session = this.rocks.session();
byte[] key11 = new byte[]{1, 1};
byte[] value11 = b("value-1-1");
session.put(TABLE, key11, value11);
byte[] key12 = new byte[]{1, 2};
byte[] value12 = b("value-1-2");
session.put(TABLE, key12, value12);
byte[] key13 = new byte[]{1, -3};
byte[] value13 = b("value-1-3");
session.put(TABLE, key13, value13);
byte[] key21 = new byte[]{2, 1};
byte[] value21 = b("value-2-1");
session.put(TABLE, key21, value21);
this.commit();
Iterator<BackendColumn> itor;
itor = session.scan(TABLE, new byte[]{1, -1}, new byte[]{1, 3});
Assert.assertFalse(itor.hasNext());
itor = session.scan(TABLE, new byte[]{1, 1}, new byte[]{1, -1});
Map<ByteBuffer, byte[]> results = new HashMap<>();
while (itor.hasNext()) {
BackendColumn col = itor.next();
results.put(ByteBuffer.wrap(col.name), col.value);
}
Assert.assertEquals(3, results.size());
Assert.assertArrayEquals(value11, results.get(ByteBuffer.wrap(key11)));
Assert.assertArrayEquals(value12, results.get(ByteBuffer.wrap(key12)));
Assert.assertArrayEquals(value13, results.get(ByteBuffer.wrap(key13)));
Assert.assertArrayEquals(value21, session.get(TABLE, key21));
}
@Test
public void testUpdate() throws RocksDBException {
put("person:1gname", "James");
put("person:1gage", "19");
put("person:1gcity", "Beijing");
Assert.assertEquals("James", get("person:1gname"));
Assert.assertEquals("19", get("person:1gage"));
Assert.assertEquals("Beijing", get("person:1gcity"));
put("person:1gage", "20");
Assert.assertEquals("James", get("person:1gname"));
Assert.assertEquals("20", get("person:1gage"));
Assert.assertEquals("Beijing", get("person:1gcity"));
}
@Test
public void testDeleteByKey() throws RocksDBException {
put("person:1gname", "James");
put("person:1gage", "19");
put("person:1gcity", "Beijing");
Assert.assertEquals("James", get("person:1gname"));
Assert.assertEquals("19", get("person:1gage"));
Assert.assertEquals("Beijing", get("person:1gcity"));
this.rocks.session().remove(TABLE, b("person:1gage"));
this.commit();
Assert.assertEquals("James", get("person:1gname"));
Assert.assertEquals(null, get("person:1gage"));
Assert.assertEquals("Beijing", get("person:1gcity"));
}
@Test
public void testDeleteByKeyButNotExist() throws RocksDBException {
put("person:1gname", "James");
put("person:1gage", "19");
put("person:1gcity", "Beijing");
Assert.assertEquals("James", get("person:1gname"));
Assert.assertEquals("19", get("person:1gage"));
Assert.assertEquals("Beijing", get("person:1gcity"));
this.rocks.session().remove(TABLE, b("person:1"));
Assert.assertEquals("James", get("person:1gname"));
Assert.assertEquals("19", get("person:1gage"));
Assert.assertEquals("Beijing", get("person:1gcity"));
}
@Test
public void testDeleteByPrefix() throws RocksDBException {
put("person:1gname", "James");
put("person:1gage", "19");
put("person:1gcity", "Beijing");
put("person:2gname", "Lisa");
put("person:2gage", "20");
put("person:2gcity", "Beijing");
Assert.assertEquals("James", get("person:1gname"));
Assert.assertEquals("19", get("person:1gage"));
Assert.assertEquals("Beijing", get("person:1gcity"));
this.rocks.session().delete(TABLE, b("person:1"));
this.commit();
Assert.assertEquals(null, get("person:1gname"));
Assert.assertEquals(null, get("person:1gage"));
Assert.assertEquals(null, get("person:1gcity"));
Assert.assertEquals("Lisa", get("person:2gname"));
}
@Test
public void testDeleteByRange() throws RocksDBException {
put("person:1gname", "James");
put("person:1gage", "19");
put("person:1gcity", "Beijing");
put("person:2gname", "Lisa");
put("person:2gage", "20");
put("person:2gcity", "Beijing");
put("person:3gname", "Hebe");
put("person:3gage", "21");
put("person:3gcity", "Taipei");
Assert.assertEquals("James", get("person:1gname"));
Assert.assertEquals("Lisa", get("person:2gname"));
Assert.assertEquals("Hebe", get("person:3gname"));
this.rocks.session().delete(TABLE, b("person:1"), b("person:3"));
this.commit();
Assert.assertEquals(null, get("person:1gname"));
Assert.assertEquals(null, get("person:1gage"));
Assert.assertEquals(null, get("person:1gcity"));
Assert.assertEquals(null, get("person:2gname"));
Assert.assertEquals(null, get("person:2gage"));
Assert.assertEquals(null, get("person:2gcity"));
Assert.assertEquals("Hebe", get("person:3gname"));
Assert.assertEquals("21", get("person:3gage"));
Assert.assertEquals("Taipei", get("person:3gcity"));
}
@Test
public void testDeleteByRangeWithBytes() throws RocksDBException {
Session session = this.rocks.session();
byte[] key11 = new byte[]{1, 1};
byte[] value11 = b("value-1-1");
session.put(TABLE, key11, value11);
byte[] key12 = new byte[]{1, 2};
byte[] value12 = b("value-1-2");
session.put(TABLE, key12, value12);
byte[] key21 = new byte[]{2, 1};
byte[] value21 = b("value-2-1");
session.put(TABLE, key21, value21);
session.delete(TABLE, key11, new byte[]{1, 3});
this.commit();
Assert.assertArrayEquals(null, session.get(TABLE, key11));
Assert.assertArrayEquals(null, session.get(TABLE, key12));
Assert.assertArrayEquals(value21, session.get(TABLE, key21));
}
@Test
public void testDeleteByRangeWithSignedBytes() throws RocksDBException {
Session session = this.rocks.session();
byte[] key11 = new byte[]{1, 1};
byte[] value11 = b("value-1-1");
session.put(TABLE, key11, value11);
byte[] key12 = new byte[]{1, -2};
byte[] value12 = b("value-1-2");
session.put(TABLE, key12, value12);
byte[] key21 = new byte[]{2, 1};
byte[] value21 = b("value-2-1");
session.put(TABLE, key21, value21);
session.delete(TABLE, new byte[]{1, -3}, new byte[]{1, 3});
this.commit();
Assert.assertArrayEquals(value11, session.get(TABLE, key11));
Assert.assertArrayEquals(value12, session.get(TABLE, key12));
Assert.assertArrayEquals(value21, session.get(TABLE, key21));
session.delete(TABLE, new byte[]{1, 1}, new byte[]{1, -1});
this.commit();
Assert.assertArrayEquals(null, session.get(TABLE, key11));
Assert.assertArrayEquals(null, session.get(TABLE, key12));
Assert.assertArrayEquals(value21, session.get(TABLE, key21));
}
@Test
public void testDeleteByRangeWithMinMaxByteValue() throws RocksDBException {
Session session = this.rocks.session();
byte[] key11 = new byte[]{1, 0};
byte[] value11 = b("value-1-1");
session.put(TABLE, key11, value11);
byte[] key12 = new byte[]{1, 127};
byte[] value12 = b("value-1-2");
session.put(TABLE, key12, value12);
byte[] key13 = new byte[]{1, (byte) 0x80}; // 128
byte[] value13 = b("value-1-3");
session.put(TABLE, key13, value13);
byte[] key14 = new byte[]{1, (byte) 0xff}; // 255
byte[] value14 = b("value-1-4");
session.put(TABLE, key14, value14);
byte[] key20 = new byte[]{2, 0};
byte[] value20 = b("value-2-0");
session.put(TABLE, key20, value20);
session.delete(TABLE, new byte[]{1, 0}, new byte[]{1, (byte) 0xff});
this.commit();
Assert.assertArrayEquals(null, session.get(TABLE, key11));
Assert.assertArrayEquals(null, session.get(TABLE, key12));
Assert.assertArrayEquals(null, session.get(TABLE, key13));
Assert.assertArrayEquals(value14, session.get(TABLE, key14));
Assert.assertArrayEquals(value20, session.get(TABLE, key20));
session.delete(TABLE, new byte[]{1, (byte) 0xff}, new byte[]{2, 0});
this.commit();
Assert.assertArrayEquals(null, session.get(TABLE, key11));
Assert.assertArrayEquals(null, session.get(TABLE, key12));
Assert.assertArrayEquals(null, session.get(TABLE, key13));
Assert.assertArrayEquals(null, session.get(TABLE, key14));
Assert.assertArrayEquals(value20, session.get(TABLE, key20));
}
}

1
hugegraph-test/src/main/resources/hugegraph.properties
View File

@ -6,7 +6,6 @@ serializer=cassandra
store=hugegraph
store.schema=huge_schema
store.graph=huge_graph
store.index=huge_index
# cassandra backend config
cassandra.host=localhost

7
pom.xml
View File

@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph</artifactId>
<version>0.4.1-SNAPSHOT</version>
<version>0.4.2-SNAPSHOT</version>
<packaging>pom</packaging>
<prerequisites>
<maven>3.3.9</maven>
@ -96,12 +96,13 @@
</properties>
<modules>
<module>hugegraph-core</module>
<module>hugegraph-cassandra</module>
<module>hugegraph-dist</module>
<module>hugegraph-api</module>
<module>hugegraph-example</module>
<module>hugegraph-dist</module>
<module>hugegraph-test</module>
<module>hugegraph-cassandra</module>
<module>hugegraph-scylladb</module>
<module>hugegraph-rocksdb</module>
</modules>
<dependencyManagement>