From 3f2edb0d9a3ff0df85a461c864bf69148e539618 Mon Sep 17 00:00:00 2001 From: Tsukilc <153273766+Tsukilc@users.noreply.github.com> Date: Wed, 3 Sep 2025 15:26:45 +0800 Subject: [PATCH] feat(hugegraph-struct): initialize module with core type interfaces and project configuration --- .licenserc.yaml | 2 + LICENSE | 2 + hugegraph-struct/pom.xml | 197 +++ .../apache/hugegraph/HugeGraphSupplier.java | 79 ++ .../org/apache/hugegraph/SchemaDriver.java | 860 ++++++++++++ .../org/apache/hugegraph/SchemaGraph.java | 182 +++ .../apache/hugegraph/analyzer/Analyzer.java | 27 + .../hugegraph/analyzer/AnalyzerFactory.java | 102 ++ .../hugegraph/analyzer/AnsjAnalyzer.java | 87 ++ .../hugegraph/analyzer/HanLPAnalyzer.java | 108 ++ .../apache/hugegraph/analyzer/IKAnalyzer.java | 73 + .../hugegraph/analyzer/JcsegAnalyzer.java | 77 ++ .../hugegraph/analyzer/JiebaAnalyzer.java | 63 + .../hugegraph/analyzer/MMSeg4JAnalyzer.java | 92 ++ .../hugegraph/analyzer/SmartCNAnalyzer.java | 66 + .../hugegraph/analyzer/WordAnalyzer.java | 74 + .../apache/hugegraph/auth/AuthConstant.java | 30 + .../apache/hugegraph/auth/TokenGenerator.java | 70 + .../hugegraph/backend/BackendColumn.java | 69 + .../apache/hugegraph/backend/BinaryId.java | 103 ++ .../org/apache/hugegraph/backend/Shard.java | 71 + .../hugegraph/exception/BackendException.java | 53 + .../exception/ErrorCodeProvider.java | 27 + .../hugegraph/exception/HugeException.java | 70 + .../exception/LimitExceedException.java | 33 + .../exception/NotAllowException.java | 33 + .../exception/NotFoundException.java | 37 + .../exception/NotSupportException.java | 34 + .../java/org/apache/hugegraph/id/EdgeId.java | 350 +++++ .../main/java/org/apache/hugegraph/id/Id.java | 90 ++ .../org/apache/hugegraph/id/IdGenerator.java | 465 +++++++ .../java/org/apache/hugegraph/id/IdUtil.java | 162 +++ .../hugegraph/id/SplicingIdGenerator.java | 150 ++ .../apache/hugegraph/options/AuthOptions.java | 153 +++ .../apache/hugegraph/options/CoreOptions.java | 666 +++++++++ .../org/apache/hugegraph/query/Aggregate.java | 61 + .../hugegraph/query/AggregateFuncDefine.java | 37 + .../org/apache/hugegraph/query/Condition.java | 1040 ++++++++++++++ .../hugegraph/query/ConditionQuery.java | 1217 +++++++++++++++++ .../org/apache/hugegraph/query/IdQuery.java | 127 ++ .../apache/hugegraph/query/MatchedIndex.java | 81 ++ .../org/apache/hugegraph/query/Query.java | 720 ++++++++++ .../serializer/AbstractSerializerAdapter.java | 62 + .../query/serializer/QueryAdapter.java | 148 ++ .../query/serializer/QueryIdAdapter.java | 46 + .../apache/hugegraph/schema/EdgeLabel.java | 449 ++++++ .../apache/hugegraph/schema/IndexLabel.java | 498 +++++++ .../apache/hugegraph/schema/PropertyKey.java | 646 +++++++++ .../hugegraph/schema/SchemaElement.java | 259 ++++ .../apache/hugegraph/schema/SchemaLabel.java | 204 +++ .../org/apache/hugegraph/schema/Userdata.java | 64 + .../apache/hugegraph/schema/VertexLabel.java | 414 ++++++ .../schema/builder/SchemaBuilder.java | 42 + .../serializer/BinaryElementSerializer.java | 528 +++++++ .../hugegraph/serializer/BytesBuffer.java | 1012 ++++++++++++++ .../serializer/DirectBinarySerializer.java | 128 ++ .../apache/hugegraph/structure/BaseEdge.java | 288 ++++ .../hugegraph/structure/BaseElement.java | 355 +++++ .../hugegraph/structure/BaseProperty.java | 68 + .../hugegraph/structure/BaseRawElement.java | 57 + .../hugegraph/structure/BaseVertex.java | 168 +++ .../org/apache/hugegraph/structure/Index.java | 334 +++++ .../apache/hugegraph/structure/KvElement.java | 101 ++ .../structure/builder/IndexBuilder.java | 327 +++++ .../org/apache/hugegraph/type/GraphType.java | 23 + .../org/apache/hugegraph/type/HugeType.java | 213 +++ .../org/apache/hugegraph/type/Idfiable.java | 27 + .../apache/hugegraph/type/Indexfiable.java | 29 + .../org/apache/hugegraph/type/Namifiable.java | 31 + .../org/apache/hugegraph/type/Propfiable.java | 29 + .../org/apache/hugegraph/type/Typifiable.java | 26 + .../apache/hugegraph/type/define/Action.java | 76 + .../hugegraph/type/define/AggregateType.java | 93 ++ .../hugegraph/type/define/Cardinality.java | 69 + .../hugegraph/type/define/CollectionType.java | 68 + .../hugegraph/type/define/DataType.java | 224 +++ .../hugegraph/type/define/Directions.java | 89 ++ .../hugegraph/type/define/EdgeLabelType.java | 72 + .../hugegraph/type/define/Frequency.java | 51 + .../hugegraph/type/define/HugeKeys.java | 108 ++ .../hugegraph/type/define/IdStrategy.java | 71 + .../hugegraph/type/define/IndexType.java | 122 ++ .../hugegraph/type/define/SchemaStatus.java | 67 + .../hugegraph/type/define/SerialEnum.java | 83 ++ .../hugegraph/type/define/WriteType.java | 67 + .../java/org/apache/hugegraph/util/Blob.java | 73 + .../org/apache/hugegraph/util/GraphUtils.java | 34 + .../org/apache/hugegraph/util/LZ4Util.java | 95 ++ .../apache/hugegraph/util/StringEncoding.java | 203 +++ .../util/collection/CollectionFactory.java | 264 ++++ .../hugegraph/util/collection/IdSet.java | 120 ++ pom.xml | 25 +- 92 files changed, 16682 insertions(+), 8 deletions(-) create mode 100644 hugegraph-struct/pom.xml create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/HugeGraphSupplier.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/SchemaDriver.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/SchemaGraph.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/Analyzer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/AnalyzerFactory.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/AnsjAnalyzer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/HanLPAnalyzer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/IKAnalyzer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/JcsegAnalyzer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/JiebaAnalyzer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/MMSeg4JAnalyzer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/SmartCNAnalyzer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/WordAnalyzer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/auth/AuthConstant.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/auth/TokenGenerator.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/backend/BackendColumn.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/backend/BinaryId.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/backend/Shard.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/exception/BackendException.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/exception/ErrorCodeProvider.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/exception/HugeException.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/exception/LimitExceedException.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotAllowException.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotFoundException.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotSupportException.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/id/EdgeId.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/id/Id.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/id/IdGenerator.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/id/IdUtil.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/id/SplicingIdGenerator.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/options/AuthOptions.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/options/CoreOptions.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/query/Aggregate.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/query/AggregateFuncDefine.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/query/Condition.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/query/ConditionQuery.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/query/IdQuery.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/query/MatchedIndex.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/query/Query.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/AbstractSerializerAdapter.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryAdapter.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryIdAdapter.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/schema/EdgeLabel.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/schema/IndexLabel.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/schema/PropertyKey.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/schema/SchemaElement.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/schema/SchemaLabel.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/schema/Userdata.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/schema/VertexLabel.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/schema/builder/SchemaBuilder.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BinaryElementSerializer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BytesBuffer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/DirectBinarySerializer.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseEdge.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseElement.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseProperty.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseRawElement.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseVertex.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/structure/Index.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/structure/KvElement.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/structure/builder/IndexBuilder.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/GraphType.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/HugeType.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/Idfiable.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/Indexfiable.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/Namifiable.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/Propfiable.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/Typifiable.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Action.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/AggregateType.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Cardinality.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/CollectionType.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/DataType.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Directions.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/EdgeLabelType.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Frequency.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/HugeKeys.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/IdStrategy.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/IndexType.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/SchemaStatus.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/SerialEnum.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/WriteType.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/util/Blob.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/util/GraphUtils.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/util/LZ4Util.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/util/StringEncoding.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/util/collection/CollectionFactory.java create mode 100644 hugegraph-struct/src/main/java/org/apache/hugegraph/util/collection/IdSet.java diff --git a/.licenserc.yaml b/.licenserc.yaml index 573ba55c4..3ebf89162 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -100,6 +100,8 @@ header: # `header` section is configurations for source codes license header. - 'hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/StructureBasicSuite.java' - 'hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherOpProcessor.java' - 'hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherPlugin.java' + - 'hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Cardinality.java' + - 'hugegraph-struct/src/main/java/org/apache/hugegraph/type/Namifiable.java' comment: on-failure # on what condition license-eye will comment on the pull request, `on-failure`, `always`, `never`. # license-location-threshold specifies the index threshold where the license header can be located, diff --git a/LICENSE b/LICENSE index c8b9d6ed0..8445ec58d 100644 --- a/LICENSE +++ b/LICENSE @@ -216,3 +216,5 @@ hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/type/define/C hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/StringEncoding.java from https://github.com/JanusGraph/janusgraph hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherOpProcessor.java from https://github.com/opencypher/cypher-for-gremlin hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherPlugin.java from https://github.com/opencypher/cypher-for-gremlin +hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Cardinality.java from https://github.com/JanusGraph/janusgraph +hugegraph-struct/src/main/java/org/apache/hugegraph/type/Namifiable.java from https://github.com/JanusGraph/janusgraph diff --git a/hugegraph-struct/pom.xml b/hugegraph-struct/pom.xml new file mode 100644 index 000000000..62ad58ee9 --- /dev/null +++ b/hugegraph-struct/pom.xml @@ -0,0 +1,197 @@ + + + + 4.0.0 + + hugegraph-struct + + + org.apache.hugegraph + hugegraph + ${revision} + ../pom.xml + + + + 11 + 11 + UTF-8 + 25.1-jre + 3.5.1 + + + + + org.apache.hugegraph + hg-pd-client + ${project.version} + + + + jakarta.ws.rs + jakarta.ws.rs-api + 3.0.0 + + + + org.apache.tinkerpop + gremlin-test + ${tinkerpop.version} + + + + com.google.code.gson + gson + 2.8.9 + + + + org.apache.hugegraph + hugegraph-common + ${project.version} + + + org.glassfish.jersey.core + jersey-client + + + + + com.google.guava + guava + ${guava.version} + + + + + + + + org.apache.tinkerpop + gremlin-shaded + 3.5.1 + + + org.mindrot + jbcrypt + 0.4 + + + org.eclipse.collections + eclipse-collections-api + 10.4.0 + + + org.eclipse.collections + eclipse-collections + 10.4.0 + + + it.unimi.dsi + fastutil + 8.1.0 + + + org.lz4 + lz4-java + 1.7.1 + + + org.apache.commons + commons-text + 1.10.0 + + + + org.apdplat + word + 1.3 + + + ch.qos.logback + logback-classic + + + slf4j-api + org.slf4j + + + + + org.ansj + ansj_seg + 5.1.6 + + + com.hankcs + hanlp + portable-1.5.0 + + + org.apache.lucene + lucene-analyzers-smartcn + 7.4.0 + + + org.apache.lucene + lucene-core + 7.4.0 + + + io.jsonwebtoken + jjwt-api + 0.11.2 + + + io.jsonwebtoken + jjwt-impl + 0.11.2 + runtime + + + io.jsonwebtoken + jjwt-jackson + 0.11.2 + runtime + + + com.huaban + jieba-analysis + 1.0.2 + + + org.lionsoul + jcseg-core + 2.2.0 + + + com.chenlb.mmseg4j + mmseg4j-core + 1.10.0 + + + com.janeluo + ikanalyzer + 2012_u6 + + + + + diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/HugeGraphSupplier.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/HugeGraphSupplier.java new file mode 100644 index 000000000..91c747676 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/HugeGraphSupplier.java @@ -0,0 +1,79 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph; + +import java.util.Collection; +import java.util.List; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.util.DateUtil; + +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.schema.EdgeLabel; +import org.apache.hugegraph.schema.IndexLabel; +import org.apache.hugegraph.schema.PropertyKey; +import org.apache.hugegraph.schema.VertexLabel; + +/** + * Acturally, it would be better if this interface be called + * "HugeGraphSchemaSupplier". + */ +public interface HugeGraphSupplier { + + public List mapPkId2Name(Collection ids); + + public List mapIlId2Name(Collection ids); + + public PropertyKey propertyKey(Id key); + + public Collection propertyKeys(); + + public VertexLabel vertexLabelOrNone(Id id); + + public boolean existsLinkLabel(Id vertexLabel); + + public VertexLabel vertexLabel(Id label); + + public VertexLabel vertexLabel(String label); + + + public default EdgeLabel edgeLabelOrNone(Id id) { + EdgeLabel el = this.edgeLabel(id); + if (el == null) { + el = EdgeLabel.undefined(this, id); + } + return el; + } + public EdgeLabel edgeLabel(Id label); + + public EdgeLabel edgeLabel(String label); + + public IndexLabel indexLabel(Id id); + + public Collection indexLabels(); + + public String name(); + + public HugeConfig configuration(); + + default long now() { + return DateUtil.now().getTime(); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/SchemaDriver.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/SchemaDriver.java new file mode 100644 index 000000000..9bd3699b3 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/SchemaDriver.java @@ -0,0 +1,860 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.Log; +import org.apache.tinkerpop.shaded.jackson.core.JsonProcessingException; +import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper; +import org.slf4j.Logger; + +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.exception.NotAllowException; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.pd.client.KvClient; +import org.apache.hugegraph.pd.client.PDConfig; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.kv.KResponse; +import org.apache.hugegraph.pd.grpc.kv.ScanPrefixResponse; +import org.apache.hugegraph.pd.grpc.kv.WatchEvent; +import org.apache.hugegraph.pd.grpc.kv.WatchResponse; +import org.apache.hugegraph.pd.grpc.kv.WatchType; +import org.apache.hugegraph.schema.EdgeLabel; +import org.apache.hugegraph.schema.IndexLabel; +import org.apache.hugegraph.schema.PropertyKey; +import org.apache.hugegraph.schema.SchemaElement; +import org.apache.hugegraph.schema.VertexLabel; +import org.apache.hugegraph.type.HugeType; + +public class SchemaDriver { + private static Logger log = Log.logger(SchemaDriver.class); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + public static final String DELIMITER = "-"; + public static final String META_PATH_DELIMITER = "/"; + public static final String META_PATH_HUGEGRAPH = "HUGEGRAPH"; + public static final String META_PATH_GRAPHSPACE = "GRAPHSPACE"; + public static final String META_PATH_GRAPH = "GRAPH"; + public static final String META_PATH_CLUSTER = "hg"; + public static final String META_PATH_SCHEMA = "SCHEMA"; + public static final String META_PATH_GRAPH_CONF = "GRAPH_CONF"; + public static final String META_PATH_PROPERTY_KEY = "PROPERTY_KEY"; + public static final String META_PATH_VERTEX_LABEL = "VERTEX_LABEL"; + public static final String META_PATH_EDGE_LABEL = "EDGE_LABEL"; + public static final String META_PATH_INDEX_LABEL = "INDEX_LABEL"; + public static final String META_PATH_NAME = "NAME"; + public static final String META_PATH_ID = "ID"; + public static final String META_PATH_EVENT = "EVENT"; + public static final String META_PATH_REMOVE = "REMOVE"; + public static final String META_PATH_CLEAR = "CLEAR"; + + private static final AtomicReference INSTANCE = + new AtomicReference<>(); + // Client for accessing PD + private final KvClient client; + + private SchemaCaches caches; + + private SchemaDriver(PDConfig pdConfig, int cacheSize, + long expiration) { + this.client = new KvClient<>(pdConfig); + this.caches = new SchemaCaches(cacheSize, expiration); + this.listenMetaChanges(); + log.info(String.format( + "The SchemaDriver initialized successfully, cacheSize = %s," + + " expiration = %s s", cacheSize, expiration / 1000)); + } + + + public static void init(PDConfig pdConfig) { + init(pdConfig, 300, 300 * 1000); + } + + public static void init(PDConfig pdConfig, int cacheSize, long expiration) { + SchemaDriver instance = INSTANCE.get(); + if (instance != null) { + throw new NotAllowException( + "The SchemaDriver [cacheSize=%s, expiration=%s, " + + "client=%s] has already been initialized and is not " + + "allowed to be initialized again", instance.caches.limit(), + instance.caches.expiration(), instance.client); + } + INSTANCE.compareAndSet(null, new SchemaDriver(pdConfig, cacheSize, + expiration)); + } + + public static void destroy() { + SchemaDriver instance = INSTANCE.get(); + if (instance != null) { + instance.caches.cancelScheduleCacheClean(); + instance.caches.destroyAll(); + INSTANCE.set(null); + } + } + + public SchemaCaches schemaCaches() { + return this.caches; + } + + public static SchemaDriver getInstance() { + return INSTANCE.get(); + } + + private void listenMetaChanges() { + this.listen(graphSpaceRemoveKey(), this::graphSpaceRemoveHandler); + this.listen(graphRemoveKey(), this::graphRemoveHandler); + this.listen(graphClearKey(), this::graphClearHandler); + this.listen(schemaCacheClearKey(), this::schemaCacheClearHandler); + } + + private void schemaCacheClearHandler(T response) { + List names = this.extractValuesFromResponse(response); + for (String gs : names) { + String[] arr = gs.split(DELIMITER); + assert arr.length == 2; + this.caches.clear(arr[0], arr[1]); + log.info(String.format( + "Graph '%s' schema clear event is received, deleting all " + + "schema caches under '%s'", gs, gs)); + } + } + + private void graphClearHandler(T response) { + List names = this.extractValuesFromResponse(response); + for (String gs : names) { + String[] arr = gs.split(DELIMITER); + assert arr.length == 2; + this.caches.clear(arr[0], arr[1]); + log.info(String.format( + "Graph '%s' clear event is received, deleting all " + + "schema caches under '%s'", gs, gs)); + } + } + + private void graphRemoveHandler(T response) { + List names = this.extractValuesFromResponse(response); + for (String gs : names) { + String[] arr = gs.split(DELIMITER); + assert arr.length == 2; + this.caches.destroy(arr[0], arr[1]); + log.info(String.format( + "Graph '%s' delete event is received, deleting all " + + "schema caches under '%s'", gs, gs)); + } + } + + private void graphSpaceRemoveHandler(T response) { + List names = this.extractValuesFromResponse(response); + for (String gs : names) { + this.caches.destroy(gs); + log.info(String.format( + "graph space '%s' delete event is received, deleting all " + + "schema caches under '%s'", gs, gs)); + } + } + + + public List extractValuesFromResponse(T response) { + List values = new ArrayList<>(); + WatchResponse res = (WatchResponse) response; + for (WatchEvent event : res.getEventsList()) { + // Skip if not PUT event + if (!event.getType().equals(WatchType.Put)) { + return null; + } + String value = event.getCurrent().getValue(); + values.add(value); + } + return values; + } + + + public void listen(String key, Consumer consumer) { + try { + this.client.listen(key, (Consumer) consumer); + } catch (PDException e) { + throw new HugeException("Failed to listen '%s' to pd", e, key); + } + } + + public Map graphConfig(String graphSpace, String graph) { + String content = this.get(graphConfKey(graphSpace, graph)); + if (content == null || content.length() == 0) { + return new HashMap<>(); + } else { + return fromJson(content, Map.class); + } + } + + public PropertyKey propertyKey(String graphSpace, String graph, Id id, + HugeGraphSupplier schemaGraph) { + SchemaElement pk = + this.caches.get(graphSpace, graph, HugeType.PROPERTY_KEY, id); + if (pk == null) { + pk = getPropertyKey(graphSpace, graph, id, schemaGraph); + E.checkArgument(pk != null, "no such propertyKey: id = '%s'", id); + this.caches.set(graphSpace, graph, HugeType.PROPERTY_KEY, pk.id(), pk); + this.caches.set(graphSpace, graph, HugeType.PROPERTY_KEY, pk.name(), pk); + } + return (PropertyKey) pk; + } + + public PropertyKey propertyKey(String graphSpace, String graph, + String name, HugeGraphSupplier schemaGraph) { + SchemaElement pk = + this.caches.get(graphSpace, graph, HugeType.PROPERTY_KEY, name); + if (pk == null) { + pk = getPropertyKey(graphSpace, graph, name, schemaGraph); + E.checkArgument(pk != null, "no such propertyKey: name = '%s'", + name); + this.caches.set(graphSpace, graph, HugeType.PROPERTY_KEY, pk.id(), pk); + this.caches.set(graphSpace, graph, HugeType.PROPERTY_KEY, pk.name(), pk); + } + return (PropertyKey) pk; + } + + public List propertyKeys(String graphSpace, String graph, + HugeGraphSupplier schemaGraph) { + Map propertyKeysKvs = + this.scanWithPrefix(propertyKeyPrefix(graphSpace, graph)); + List propertyKeys = + new ArrayList<>(propertyKeysKvs.size()); + for (String value : propertyKeysKvs.values()) { + PropertyKey pk = + PropertyKey.fromMap(fromJson(value, Map.class), schemaGraph); + this.caches.set(graphSpace, graph, HugeType.PROPERTY_KEY, pk.id(), pk); + this.caches.set(graphSpace, graph, HugeType.PROPERTY_KEY, pk.name(), pk); + propertyKeys.add(pk); + } + return propertyKeys; + } + + public List vertexLabels(String graphSpace, String graph, + HugeGraphSupplier schemaGraph) { + Map vertexLabelKvs = this.scanWithPrefix( + vertexLabelPrefix(graphSpace, graph)); + List vertexLabels = + new ArrayList<>(vertexLabelKvs.size()); + for (String value : vertexLabelKvs.values()) { + VertexLabel vl = + VertexLabel.fromMap(fromJson(value, Map.class), + schemaGraph); + this.caches.set(graphSpace, graph, HugeType.VERTEX_LABEL, vl.id(), vl); + this.caches.set(graphSpace, graph, HugeType.VERTEX_LABEL, vl.name(), vl); + vertexLabels.add(vl); + } + return vertexLabels; + } + + public List edgeLabels(String graphSpace, String graph, + HugeGraphSupplier schemaGraph) { + Map edgeLabelKvs = this.scanWithPrefix( + edgeLabelPrefix(graphSpace, graph)); + List edgeLabels = + new ArrayList<>(edgeLabelKvs.size()); + for (String value : edgeLabelKvs.values()) { + EdgeLabel el = + EdgeLabel.fromMap(fromJson(value, Map.class), schemaGraph); + this.caches.set(graphSpace, graph, HugeType.EDGE_LABEL, el.id(), el); + this.caches.set(graphSpace, graph, HugeType.EDGE_LABEL, el.name(), el); + edgeLabels.add(el); + } + return edgeLabels; + } + + public List indexLabels(String graphSpace, String graph, + HugeGraphSupplier schemaGraph) { + Map indexLabelKvs = this.scanWithPrefix( + indexLabelPrefix(graphSpace, graph)); + List indexLabels = + new ArrayList<>(indexLabelKvs.size()); + for (String value : indexLabelKvs.values()) { + IndexLabel il = + IndexLabel.fromMap(fromJson(value, Map.class), schemaGraph); + this.caches.set(graphSpace, graph, HugeType.INDEX_LABEL, il.id(), il); + this.caches.set(graphSpace, graph, HugeType.INDEX_LABEL, il.name(), il); + indexLabels.add(il); + } + return indexLabels; + } + + private String propertyKeyPrefix(String graphSpace, String graph) { + // HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph + // }/SCHEMA/PROPERTY_KEY/NAME + return stringJoin(META_PATH_DELIMITER, + META_PATH_HUGEGRAPH, + META_PATH_CLUSTER, + META_PATH_GRAPHSPACE, + graphSpace, + graph, + META_PATH_SCHEMA, + META_PATH_PROPERTY_KEY, + META_PATH_NAME); + } + + private String vertexLabelPrefix(String graphSpace, String graph) { + // HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph + // }/SCHEMA/VERTEX_LABEL/NAME + return stringJoin(META_PATH_DELIMITER, + META_PATH_HUGEGRAPH, + META_PATH_CLUSTER, + META_PATH_GRAPHSPACE, + graphSpace, + graph, + META_PATH_SCHEMA, + META_PATH_VERTEX_LABEL, + META_PATH_NAME); + } + + private String edgeLabelPrefix(String graphSpace, String graph) { + // HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph + // }/SCHEMA/EDGELABEL/NAME + return stringJoin(META_PATH_DELIMITER, + META_PATH_HUGEGRAPH, + META_PATH_CLUSTER, + META_PATH_GRAPHSPACE, + graphSpace, + graph, + META_PATH_SCHEMA, + META_PATH_EDGE_LABEL, + META_PATH_NAME); + } + + private String indexLabelPrefix(String graphSpace, String graph) { + // HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH/{graph + // }/SCHEMA/INDEX_LABEL/NAME + return stringJoin(META_PATH_DELIMITER, + META_PATH_HUGEGRAPH, + META_PATH_CLUSTER, + META_PATH_GRAPHSPACE, + graphSpace, + graph, + META_PATH_SCHEMA, + META_PATH_INDEX_LABEL, + META_PATH_NAME); + } + + public VertexLabel vertexLabel(String graphSpace, String graph, Id id, + HugeGraphSupplier schemaGraph) { + SchemaElement vl = + this.caches.get(graphSpace, graph, HugeType.VERTEX_LABEL, id); + if (vl == null) { + vl = getVertexLabel(graphSpace, graph, id, schemaGraph); + E.checkArgument(vl != null, "no such vertex label: id = '%s'", id); + this.caches.set(graphSpace, graph, HugeType.VERTEX_LABEL, vl.id(), vl); + this.caches.set(graphSpace, graph, HugeType.VERTEX_LABEL, vl.name(), vl); + } + return (VertexLabel) vl; + } + + public VertexLabel vertexLabel(String graphSpace, String graph, + String name, HugeGraphSupplier schemaGraph) { + SchemaElement vl = + this.caches.get(graphSpace, graph, HugeType.VERTEX_LABEL, name); + if (vl == null) { + vl = getVertexLabel(graphSpace, graph, name, schemaGraph); + E.checkArgument(vl != null, "no such vertex label: name = '%s'", + name); + this.caches.set(graphSpace, graph, HugeType.VERTEX_LABEL, vl.id(), vl); + this.caches.set(graphSpace, graph, HugeType.VERTEX_LABEL, vl.name(), vl); + } + return (VertexLabel) vl; + } + + public EdgeLabel edgeLabel(String graphSpace, String graph, Id id, + HugeGraphSupplier schemaGraph) { + SchemaElement el = + this.caches.get(graphSpace, graph, HugeType.EDGE_LABEL, id); + if (el == null) { + el = getEdgeLabel(graphSpace, graph, id, schemaGraph); + E.checkArgument(el != null, "no such edge label: id = '%s'", id); + this.caches.set(graphSpace, graph, HugeType.EDGE_LABEL, el.id(), el); + this.caches.set(graphSpace, graph, HugeType.EDGE_LABEL, el.name(), el); + } + return (EdgeLabel) el; + } + + public EdgeLabel edgeLabel(String graphSpace, String graph, String name, + HugeGraphSupplier schemaGraph) { + SchemaElement el = + this.caches.get(graphSpace, graph, HugeType.EDGE_LABEL, name); + if (el == null) { + el = getEdgeLabel(graphSpace, graph, name, schemaGraph); + E.checkArgument(el != null, "no such edge label: name = '%s'", + name); + this.caches.set(graphSpace, graph, HugeType.EDGE_LABEL, el.id(), el); + this.caches.set(graphSpace, graph, HugeType.EDGE_LABEL, el.name(), el); + } + return (EdgeLabel) el; + } + + public IndexLabel indexLabel(String graphSpace, String graph, Id id, + HugeGraphSupplier schemaGraph) { + SchemaElement il = + this.caches.get(graphSpace, graph, HugeType.INDEX_LABEL, id); + if (il == null) { + il = getIndexLabel(graphSpace, graph, id, schemaGraph); + E.checkArgument(il != null, "no such index label: id = '%s'", id); + this.caches.set(graphSpace, graph, HugeType.INDEX_LABEL, il.id(), il); + this.caches.set(graphSpace, graph, HugeType.INDEX_LABEL, il.name(), il); + } + return (IndexLabel) il; + } + + public IndexLabel indexLabel(String graphSpace, String graph, String name, + HugeGraphSupplier schemaGraph) { + SchemaElement il = + this.caches.get(graphSpace, graph, HugeType.INDEX_LABEL, name); + if (il == null) { + il = getIndexLabel(graphSpace, graph, name, schemaGraph); + E.checkArgument(il != null, "no such index label: name = '%s'", + name); + this.caches.set(graphSpace, graph, HugeType.INDEX_LABEL, il.id(), il); + this.caches.set(graphSpace, graph, HugeType.INDEX_LABEL, il.name(), il); + } + return (IndexLabel) il; + } + + private String get(String key) { + try { + KResponse response = this.client.get(key); + return response.getValue(); + } catch (PDException e) { + throw new HugeException("Failed to get '%s' from pd", e, key); + } + } + + private Map scanWithPrefix(String prefix) { + try { + ScanPrefixResponse response = this.client.scanPrefix(prefix); + return response.getKvsMap(); + } catch (PDException e) { + throw new HugeException("Failed to scanWithPrefix '%s' from pd", e, prefix); + } + } + + private PropertyKey getPropertyKey(String graphSpace, String graph, + Id propertyKey, HugeGraphSupplier schemaGraph) { + String content = + this.get(propertyKeyIdKey(graphSpace, graph, propertyKey)); + if (content == null || content.length() == 0) { + return null; + } else { + return PropertyKey.fromMap(fromJson(content, Map.class), schemaGraph); + } + } + + private PropertyKey getPropertyKey(String graphSpace, String graph, + String propertyKey, HugeGraphSupplier schemaGraph) { + String content = + this.get(propertyKeyNameKey(graphSpace, graph, propertyKey)); + if (content == null || content.length() == 0) { + return null; + } else { + return PropertyKey.fromMap(fromJson(content, Map.class), schemaGraph); + } + } + + private VertexLabel getVertexLabel(String graphSpace, String graph, + Id vertexLabel, HugeGraphSupplier schemaGraph) { + String content = + this.get(vertexLabelIdKey(graphSpace, graph, vertexLabel)); + if (content == null || content.length() == 0) { + return null; + } else { + return VertexLabel.fromMap(fromJson(content, Map.class), schemaGraph); + } + } + + private VertexLabel getVertexLabel(String graphSpace, String graph, + String vertexLabel, HugeGraphSupplier schemaGraph) { + String content = + this.get(vertexLabelNameKey(graphSpace, graph, vertexLabel)); + if (content == null || content.length() == 0) { + return null; + } else { + return VertexLabel.fromMap(fromJson(content, Map.class), schemaGraph); + } + } + + private EdgeLabel getEdgeLabel(String graphSpace, String graph, + Id edgeLabel, HugeGraphSupplier schemaGraph) { + String content = + this.get(edgeLabelIdKey(graphSpace, graph, edgeLabel)); + if (content == null || content.length() == 0) { + return null; + } else { + return EdgeLabel.fromMap(fromJson(content, Map.class), schemaGraph); + } + } + + private EdgeLabel getEdgeLabel(String graphSpace, String graph, + String edgeLabel, HugeGraphSupplier schemaGraph) { + String content = + this.get(edgeLabelNameKey(graphSpace, graph, edgeLabel)); + if (content == null || content.length() == 0) { + return null; + } else { + return EdgeLabel.fromMap(fromJson(content, Map.class), schemaGraph); + } + } + + + private IndexLabel getIndexLabel(String graphSpace, String graph, + Id indexLabel, HugeGraphSupplier schemaGraph) { + String content = + this.get(indexLabelIdKey(graphSpace, graph, indexLabel)); + if (content == null || content.length() == 0) { + return null; + } else { + return IndexLabel.fromMap(fromJson(content, Map.class), schemaGraph); + } + } + + private IndexLabel getIndexLabel(String graphSpace, String graph, + String indexLabel, + HugeGraphSupplier schemaGraph) { + String content = + this.get(indexLabelNameKey(graphSpace, graph, indexLabel)); + if (content == null || content.length() == 0) { + return null; + } else { + return IndexLabel.fromMap(fromJson(content, Map.class), + schemaGraph); + } + } + + + private T fromJson(String json, Class clazz) { + E.checkState(json != null, "Json value can't be null for '%s'", + clazz.getSimpleName()); + try { + return MAPPER.readValue(json, clazz); + } catch (IOException e) { + throw new HugeException("Can't read json: %s", e, e.getMessage()); + } + } + + private String toJson(Object object) { + try { + return MAPPER.writeValueAsString(object); + } catch (JsonProcessingException e) { + throw new HugeException("Can't write json: %s", e, e.getMessage()); + } + } + + private String propertyKeyIdKey(String graphSpace, String graph, Id id) { + return idKey(graphSpace, graph, id, HugeType.PROPERTY_KEY); + } + + private String propertyKeyNameKey(String graphSpace, String graph, + String name) { + return nameKey(graphSpace, graph, name, HugeType.PROPERTY_KEY); + } + + + private String vertexLabelIdKey(String graphSpace, String graph, Id id) { + return idKey(graphSpace, graph, id, HugeType.VERTEX_LABEL); + } + + private String vertexLabelNameKey(String graphSpace, String graph, + String name) { + return nameKey(graphSpace, graph, name, HugeType.VERTEX_LABEL); + } + + private String edgeLabelIdKey(String graphSpace, String graph, Id id) { + return idKey(graphSpace, graph, id, HugeType.EDGE_LABEL); + } + + private String edgeLabelNameKey(String graphSpace, String graph, + String name) { + return nameKey(graphSpace, graph, name, HugeType.EDGE_LABEL); + } + + private String indexLabelIdKey(String graphSpace, String graph, Id id) { + return idKey(graphSpace, graph, id, HugeType.INDEX_LABEL); + } + + private String indexLabelNameKey(String graphSpace, String graph, + String name) { + return nameKey(graphSpace, graph, name, HugeType.INDEX_LABEL); + } + + private String graphSpaceRemoveKey() { + // HUGEGRAPH/{cluster}/EVENT/GRAPHSPACE/REMOVE + return stringJoin(META_PATH_DELIMITER, + META_PATH_HUGEGRAPH, + META_PATH_CLUSTER, + META_PATH_EVENT, + META_PATH_GRAPHSPACE, + META_PATH_REMOVE); + } + + private String graphConfKey(String graphSpace, String graph) { + // HUGEGRAPH/{cluster}/GRAPHSPACE/{graphspace}/GRAPH_CONF/{graph} + return stringJoin(META_PATH_DELIMITER, + META_PATH_HUGEGRAPH, + META_PATH_CLUSTER, + META_PATH_GRAPHSPACE, + graphSpace, + META_PATH_GRAPH_CONF, + graph); + } + + private String nameKey(String graphSpace, String graph, + String name, HugeType type) { + // HUGEGRAPH/hg/GRAPHSPACE/{graphspace}/{graph}/SCHEMA + // /{META_PATH_TYPE}/NAME/{name} + return stringJoin(META_PATH_DELIMITER, + META_PATH_HUGEGRAPH, + META_PATH_CLUSTER, + META_PATH_GRAPHSPACE, + graphSpace, + graph, + META_PATH_SCHEMA, + hugeType2MetaPath(type), + META_PATH_NAME, + name); + } + + private String idKey(String graphSpace, String graph, + Id id, HugeType type) { + // HUGEGRAPH/hg/GRAPHSPACE/{graphspace}/{graph}/SCHEMA + // /{META_PATH_TYPE}/ID/{id} + return stringJoin(META_PATH_DELIMITER, + META_PATH_HUGEGRAPH, + META_PATH_CLUSTER, + META_PATH_GRAPHSPACE, + graphSpace, + graph, + META_PATH_SCHEMA, + hugeType2MetaPath(type), + META_PATH_ID, + id.asString()); + } + + private String schemaCacheClearKey() { + // HUGEGRAPH/{cluster}/EVENT/GRAPH/SCHEMA/CLEAR + return stringJoin(META_PATH_DELIMITER, + META_PATH_HUGEGRAPH, + META_PATH_CLUSTER, + META_PATH_EVENT, + META_PATH_GRAPH, + META_PATH_SCHEMA, + META_PATH_CLEAR); + } + + private String graphClearKey() { + // HUGEGRAPH/{cluster}/EVENT/GRAPH/CLEAR + return stringJoin(META_PATH_DELIMITER, + META_PATH_HUGEGRAPH, + META_PATH_CLUSTER, + META_PATH_EVENT, + META_PATH_GRAPH, + META_PATH_CLEAR); + } + + private String graphRemoveKey() { + // HUGEGRAPH/{cluster}/EVENT/GRAPH/REMOVE + return stringJoin(META_PATH_DELIMITER, + META_PATH_HUGEGRAPH, + META_PATH_CLUSTER, + META_PATH_EVENT, + META_PATH_GRAPH, + META_PATH_REMOVE); + } + + private String hugeType2MetaPath(HugeType type) { + String schemaType = null; + switch (type) { + case PROPERTY_KEY: + schemaType = META_PATH_PROPERTY_KEY; + break; + case VERTEX_LABEL: + schemaType = META_PATH_VERTEX_LABEL; + break; + case EDGE_LABEL: + schemaType = META_PATH_EDGE_LABEL; + break; + case INDEX_LABEL: + schemaType = META_PATH_INDEX_LABEL; + break; + default: + throw new AssertionError(String.format( + "Invalid HugeType : %s", type)); + } + return schemaType; + } + + private static String stringJoin(String delimiter, String... parts) { + StringBuilder builder = new StringBuilder(); + int size = parts.length; + for (int i = 0; i < size; i++) { + builder.append(parts[i]); + if (i < size - 1) { + builder.append(delimiter); + } + } + return builder.toString(); + } + + private static final class SchemaCaches { + private final int limit; + private final long expiration; + private final Timer timer; + + private ConcurrentHashMap> caches; + + public SchemaCaches(int limit, long expiration) { + this.expiration = expiration; + this.limit = limit; + this.timer = new Timer(); + this.caches = new ConcurrentHashMap<>(); + scheduleCacheCleanup(); + } + + public int limit() { + return this.limit; + } + + public long expiration() { + return this.expiration; + } + + private void scheduleCacheCleanup() { + timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + log.debug("schedule clear schema caches"); + clearAll(); + } + }, expiration, expiration); + } + + public void cancelScheduleCacheClean() { + timer.cancel(); + } + + public SchemaElement get(String graphSpace, String graph, HugeType type, + Id id) { + return get(graphSpace, graph, type, id.asString()); + } + + public SchemaElement get(String graphSpace, String graph, HugeType type, + String name) { + String graphName = stringJoin(DELIMITER, graphSpace, graph); + if (this.caches.get(graphName) == null) { + this.caches.put(graphName, new ConcurrentHashMap<>(this.limit)); + } + return this.caches.get(graphName) + .get(stringJoin(DELIMITER, type.string(), name)); + } + + public void set(String graphSpace, String graph, HugeType type, Id id, + SchemaElement value) { + set(graphSpace, graph, type, id.asString(), value); + } + + public void set(String graphSpace, String graph, HugeType type, + String name, SchemaElement value) { + String graphName = stringJoin(DELIMITER, graphSpace, graph); + ConcurrentHashMap + schemaCaches = this.caches.get(graphName); + if (schemaCaches == null) { + schemaCaches = this.caches.put(graphName, new ConcurrentHashMap<>(this.limit)); + } + if (schemaCaches.size() >= limit) { + log.info(String.format( + "The current '%s''s schemaCaches size '%s' reached " + + "limit '%s'", graphName, schemaCaches.size(), limit)); + return; + } + schemaCaches.put(stringJoin(DELIMITER, type.string(), name), + value); + log.debug(String.format("graph '%s' add schema caches '%s'", + graphName, + stringJoin(DELIMITER, type.string(), + name))); + } + + public void remove(String graphSpace, String graph, HugeType type, + Id id) { + remove(graphSpace, graph, type, id.asString()); + } + + public void remove(String graphSpace, String graph, HugeType type, + String name) { + String graphName = stringJoin(DELIMITER, graphSpace, graph); + + ConcurrentHashMap + schemaCaches = this.caches.get(graphName); + schemaCaches.remove(stringJoin(DELIMITER, type.string(), name)); + + } + + public void clearAll() { + for (String key : this.caches.keySet()) { + log.debug(String.format("graph in '%s' schema caches clear", + key)); + this.caches.get(key).clear(); + } + } + + public void clear(String graphSpace, String graph) { + ConcurrentHashMap + schemaCaches = + this.caches.get(stringJoin(DELIMITER, graphSpace, graph)); + if (schemaCaches != null) { + schemaCaches.clear(); + } + } + + public void destroyAll() { + this.caches.clear(); + } + + public void destroy(String graphSpace, String graph) { + this.caches.remove(stringJoin(DELIMITER, graphSpace, graph)); + + } + + public void destroy(String graphSpace) { + for (String key : this.caches.keySet()) { + String gs = key.split(DELIMITER)[0]; + if (gs.equals(graphSpace)) { + this.caches.remove(key); + } + } + } + + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/SchemaGraph.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/SchemaGraph.java new file mode 100644 index 000000000..5462949ff --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/SchemaGraph.java @@ -0,0 +1,182 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph; + +import org.apache.hugegraph.HugeGraphSupplier; +import org.apache.hugegraph.SchemaDriver; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.pd.client.PDConfig; +import org.apache.hugegraph.schema.*; + +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.MapConfiguration; +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.util.E; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class SchemaGraph implements HugeGraphSupplier { + + private final String graphSpace; + private final String graph; + private final PDConfig pdConfig; + private HugeConfig config; + + private final SchemaDriver schemaDriver; + + public SchemaGraph(String graphSpace, String graph, PDConfig pdConfig) { + this.graphSpace = graphSpace; + this.graph = graph; + this.pdConfig = pdConfig; + this.schemaDriver = schemaDriverInit(); + this.config = this.loadConfig(); + } + + private SchemaDriver schemaDriverInit() { + if (SchemaDriver.getInstance() == null) { + synchronized (SchemaDriver.class) { + if (SchemaDriver.getInstance() == null) { + SchemaDriver.init(this.pdConfig); + } + } + } + return SchemaDriver.getInstance(); + } + + private HugeConfig loadConfig() { + // Load configuration from PD + Map configs = + schemaDriver.graphConfig(this.graphSpace, this.graph); + Configuration propConfig = new MapConfiguration(configs); + return new HugeConfig(propConfig); + } + + @Override + public List mapPkId2Name(Collection ids) { + List names = new ArrayList<>(ids.size()); + for (Id id : ids) { + SchemaElement schema = this.propertyKey(id); + names.add(schema.name()); + } + return names; + } + + @Override + public List mapIlId2Name(Collection ids) { + List names = new ArrayList<>(ids.size()); + for (Id id : ids) { + SchemaElement schema = this.indexLabel(id); + names.add(schema.name()); + } + return names; + } + + @Override + public HugeConfig configuration(){ + return this.config; + } + + @Override + public PropertyKey propertyKey(Id id) { + return schemaDriver.propertyKey(this.graphSpace, this.graph, id, this); + } + + public PropertyKey propertyKey(String name) { + return schemaDriver.propertyKey(this.graphSpace, this.graph, name, this); + } + + @Override + public Collection propertyKeys() { + // TODO + return null; + } + + @Override + public VertexLabel vertexLabelOrNone(Id id) { + VertexLabel vl = vertexLabel(id); + if (vl == null) { + vl = VertexLabel.undefined(null, id); + } + return vl; + } + + @Override + public boolean existsLinkLabel(Id vertexLabel) { + List edgeLabels = + schemaDriver.edgeLabels(this.graphSpace, this.graph, this); + for (EdgeLabel edgeLabel : edgeLabels) { + if (edgeLabel.linkWithLabel(vertexLabel)) { + return true; + } + } + return false; + } + + @Override + public VertexLabel vertexLabel(Id id) { + E.checkArgumentNotNull(id, "Vertex label id can't be null"); + if (SchemaElement.OLAP_ID.equals(id)) { + return VertexLabel.OLAP_VL; + } + return schemaDriver.vertexLabel(this.graphSpace, this.graph, id, this); + } + + @Override + public VertexLabel vertexLabel(String name) { + E.checkArgumentNotNull(name, "Vertex label name can't be null"); + E.checkArgument(!name.isEmpty(), "Vertex label name can't be empty"); + if (SchemaElement.OLAP.equals(name)) { + return VertexLabel.OLAP_VL; + } + return schemaDriver.vertexLabel(this.graphSpace, this.graph, name, this); + } + + @Override + public EdgeLabel edgeLabel(Id id) { + return schemaDriver.edgeLabel(this.graphSpace, this.graph, id, this); + } + + @Override + public EdgeLabel edgeLabel(String name) { + return schemaDriver.edgeLabel(this.graphSpace, this.graph, name, this); + } + + @Override + public IndexLabel indexLabel(Id id) { + return schemaDriver.indexLabel(this.graphSpace, this.graph, id, this); + } + + @Override + public Collection indexLabels() { + return schemaDriver.indexLabels(this.graphSpace, this.graph, this); + } + + public IndexLabel indexLabel(String name) { + return schemaDriver.indexLabel(this.graphSpace, this.graph, name, this); + } + + @Override + public String name() { + return String.join("-", this.graphSpace, this.graph); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/Analyzer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/Analyzer.java new file mode 100644 index 000000000..4edd2ffa9 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/Analyzer.java @@ -0,0 +1,27 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.analyzer; + +import java.util.Set; + +public interface Analyzer { + + public Set segment(String text); +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/AnalyzerFactory.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/AnalyzerFactory.java new file mode 100644 index 000000000..bff18ab7b --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/AnalyzerFactory.java @@ -0,0 +1,102 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.analyzer; + + +import org.apache.hugegraph.exception.HugeException; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class AnalyzerFactory { + + private static Map> analyzers; + + static { + analyzers = new ConcurrentHashMap<>(); + } + + public static Analyzer analyzer(String name, String mode) { + name = name.toLowerCase(); + switch (name) { + case "word": + return new WordAnalyzer(mode); + case "ansj": + return new AnsjAnalyzer(mode); + case "hanlp": + return new HanLPAnalyzer(mode); + case "smartcn": + return new SmartCNAnalyzer(mode); + case "jieba": + return new JiebaAnalyzer(mode); + case "jcseg": + return new JcsegAnalyzer(mode); + case "mmseg4j": + return new MMSeg4JAnalyzer(mode); + case "ikanalyzer": + return new IKAnalyzer(mode); + default: + return customizedAnalyzer(name, mode); + } + } + + private static Analyzer customizedAnalyzer(String name, String mode) { + Class clazz = analyzers.get(name); + if (clazz == null) { + throw new HugeException("Not exists analyzer: %s", name); + } + + assert Analyzer.class.isAssignableFrom(clazz); + try { + return clazz.getConstructor(String.class).newInstance(mode); + } catch (Exception e) { + throw new HugeException( + "Failed to construct analyzer '%s' with mode '%s'", + e, name, mode); + } + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public static void register(String name, String classPath) { + ClassLoader classLoader = AnalyzerFactory.class.getClassLoader(); + Class clazz; + try { + clazz = classLoader.loadClass(classPath); + } catch (Exception e) { + throw new HugeException("Load class path '%s' failed", + e, classPath); + } + + // Check subclass + if (!Analyzer.class.isAssignableFrom(clazz)) { + throw new HugeException("Class '%s' is not a subclass of " + + "class Analyzer", classPath); + } + + // Check exists + if (analyzers.containsKey(name)) { + throw new HugeException("Exists analyzer: %s(%s)", + name, analyzers.get(name).getName()); + } + + // Register class + analyzers.put(name, (Class) clazz); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/AnsjAnalyzer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/AnsjAnalyzer.java new file mode 100644 index 000000000..3f041d31f --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/AnsjAnalyzer.java @@ -0,0 +1,87 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.analyzer; + +import java.util.List; +import java.util.Set; + +import org.ansj.domain.Result; +import org.ansj.domain.Term; +import org.ansj.splitWord.analysis.BaseAnalysis; +import org.ansj.splitWord.analysis.IndexAnalysis; +import org.ansj.splitWord.analysis.NlpAnalysis; +import org.ansj.splitWord.analysis.ToAnalysis; +import org.apache.hugegraph.config.ConfigException; +import org.apache.hugegraph.util.InsertionOrderUtil; + +import com.google.common.collect.ImmutableList; + +/** + * Reference from https://my.oschina.net/apdplat/blog/412921 + */ +public class AnsjAnalyzer implements Analyzer { + + public static final List SUPPORT_MODES = ImmutableList.of( + "BaseAnalysis", + "IndexAnalysis", + "ToAnalysis", + "NlpAnalysis" + ); + + private String analysis; + + public AnsjAnalyzer(String mode) { + if (!SUPPORT_MODES.contains(mode)) { + throw new ConfigException( + "Unsupported segment mode '%s' for ansj analyzer, " + + "the available values are %s", mode, SUPPORT_MODES); + } + this.analysis = mode; + } + + @Override + public Set segment(String text) { + Result terms = null; + switch (this.analysis) { + case "BaseAnalysis": + terms = BaseAnalysis.parse(text); + break; + case "ToAnalysis": + terms = ToAnalysis.parse(text); + break; + case "NlpAnalysis": + terms = NlpAnalysis.parse(text); + break; + case "IndexAnalysis": + terms = IndexAnalysis.parse(text); + break; + default: + throw new AssertionError(String.format( + "Unsupported segment mode '%s'", this.analysis)); + } + + assert terms != null; + Set result = InsertionOrderUtil.newSet(); + for (Term term : terms) { + result.add(term.getName()); + } + return result; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/HanLPAnalyzer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/HanLPAnalyzer.java new file mode 100644 index 000000000..b8175e400 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/HanLPAnalyzer.java @@ -0,0 +1,108 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.analyzer; + +import java.util.List; +import java.util.Set; + + +import com.google.common.collect.ImmutableList; +import com.hankcs.hanlp.seg.Dijkstra.DijkstraSegment; +import com.hankcs.hanlp.seg.NShort.NShortSegment; +import com.hankcs.hanlp.seg.Segment; +import com.hankcs.hanlp.seg.common.Term; +import com.hankcs.hanlp.tokenizer.IndexTokenizer; +import com.hankcs.hanlp.tokenizer.NLPTokenizer; +import com.hankcs.hanlp.tokenizer.SpeedTokenizer; +import com.hankcs.hanlp.tokenizer.StandardTokenizer; + +import org.apache.hugegraph.config.ConfigException; +import org.apache.hugegraph.util.InsertionOrderUtil; + +/** + * Reference from https://my.oschina.net/apdplat/blog/412921 + */ +public class HanLPAnalyzer implements Analyzer { + + public static final List SUPPORT_MODES = + ImmutableList.builder() + .add("standard") + .add("nlp") + .add("index") + .add("nShort") + .add("shortest") + .add("speed") + .build(); + + private static final Segment N_SHORT_SEGMENT = + new NShortSegment().enableCustomDictionary(false) + .enablePlaceRecognize(true) + .enableOrganizationRecognize(true); + private static final Segment DIJKSTRA_SEGMENT = + new DijkstraSegment().enableCustomDictionary(false) + .enablePlaceRecognize(true) + .enableOrganizationRecognize(true); + + private String tokenizer; + + public HanLPAnalyzer(String mode) { + if (!SUPPORT_MODES.contains(mode)) { + throw new ConfigException( + "Unsupported segment mode '%s' for hanlp analyzer, " + + "the available values are %s", mode, SUPPORT_MODES); + } + this.tokenizer = mode; + } + + @Override + public Set segment(String text) { + List terms = null; + switch (this.tokenizer) { + case "standard": + terms = StandardTokenizer.segment(text); + break; + case "nlp": + terms = NLPTokenizer.segment(text); + break; + case "index": + terms = IndexTokenizer.segment(text); + break; + case "nShort": + terms = N_SHORT_SEGMENT.seg(text); + break; + case "shortest": + terms = DIJKSTRA_SEGMENT.seg(text); + break; + case "speed": + terms = SpeedTokenizer.segment(text); + break; + default: + throw new AssertionError(String.format( + "Unsupported segment mode '%s'", this.tokenizer)); + } + + assert terms != null; + Set result = InsertionOrderUtil.newSet(); + for (Term term : terms) { + result.add(term.word); + } + return result; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/IKAnalyzer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/IKAnalyzer.java new file mode 100644 index 000000000..a938e8e01 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/IKAnalyzer.java @@ -0,0 +1,73 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.analyzer; + +import com.google.common.collect.ImmutableList; + +import org.apache.hugegraph.config.ConfigException; +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.util.InsertionOrderUtil; +import org.wltea.analyzer.core.IKSegmenter; +import org.wltea.analyzer.core.Lexeme; + +import java.io.StringReader; +import java.util.List; +import java.util.Set; + +/** + * Reference from https://my.oschina.net/apdplat/blog/412921 + */ +public class IKAnalyzer implements Analyzer { + + public static final List SUPPORT_MODES = ImmutableList.of( + "smart", + "max_word" + ); + + private boolean smartSegMode; + private final IKSegmenter ik; + + public IKAnalyzer(String mode) { + if (!SUPPORT_MODES.contains(mode)) { + throw new ConfigException( + "Unsupported segment mode '%s' for ikanalyzer, " + + "the available values are %s", mode, SUPPORT_MODES); + } + this.smartSegMode = SUPPORT_MODES.get(0).equals(mode); + this.ik = new IKSegmenter(new StringReader(""), + this.smartSegMode); + } + + @Override + public Set segment(String text) { + Set result = InsertionOrderUtil.newSet(); + ik.reset(new StringReader(text)); + try { + Lexeme word = null; + while ((word = ik.next()) != null) { + result.add(word.getLexemeText()); + } + } catch (Exception e) { + throw new HugeException("IKAnalyzer segment text '%s' failed", + e, text); + } + return result; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/JcsegAnalyzer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/JcsegAnalyzer.java new file mode 100644 index 000000000..0a69af838 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/JcsegAnalyzer.java @@ -0,0 +1,77 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.analyzer; + +import java.io.StringReader; +import java.util.List; +import java.util.Set; + +import org.apache.hugegraph.config.ConfigException; +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.util.InsertionOrderUtil; +import org.lionsoul.jcseg.tokenizer.core.ADictionary; +import org.lionsoul.jcseg.tokenizer.core.DictionaryFactory; +import org.lionsoul.jcseg.tokenizer.core.ISegment; +import org.lionsoul.jcseg.tokenizer.core.IWord; +import org.lionsoul.jcseg.tokenizer.core.JcsegTaskConfig; +import org.lionsoul.jcseg.tokenizer.core.SegmentFactory; +import com.google.common.collect.ImmutableList; + +/** + * Reference from https://my.oschina.net/apdplat/blog/412921 + */ +public class JcsegAnalyzer implements Analyzer { + + public static final List SUPPORT_MODES = ImmutableList.of( + "Simple", + "Complex" + ); + + private static final JcsegTaskConfig CONFIG = new JcsegTaskConfig(); + private static final ADictionary DIC = + DictionaryFactory.createDefaultDictionary(new JcsegTaskConfig()); + + private int segMode; + + public JcsegAnalyzer(String mode) { + if (!SUPPORT_MODES.contains(mode)) { + throw new ConfigException( + "Unsupported segment mode '%s' for jcseg analyzer, " + + "the available values are %s", mode, SUPPORT_MODES); + } + this.segMode = SUPPORT_MODES.indexOf(mode) + 1; + } + + @Override + public Set segment(String text) { + Set result = InsertionOrderUtil.newSet(); + try { + Object[] args = new Object[]{new StringReader(text), CONFIG, DIC}; + ISegment seg = SegmentFactory.createJcseg(this.segMode, args); + IWord word = null; + while ((word = seg.next()) != null) { + result.add(word.getValue()); + } + } catch (Exception e) { + throw new HugeException("Jcseg segment text '%s' failed", e, text); + } + return result; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/JiebaAnalyzer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/JiebaAnalyzer.java new file mode 100644 index 000000000..70cae3326 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/JiebaAnalyzer.java @@ -0,0 +1,63 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.analyzer; + +import java.util.List; +import java.util.Set; + +import org.apache.hugegraph.config.ConfigException; +import org.apache.hugegraph.util.InsertionOrderUtil; + +import com.google.common.collect.ImmutableList; +import com.huaban.analysis.jieba.JiebaSegmenter; +import com.huaban.analysis.jieba.SegToken; + +/** + * Reference from https://my.oschina.net/apdplat/blog/412921 + */ +public class JiebaAnalyzer implements Analyzer { + + public static final List SUPPORT_MODES = ImmutableList.of( + "SEARCH", + "INDEX" + ); + + private static final JiebaSegmenter JIEBA_SEGMENTER = new JiebaSegmenter(); + + private JiebaSegmenter.SegMode segMode; + + public JiebaAnalyzer(String mode) { + if (!SUPPORT_MODES.contains(mode)) { + throw new ConfigException( + "Unsupported segment mode '%s' for jieba analyzer, " + + "the available values are %s", mode, SUPPORT_MODES); + } + this.segMode = JiebaSegmenter.SegMode.valueOf(mode); + } + + @Override + public Set segment(String text) { + Set result = InsertionOrderUtil.newSet(); + for (SegToken token : JIEBA_SEGMENTER.process(text, this.segMode)) { + result.add(token.word); + } + return result; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/MMSeg4JAnalyzer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/MMSeg4JAnalyzer.java new file mode 100644 index 000000000..3316582f7 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/MMSeg4JAnalyzer.java @@ -0,0 +1,92 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.analyzer; + +import java.io.StringReader; +import java.util.List; +import java.util.Set; + +import org.apache.hugegraph.config.ConfigException; +import org.apache.hugegraph.util.InsertionOrderUtil; + +import org.apache.hugegraph.exception.HugeException; +import com.chenlb.mmseg4j.ComplexSeg; +import com.chenlb.mmseg4j.Dictionary; +import com.chenlb.mmseg4j.MMSeg; +import com.chenlb.mmseg4j.MaxWordSeg; +import com.chenlb.mmseg4j.Seg; +import com.chenlb.mmseg4j.SimpleSeg; +import com.chenlb.mmseg4j.Word; +import com.google.common.collect.ImmutableList; + +/** + * Reference from https://my.oschina.net/apdplat/blog/412921 + */ +public class MMSeg4JAnalyzer implements Analyzer { + + public static final List SUPPORT_MODES = ImmutableList.of( + "Simple", + "Complex", + "MaxWord" + ); + + private static final Dictionary DIC = Dictionary.getInstance(); + + private Seg seg; + + public MMSeg4JAnalyzer(String mode) { + if (!SUPPORT_MODES.contains(mode)) { + throw new ConfigException( + "Unsupported segment mode '%s' for mmseg4j analyzer, " + + "the available values are %s", mode, SUPPORT_MODES); + } + int index = SUPPORT_MODES.indexOf(mode); + switch (index) { + case 0: + this.seg = new SimpleSeg(DIC); + break; + case 1: + this.seg = new ComplexSeg(DIC); + break; + case 2: + this.seg = new MaxWordSeg(DIC); + break; + default: + throw new AssertionError(String.format( + "Unsupported segment mode '%s'", this.seg)); + } + } + + @Override + public Set segment(String text) { + Set result = InsertionOrderUtil.newSet(); + MMSeg mmSeg = new MMSeg(new StringReader(text), this.seg); + try { + Word word = null; + while ((word = mmSeg.next()) != null) { + result.add(word.getString()); + } + } catch (Exception e) { + throw new HugeException("MMSeg4j segment text '%s' failed", + e, text); + } + return result; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/SmartCNAnalyzer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/SmartCNAnalyzer.java new file mode 100644 index 000000000..34c0ea2fb --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/SmartCNAnalyzer.java @@ -0,0 +1,66 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.analyzer; + +import java.io.Reader; +import java.io.StringReader; +import java.util.List; +import java.util.Set; + +import org.apache.hugegraph.util.InsertionOrderUtil; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; + +import org.apache.hugegraph.exception.HugeException; +import com.google.common.collect.ImmutableList; + +/** + * Reference from https://my.oschina.net/apdplat/blog/412921 + */ +public class SmartCNAnalyzer implements Analyzer { + + public static final List SUPPORT_MODES = ImmutableList.of(); + + private static final SmartChineseAnalyzer ANALYZER = + new SmartChineseAnalyzer(); + + public SmartCNAnalyzer(String mode) { + // pass + } + + @Override + public Set segment(String text) { + Set result = InsertionOrderUtil.newSet(); + Reader reader = new StringReader(text); + try (TokenStream tokenStream = ANALYZER.tokenStream("text", reader)) { + tokenStream.reset(); + CharTermAttribute term = null; + while (tokenStream.incrementToken()) { + term = tokenStream.getAttribute(CharTermAttribute.class); + result.add(term.toString()); + } + } catch (Exception e) { + throw new HugeException("SmartCN segment text '%s' failed", + e, text); + } + return result; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/WordAnalyzer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/WordAnalyzer.java new file mode 100644 index 000000000..0a7ebd07f --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/analyzer/WordAnalyzer.java @@ -0,0 +1,74 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.analyzer; + +import java.util.List; +import java.util.Set; + +import org.apache.hugegraph.config.ConfigException; +import org.apache.hugegraph.util.InsertionOrderUtil; +import org.apdplat.word.WordSegmenter; +import org.apdplat.word.segmentation.SegmentationAlgorithm; +import org.apdplat.word.segmentation.Word; + +import com.google.common.collect.ImmutableList; + +/** + * Reference from https://my.oschina.net/apdplat/blog/412921 + */ +public class WordAnalyzer implements Analyzer { + + public static final List SUPPORT_MODES = + ImmutableList.builder() + .add("MaximumMatching") + .add("ReverseMaximumMatching") + .add("MinimumMatching") + .add("ReverseMinimumMatching") + .add("BidirectionalMaximumMatching") + .add("BidirectionalMinimumMatching") + .add("BidirectionalMaximumMinimumMatching") + .add("FullSegmentation") + .add("MinimalWordCount") + .add("MaxNgramScore") + .add("PureEnglish") + .build(); + + private SegmentationAlgorithm algorithm; + + public WordAnalyzer(String mode) { + try { + this.algorithm = SegmentationAlgorithm.valueOf(mode); + } catch (Exception e) { + throw new ConfigException( + "Unsupported segment mode '%s' for word analyzer, " + + "the available values are %s", e, mode, SUPPORT_MODES); + } + } + + @Override + public Set segment(String text) { + Set result = InsertionOrderUtil.newSet(); + List words = WordSegmenter.segWithStopWords(text, this.algorithm); + for (Word word : words) { + result.add(word.getText()); + } + return result; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/auth/AuthConstant.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/auth/AuthConstant.java new file mode 100644 index 000000000..97bd1a0e1 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/auth/AuthConstant.java @@ -0,0 +1,30 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.auth; + +public interface AuthConstant { + + /* + * Fields in token + */ + String TOKEN_USER_NAME = "user_name"; + String TOKEN_USER_ID = "user_id"; + String TOKEN_USER_PASSWORD = "user_password"; +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/auth/TokenGenerator.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/auth/TokenGenerator.java new file mode 100644 index 000000000..f803894fc --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/auth/TokenGenerator.java @@ -0,0 +1,70 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.auth; + +import org.apache.hugegraph.options.AuthOptions; + +import io.jsonwebtoken.*; +import io.jsonwebtoken.security.Keys; +import jakarta.ws.rs.NotAuthorizedException; + +import org.apache.hugegraph.config.HugeConfig; + +import javax.crypto.SecretKey; + +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.Map; + +public class TokenGenerator { + + private final SecretKey key; + + public TokenGenerator(HugeConfig config) { + String secretKey = config.get(AuthOptions.AUTH_TOKEN_SECRET); + this.key = Keys.hmacShaKeyFor(secretKey.getBytes(StandardCharsets.UTF_8)); + } + + public TokenGenerator(String secretKey) { + this.key = Keys.hmacShaKeyFor(secretKey.getBytes(StandardCharsets.UTF_8)); + } + + public String create(Map payload, long expire) { + return Jwts.builder() + .setClaims(payload) + .setExpiration(new Date(System.currentTimeMillis() + expire)) + .signWith(this.key, SignatureAlgorithm.HS256) + .compact(); + } + + public Claims verify(String token) { + try { + Jws claimsJws = Jwts.parserBuilder() + .setSigningKey(key) + .build() + .parseClaimsJws(token); + return claimsJws.getBody(); + } catch (ExpiredJwtException e) { + throw new NotAuthorizedException("The token is expired", e); + } catch (JwtException e) { + throw new NotAuthorizedException("Invalid token", e); + } + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/backend/BackendColumn.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/backend/BackendColumn.java new file mode 100644 index 000000000..342f3ff60 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/backend/BackendColumn.java @@ -0,0 +1,69 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.backend; + +import java.util.Arrays; + +import org.apache.hugegraph.util.Bytes; + +import org.apache.hugegraph.util.StringEncoding; + +public class BackendColumn implements Comparable { + + public byte[] name; + public byte[] value; + + public static BackendColumn of(byte[] name, byte[] value) { + BackendColumn col = new BackendColumn(); + col.name = name; + col.value = value; + return col; + } + + @Override + public String toString() { + return String.format("%s=%s", + StringEncoding.decode(name), + StringEncoding.decode(value)); + } + + @Override + public int compareTo(BackendColumn other) { + if (other == null) { + return 1; + } + return Bytes.compare(this.name, other.name); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof BackendColumn)) { + return false; + } + BackendColumn other = (BackendColumn) obj; + return Bytes.equals(this.name, other.name) && + Bytes.equals(this.value, other.value); + } + + @Override + public int hashCode() { + return Arrays.hashCode(this.name) | Arrays.hashCode(this.value); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/backend/BinaryId.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/backend/BinaryId.java new file mode 100644 index 000000000..685a934fd --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/backend/BinaryId.java @@ -0,0 +1,103 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.backend; + +import org.apache.hugegraph.id.Id; + +import org.apache.hugegraph.util.Bytes; +import org.apache.hugegraph.util.E; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +public final class BinaryId implements Id { + + private final byte[] bytes; + private final Id id; + + public BinaryId(byte[] bytes, Id id) { + this.bytes = bytes; + this.id = id; + } + + public Id origin() { + return this.id; + } + + @Override + public IdType type() { + return IdType.UNKNOWN; + } + + @Override + public Object asObject() { + return ByteBuffer.wrap(this.bytes); + } + + @Override + public String asString() { + throw new UnsupportedOperationException(); + } + + @Override + public long asLong() { + throw new UnsupportedOperationException(); + } + + @Override + public int compareTo(Id other) { + return Bytes.compare(this.bytes, other.asBytes()); + } + + @Override + public byte[] asBytes() { + return this.bytes; + } + + public byte[] asBytes(int offset) { + E.checkArgument(offset < this.bytes.length, + "Invalid offset %s, must be < length %s", + offset, this.bytes.length); + return Arrays.copyOfRange(this.bytes, offset, this.bytes.length); + } + + @Override + public int length() { + return this.bytes.length; + } + + @Override + public int hashCode() { + return ByteBuffer.wrap(this.bytes).hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof BinaryId)) { + return false; + } + return Arrays.equals(this.bytes, ((BinaryId) other).bytes); + } + + @Override + public String toString() { + return "0x" + Bytes.toHex(this.bytes); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/backend/Shard.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/backend/Shard.java new file mode 100644 index 000000000..7d69166c6 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/backend/Shard.java @@ -0,0 +1,71 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.backend; + +/** + * Shard is used for backend storage (like cassandra, hbase) scanning + * operations. Each shard represents a range of tokens for a node. + * Reading data from a given shard does not cross multiple nodes. + */ +public class Shard { + + // token range start + private String start; + // token range end + private String end; + // partitions count in this range + private long length; + + public Shard(String start, String end, long length) { + this.start = start; + this.end = end; + this.length = length; + } + + public String start() { + return this.start; + } + + public void start(String start) { + this.start = start; + } + + public String end() { + return this.end; + } + + public void end(String end) { + this.end = end; + } + + public long length() { + return this.length; + } + + public void length(long length) { + this.length = length; + } + + @Override + public String toString() { + return String.format("Shard{start=%s, end=%s, length=%s}", + this.start, this.end, this.length); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/BackendException.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/BackendException.java new file mode 100644 index 000000000..3fffd5ea1 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/BackendException.java @@ -0,0 +1,53 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.exception; + +public class BackendException extends HugeException { + + private static final long serialVersionUID = -1947589125372576298L; + + public BackendException(String message) { + super(message); + } + + public BackendException(String message, Throwable cause) { + super(message, cause); + } + + public BackendException(String message, Object... args) { + super(message, args); + } + + public BackendException(String message, Throwable cause, Object... args) { + super(message, cause, args); + } + + public BackendException(Throwable cause) { + this("Exception in backend", cause); + } + + public static final void check(boolean expression, + String message, Object... args) + throws BackendException { + if (!expression) { + throw new BackendException(message, args); + } + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/ErrorCodeProvider.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/ErrorCodeProvider.java new file mode 100644 index 000000000..d5034b703 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/ErrorCodeProvider.java @@ -0,0 +1,27 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.exception; + +public interface ErrorCodeProvider { + + public String format(Object... args); + + public String with(String message); +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/HugeException.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/HugeException.java new file mode 100644 index 000000000..b7d8a4588 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/HugeException.java @@ -0,0 +1,70 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.exception; + +public class HugeException extends RuntimeException { + + private static final long serialVersionUID = -8711375282196157058L; + + public HugeException(String message) { + super(message); + } + + public HugeException(ErrorCodeProvider code, String message) { + super(code.with(message)); + } + + public HugeException(String message, Throwable cause) { + super(message, cause); + } + + public HugeException(ErrorCodeProvider code, String message, Throwable cause) { + super(code.with(message), cause); + } + + public HugeException(String message, Object... args) { + super(String.format(message, args)); + } + + public HugeException(ErrorCodeProvider code, Object... args) { + super(code.format(args)); + } + + public HugeException(String message, Throwable cause, Object... args) { + super(String.format(message, args), cause); + } + + public HugeException(ErrorCodeProvider code, Throwable cause, Object... args) { + super(code.format(args), cause); + } + + public Throwable rootCause() { + return rootCause(this); + } + + public static Throwable rootCause(Throwable e) { + Throwable cause = e; + while (cause.getCause() != null) { + cause = cause.getCause(); + } + return cause; + } + +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/LimitExceedException.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/LimitExceedException.java new file mode 100644 index 000000000..10652dca2 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/LimitExceedException.java @@ -0,0 +1,33 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.exception; + +public class LimitExceedException extends HugeException { + + private static final long serialVersionUID = 7384276720045597709L; + + public LimitExceedException(String message) { + super(message); + } + + public LimitExceedException(String message, Object... args) { + super(message, args); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotAllowException.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotAllowException.java new file mode 100644 index 000000000..3781b6d48 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotAllowException.java @@ -0,0 +1,33 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.exception; + +public class NotAllowException extends HugeException { + + private static final long serialVersionUID = -1407924451828873200L; + + public NotAllowException(String message) { + super(message); + } + + public NotAllowException(String message, Object... args) { + super(message, args); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotFoundException.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotFoundException.java new file mode 100644 index 000000000..8567ceb01 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotFoundException.java @@ -0,0 +1,37 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.exception; + +public class NotFoundException extends HugeException { + + private static final long serialVersionUID = -5912665926327173032L; + + public NotFoundException(String message) { + super(message); + } + + public NotFoundException(String message, Object... args) { + super(message, args); + } + + public NotFoundException(String message, Throwable cause, Object... args) { + super(message, cause, args); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotSupportException.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotSupportException.java new file mode 100644 index 000000000..49d3dad49 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/exception/NotSupportException.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.exception; + +public class NotSupportException extends HugeException { + + private static final long serialVersionUID = -2914329541122906234L; + private static final String PREFIX = "Not support "; + + public NotSupportException(String message) { + super(PREFIX + message); + } + + public NotSupportException(String message, Object... args) { + super(PREFIX + message, args); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/id/EdgeId.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/id/EdgeId.java new file mode 100644 index 000000000..2b03e97d3 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/id/EdgeId.java @@ -0,0 +1,350 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.id; + +import org.apache.hugegraph.perf.PerfUtil.Watched; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.util.E; + +import org.apache.hugegraph.exception.NotFoundException; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.Directions; +import org.apache.hugegraph.type.define.HugeKeys; +import org.apache.hugegraph.util.StringEncoding; + +/** + * Class used to format and parse id of edge, the edge id consists of: + * EdgeId = { source-vertex-id > direction > parentEdgeLabelId > subEdgeLabelId + * >sortKeys > target-vertex-id } + * NOTE: + * 1. for edges with edgeLabel-type=NORMAL,edgelabelId=parentEdgeLabelId=subEdgeLabelId, + * for edges with edgeLabel type=PARENT，edgelabelId = subEdgeLabelId , + * parentEdgeLabelId = edgelabelId.fatherId + * + * 2.if we use `entry.type()` which is IN or OUT as a part of id, + * an edge's id will be different due to different directions (belongs + * to 2 owner vertex) + */ +public class EdgeId implements Id { + + public static final HugeKeys[] KEYS = new HugeKeys[] { + HugeKeys.OWNER_VERTEX, + HugeKeys.DIRECTION, + HugeKeys.LABEL, + HugeKeys.SUB_LABEL, + HugeKeys.SORT_VALUES, + HugeKeys.OTHER_VERTEX + }; + + private final Id ownerVertexId; + private final Directions direction; + private final Id edgeLabelId; + private final Id subLabelId; + private final String sortValues; + private final Id otherVertexId; + + private final boolean directed; + private String cache; + + + public EdgeId(Id ownerVertexId, Directions direction, Id edgeLabelId, + Id subLabelId, String sortValues, + Id otherVertexId) { + this(ownerVertexId, direction, edgeLabelId, + subLabelId, sortValues, otherVertexId, false); + } + + public EdgeId(Id ownerVertexId, Directions direction, Id edgeLabelId, + Id subLabelId, String sortValues, + Id otherVertexId, boolean directed) { + this.ownerVertexId = ownerVertexId; + this.direction = direction; + this.edgeLabelId = edgeLabelId; + this.sortValues = sortValues; + this.subLabelId = subLabelId; + this.otherVertexId = otherVertexId; + this.directed = directed; + this.cache = null; + } + + @Watched + public EdgeId switchDirection() { + Directions direction = this.direction.opposite(); + return new EdgeId(this.otherVertexId, direction, this.edgeLabelId, + this.subLabelId, this.sortValues, this.ownerVertexId, + this.directed); + } + + public EdgeId directed(boolean directed) { + return new EdgeId(this.ownerVertexId, this.direction, this.edgeLabelId, + this.subLabelId, this.sortValues, this.otherVertexId, directed); + } + + private Id sourceVertexId() { + return this.direction == Directions.OUT ? + this.ownerVertexId : + this.otherVertexId; + } + + private Id targetVertexId() { + return this.direction == Directions.OUT ? + this.otherVertexId : + this.ownerVertexId; + } + + public Id subLabelId(){ + return this.subLabelId; + } + + public Id ownerVertexId() { + return this.ownerVertexId; + } + + public Id edgeLabelId() { + return this.edgeLabelId; + } + + public Directions direction() { + return this.direction; + } + + public byte directionCode() { + return directionToCode(this.direction); + } + + public String sortValues() { + return this.sortValues; + } + + public Id otherVertexId() { + return this.otherVertexId; + } + + @Override + public Object asObject() { + return this.asString(); + } + + @Override + public String asString() { + if (this.cache != null) { + return this.cache; + } + if (this.directed) { + this.cache = SplicingIdGenerator.concat( + IdUtil.writeString(this.ownerVertexId), + this.direction.type().string(), + IdUtil.writeLong(this.edgeLabelId), + IdUtil.writeLong(this.subLabelId), + this.sortValues, + IdUtil.writeString(this.otherVertexId)); + } else { + this.cache = SplicingIdGenerator.concat( + IdUtil.writeString(this.sourceVertexId()), + IdUtil.writeLong(this.edgeLabelId), + IdUtil.writeLong(this.subLabelId), + this.sortValues, + IdUtil.writeString(this.targetVertexId())); + } + return this.cache; + } + + @Override + public long asLong() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] asBytes() { + return StringEncoding.encode(this.asString()); + } + + @Override + public int length() { + return this.asString().length(); + } + + @Override + public IdType type() { + return IdType.EDGE; + } + + @Override + public int compareTo(Id other) { + return this.asString().compareTo(other.asString()); + } + + @Override + public int hashCode() { + if (this.directed) { + return this.ownerVertexId.hashCode() ^ + this.direction.hashCode() ^ + this.edgeLabelId.hashCode() ^ + this.subLabelId.hashCode() ^ + this.sortValues.hashCode() ^ + this.otherVertexId.hashCode(); + } else { + return this.sourceVertexId().hashCode() ^ + this.edgeLabelId.hashCode() ^ + this.subLabelId.hashCode() ^ + this.sortValues.hashCode() ^ + this.targetVertexId().hashCode(); + } + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof EdgeId)) { + return false; + } + EdgeId other = (EdgeId) object; + if (this.directed) { + return this.ownerVertexId.equals(other.ownerVertexId) && + this.direction == other.direction && + this.edgeLabelId.equals(other.edgeLabelId) && + this.sortValues.equals(other.sortValues) && + this.subLabelId.equals(other.subLabelId) && + this.otherVertexId.equals(other.otherVertexId); + } else { + return this.sourceVertexId().equals(other.sourceVertexId()) && + this.edgeLabelId.equals(other.edgeLabelId) && + this.sortValues.equals(other.sortValues) && + this.subLabelId.equals(other.subLabelId) && + this.targetVertexId().equals(other.targetVertexId()); + } + } + + @Override + public String toString() { + return this.asString(); + } + + public static byte directionToCode(Directions direction) { + return direction.type().code(); + } + + public static Directions directionFromCode(byte code) { + return (code == HugeType.EDGE_OUT.code()) ? Directions.OUT : Directions.IN; + } + + public static boolean isOutDirectionFromCode(byte code) { + return code == HugeType.EDGE_OUT.code(); + } + + public static EdgeId parse(String id) throws NotFoundException { + return parse(id, false); + } + + public static EdgeId parse(String id, boolean returnNullIfError) + throws NotFoundException { + String[] idParts = SplicingIdGenerator.split(id); + if (!(idParts.length == 5 || idParts.length == 6)) { + if (returnNullIfError) { + return null; + } + throw new NotFoundException("Edge id must be formatted as 5~6 " + + "parts, but got %s parts: '%s'", + idParts.length, id); + } + try { + if (idParts.length == 5) { + Id ownerVertexId = IdUtil.readString(idParts[0]); + Id edgeLabelId = IdUtil.readLong(idParts[1]); + Id subLabelId = IdUtil.readLong(idParts[2]); + String sortValues = idParts[3]; + Id otherVertexId = IdUtil.readString(idParts[4]); + return new EdgeId(ownerVertexId, Directions.OUT, edgeLabelId, + subLabelId, sortValues, otherVertexId); + } else { + assert idParts.length == 6; + Id ownerVertexId = IdUtil.readString(idParts[0]); + HugeType direction = HugeType.fromString(idParts[1]); + Id edgeLabelId = IdUtil.readLong(idParts[2]); + Id subLabelId = IdUtil.readLong(idParts[3]); + String sortValues = idParts[4]; + Id otherVertexId = IdUtil.readString(idParts[5]); + return new EdgeId(ownerVertexId, Directions.convert(direction), + edgeLabelId, subLabelId, + sortValues, otherVertexId); + } + } catch (Throwable e) { + if (returnNullIfError) { + return null; + } + throw new NotFoundException("Invalid format of edge id '%s'", + e, id); + } + } + + public static Id parseStoredString(String id) { + String[] idParts = split(id); + E.checkArgument(idParts.length == 5, "Invalid id format: %s", id); + Id ownerVertexId = IdUtil.readStoredString(idParts[0]); + Id edgeLabelId = IdGenerator.ofStoredString(idParts[1], IdType.LONG); + Id subLabelId = IdGenerator.ofStoredString(idParts[2], IdType.LONG); + String sortValues = idParts[3]; + Id otherVertexId = IdUtil.readStoredString(idParts[4]); + return new EdgeId(ownerVertexId, Directions.OUT, edgeLabelId, + subLabelId, sortValues, otherVertexId); + } + + public static String asStoredString(Id id) { + EdgeId eid = (EdgeId) id; + return SplicingIdGenerator.concat( + IdUtil.writeStoredString(eid.sourceVertexId()), + IdGenerator.asStoredString(eid.edgeLabelId()), + IdGenerator.asStoredString(eid.subLabelId()), + eid.sortValues(), + IdUtil.writeStoredString(eid.targetVertexId())); + } + + public static String concat(String... ids) { + return SplicingIdGenerator.concat(ids); + } + + public static String[] split(Id id) { + return EdgeId.split(id.asString()); + } + + public static String[] split(String id) { + return SplicingIdGenerator.split(id); + } + + + public static void main(String[] args) { + EdgeId edgeId1 = new EdgeId(IdGenerator.of("1:marko"), Directions.OUT, + IdGenerator.of(1), + IdGenerator.of(1), "", + IdGenerator.of("1:josh")); + EdgeId edgeId2 = new EdgeId(IdGenerator.of("1:marko"), Directions.OUT, + IdGenerator.of(1), + IdGenerator.of(1), "", + IdGenerator.of("1:josh")); + EdgeId edgeId3 = new EdgeId(IdGenerator.of("1:josh"), Directions.IN, + IdGenerator.of(1), + IdGenerator.of(1), "", + IdGenerator.of("1:marko")); + Assert.assertTrue(edgeId1.equals(edgeId2)); + Assert.assertTrue(edgeId2.equals(edgeId1)); + Assert.assertTrue(edgeId1.equals(edgeId3)); + Assert.assertTrue(edgeId3.equals(edgeId1)); + } + +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/id/Id.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/id/Id.java new file mode 100644 index 000000000..aeb7810a9 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/id/Id.java @@ -0,0 +1,90 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.id; + +import java.io.Serializable; + +import org.apache.hugegraph.util.E; + +public interface Id extends Comparable, Serializable { + + public static final int UUID_LENGTH = 16; + + public Object asObject(); + + public String asString(); + + public long asLong(); + + public byte[] asBytes(); + + public int length(); + + public IdType type(); + + public default boolean number() { + return this.type() == IdType.LONG; + } + + public default boolean uuid() { + return this.type() == IdType.UUID; + } + + public default boolean string() { + return this.type() == IdType.STRING; + } + + public default boolean edge() { + return this.type() == IdType.EDGE; + } + + public enum IdType { + + UNKNOWN, + LONG, + UUID, + STRING, + EDGE; + + public char prefix() { + if (this == UNKNOWN) { + return 'N'; + } + return this.name().charAt(0); + } + + public static IdType valueOfPrefix(String id) { + E.checkArgument(id != null && id.length() > 0, + "Invalid id '%s'", id); + switch (id.charAt(0)) { + case 'L': + return IdType.LONG; + case 'U': + return IdType.UUID; + case 'S': + return IdType.STRING; + case 'E': + return IdType.EDGE; + default: + return IdType.UNKNOWN; + } + } + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/id/IdGenerator.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/id/IdGenerator.java new file mode 100644 index 000000000..b6687262d --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/id/IdGenerator.java @@ -0,0 +1,465 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.id; + +import org.apache.hugegraph.serializer.BytesBuffer; +import org.apache.hugegraph.structure.BaseVertex; +import org.apache.hugegraph.util.StringEncoding; +import com.google.common.primitives.Longs; + +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.LongEncoding; +import org.apache.hugegraph.util.NumericUtil; + +import java.nio.charset.Charset; +import java.util.Objects; +import java.util.UUID; + +public abstract class IdGenerator { + + public static final Id ZERO = IdGenerator.of(0L); + + public abstract Id generate(BaseVertex vertex); + + public final static Id of(String id) { + return new StringId(id); + } + + public final static Id of(UUID id) { + return new UuidId(id); + } + + public final static Id of(String id, boolean uuid) { + return uuid ? new UuidId(id) : new StringId(id); + } + + public final static Id of(long id) { + return new LongId(id); + } + + public static Id of(Object id) { + if (id instanceof Id) { + return (Id) id; + } else if (id instanceof String) { + return of((String) id); + } else if (id instanceof Number) { + return of(((Number) id).longValue()); + } else if (id instanceof UUID) { + return of((UUID) id); + } + return new ObjectId(id); + } + + public final static Id of(byte[] bytes, Id.IdType type) { + switch (type) { + case LONG: + return new LongId(bytes); + case UUID: + return new UuidId(bytes); + case STRING: + return new StringId(bytes); + default: + throw new AssertionError("Invalid id type " + type); + } + } + + public final static Id ofStoredString(String id, Id.IdType type) { + switch (type) { + case LONG: + return of(LongEncoding.decodeSignedB64(id)); + case UUID: + byte[] bytes = StringEncoding.decodeBase64(id); + return of(bytes, Id.IdType.UUID); + case STRING: + return of(id); + default: + throw new AssertionError("Invalid id type " + type); + } + } + + public final static String asStoredString(Id id) { + switch (id.type()) { + case LONG: + return LongEncoding.encodeSignedB64(id.asLong()); + case UUID: + return StringEncoding.encodeBase64(id.asBytes()); + case STRING: + return id.asString(); + default: + throw new AssertionError("Invalid id type " + id.type()); + } + } + + public final static Id.IdType idType(Id id) { + if (id instanceof LongId) { + return Id.IdType.LONG; + } + if (id instanceof UuidId) { + return Id.IdType.UUID; + } + if (id instanceof StringId) { + return Id.IdType.STRING; + } + if (id instanceof EdgeId) { + return Id.IdType.EDGE; + } + return Id.IdType.UNKNOWN; + } + + private final static int compareType(Id id1, Id id2) { + return idType(id1).ordinal() - idType(id2).ordinal(); + } + + /****************************** id defines ******************************/ + + public static final class StringId implements Id { + + private final String id; + private static final Charset CHARSET = Charset.forName("UTF-8"); + + public StringId(String id) { + E.checkArgument(!id.isEmpty(), "The id can't be empty"); + this.id = id; + } + + public StringId(byte[] bytes) { + this.id = StringEncoding.decode(bytes); + } + + @Override + public IdType type() { + return IdType.STRING; + } + + @Override + public Object asObject() { + return this.id; + } + + @Override + public String asString() { + return this.id; + } + + @Override + public long asLong() { + return Long.parseLong(this.id); + } + + @Override + public byte[] asBytes() { + return this.id.getBytes(CHARSET); + } + + @Override + public int length() { + return this.id.length(); + } + + @Override + public int compareTo(Id other) { + int cmp = compareType(this, other); + if (cmp != 0) { + return cmp; + } + return this.id.compareTo(other.asString()); + } + + @Override + public int hashCode() { + return this.id.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof StringId)) { + return false; + } + return this.id.equals(((StringId) other).id); + } + + @Override + public String toString() { + return this.id; + } + } + + public static final class LongId extends Number implements Id { + + private static final long serialVersionUID = -7732461469037400190L; + + private final long id; + + public LongId(long id) { + this.id = id; + } + + public LongId(byte[] bytes) { + this.id = NumericUtil.bytesToLong(bytes); + } + + @Override + public IdType type() { + return IdType.LONG; + } + + @Override + public Object asObject() { + return this.id; + } + + @Override + public String asString() { + // TODO: encode with base64 + return Long.toString(this.id); + } + + @Override + public long asLong() { + return this.id; + } + + @Override + public byte[] asBytes() { + return Longs.toByteArray(this.id); + // return NumericUtil.longToBytes(this.id); + } + + @Override + public int length() { + return Long.BYTES; + } + + @Override + public int compareTo(Id other) { + int cmp = compareType(this, other); + if (cmp != 0) { + return cmp; + } + return Long.compare(this.id, other.asLong()); + } + + @Override + public int hashCode() { + return Long.hashCode(this.id); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Number)) { + if (idDigitalObject(other)) { + return this.id == (long) Double.parseDouble(other.toString()); + } + return false; + } + return this.id == ((Number) other).longValue(); + } + + private static boolean idDigitalObject(Object object) { + String string = object.toString(); + for (int i = string.length(); --i >= 0; ) { + char c = string.charAt(i); + if (!Character.isDigit(c) && + '.' != c) { + return false; + } + } + return true; + } + + @Override + public String toString() { + return String.valueOf(this.id); + } + + @Override + public int intValue() { + return (int) this.id; + } + + @Override + public long longValue() { + return this.id; + } + + @Override + public float floatValue() { + return this.id; + } + + @Override + public double doubleValue() { + return this.id; + } + } + + public static final class UuidId implements Id { + + private final UUID uuid; + + public UuidId(String string) { + this(StringEncoding.uuid(string)); + } + + public UuidId(byte[] bytes) { + this(fromBytes(bytes)); + } + + public UuidId(UUID uuid) { + E.checkArgument(uuid != null, "The uuid can't be null"); + this.uuid = uuid; + } + + @Override + public IdType type() { + return IdType.UUID; + } + + @Override + public Object asObject() { + return this.uuid; + } + + @Override + public String asString() { + return this.uuid.toString(); + } + + @Override + public long asLong() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] asBytes() { + BytesBuffer buffer = BytesBuffer.allocate(16); + buffer.writeLong(this.uuid.getMostSignificantBits()); + buffer.writeLong(this.uuid.getLeastSignificantBits()); + return buffer.bytes(); + } + + private static UUID fromBytes(byte[] bytes) { + E.checkArgument(bytes != null, "The UUID can't be null"); + BytesBuffer buffer = BytesBuffer.wrap(bytes); + long high = buffer.readLong(); + long low = buffer.readLong(); + return new UUID(high, low); + } + + @Override + public int length() { + return UUID_LENGTH; + } + + @Override + public int compareTo(Id other) { + E.checkNotNull(other, "compare id"); + int cmp = compareType(this, other); + if (cmp != 0) { + return cmp; + } + return this.uuid.compareTo(((UuidId) other).uuid); + } + + @Override + public int hashCode() { + return this.uuid.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof UuidId)) { + return false; + } + return this.uuid.equals(((UuidId) other).uuid); + } + + @Override + public String toString() { + return this.uuid.toString(); + } + } + + /** + * This class is just used by backend store for wrapper object as Id + */ + public static final class ObjectId implements Id { + + private final Object object; + + public ObjectId(Object object) { + E.checkNotNull(object, "object"); + this.object = object; + } + + @Override + public IdType type() { + return IdType.UNKNOWN; + } + + @Override + public Object asObject() { + return this.object; + } + + @Override + public String asString() { + throw new UnsupportedOperationException(); + } + + @Override + public long asLong() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] asBytes() { + throw new UnsupportedOperationException(); + } + + @Override + public int length() { + throw new UnsupportedOperationException(); + } + + @Override + public int compareTo(Id o) { + throw new UnsupportedOperationException(); + } + + @Override + public int hashCode() { + return this.object.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ObjectId)) { + return false; + } + return Objects.equals(this.object, ((ObjectId) other).object); + } + + @Override + public String toString() { + return this.object.toString(); + } + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/id/IdUtil.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/id/IdUtil.java new file mode 100644 index 000000000..b394c79a1 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/id/IdUtil.java @@ -0,0 +1,162 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.id; + +import java.nio.ByteBuffer; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.hugegraph.serializer.BytesBuffer; + +public final class IdUtil { + + public static String writeStoredString(Id id) { + String idString; + switch (id.type()) { + case LONG: + case STRING: + case UUID: + idString = IdGenerator.asStoredString(id); + break; + case EDGE: + idString = EdgeId.asStoredString(id); + break; + default: + throw new AssertionError("Invalid id type " + id.type()); + } + return id.type().prefix() + idString; + } + + public static Id readStoredString(String id) { + Id.IdType type = Id.IdType.valueOfPrefix(id); + String idContent = id.substring(1); + switch (type) { + case LONG: + case STRING: + case UUID: + return IdGenerator.ofStoredString(idContent, type); + case EDGE: + return EdgeId.parseStoredString(idContent); + default: + throw new IllegalArgumentException("Invalid id: " + id); + } + } + + public static Object writeBinString(Id id) { + int len = id.edge() ? BytesBuffer.BUF_EDGE_ID : id.length() + 1; + BytesBuffer buffer = BytesBuffer.allocate(len).writeId(id); + buffer.forReadWritten(); + return buffer.asByteBuffer(); + } + + public static Id readBinString(Object id) { + BytesBuffer buffer = BytesBuffer.wrap((ByteBuffer) id); + return buffer.readId(); + } + + public static byte[] asBytes(Id id) { + int len = id.edge() ? BytesBuffer.BUF_EDGE_ID : id.length() + 1; + BytesBuffer buffer = BytesBuffer.allocate(len).writeId(id); + return buffer.bytes(); + } + + public static Id fromBytes(byte[] bytes) { + BytesBuffer buffer = BytesBuffer.wrap(bytes); + return buffer.readId(); + } + + + public static String writeString(Id id) { + String idString = id.asString(); + StringBuilder sb = new StringBuilder(1 + idString.length()); + sb.append(id.type().prefix()).append(idString); + return sb.toString(); + } + + public static Id readString(String id) { + Id.IdType type = Id.IdType.valueOfPrefix(id); + String idContent = id.substring(1); + switch (type) { + case LONG: + return IdGenerator.of(Long.parseLong(idContent)); + case STRING: + case UUID: + return IdGenerator.of(idContent, type == Id.IdType.UUID); + case EDGE: + return EdgeId.parse(idContent); + default: + throw new IllegalArgumentException("Invalid id: " + id); + } + } + + public static String writeLong(Id id) { + return String.valueOf(id.asLong()); + } + + public static Id readLong(String id) { + return IdGenerator.of(Long.parseLong(id)); + } + + public static String escape(char splitor, char escape, String... values) { + int length = values.length + 4; + for (String value : values) { + length += value.length(); + } + StringBuilder escaped = new StringBuilder(length); + // Do escape for every item in values + for (String value : values) { + if (escaped.length() > 0) { + escaped.append(splitor); + } + + if (value.indexOf(splitor) == -1) { + escaped.append(value); + continue; + } + + // Do escape for current item + for (int i = 0, n = value.length(); i < n; i++) { + char ch = value.charAt(i); + if (ch == splitor) { + escaped.append(escape); + } + escaped.append(ch); + } + } + return escaped.toString(); + } + + public static String[] unescape(String id, String splitor, String escape) { + /* + * Note that the `splitor`/`escape` maybe special characters in regular + * expressions, but this is a frequently called method, for faster + * execution, we forbid the use of special characters as delimiter + * or escape sign. + * The `limit` param -1 in split method can ensure empty string be + * splited to a part. + */ + String[] parts = id.split("(?'; + private static final char ID_SPLITOR = ':'; + private static final char NAME_SPLITOR = '!'; + + public static final String ESCAPE_STR = String.valueOf(ESCAPE); + public static final String IDS_SPLITOR_STR = String.valueOf(IDS_SPLITOR); + public static final String ID_SPLITOR_STR = String.valueOf(ID_SPLITOR); + + /****************************** id generate ******************************/ + + /** + * Generate a string id of HugeVertex from Vertex name + */ + @Override + public Id generate(BaseVertex vertex) { + /* + * Hash for row-key which will be evenly distributed. + * We can also use LongEncoding.encode() to encode the int/long hash + * if needed. + * id = String.format("%s%s%s", HashUtil.hash(id), ID_SPLITOR, id); + */ + // TODO: use binary Id with binary fields instead of string id + return splicing(vertex.schemaLabel().id().asString(), vertex.name()); + } + + /** + * Concat multiple ids into one composite id with IDS_SPLITOR + * @param ids the string id values to be concatted + * @return concatted string value + */ + public static String concat(String... ids) { + // NOTE: must support string id when using this method + return IdUtil.escape(IDS_SPLITOR, ESCAPE, ids); + } + + /** + * Split a composite id into multiple ids with IDS_SPLITOR + * @param ids the string id value to be splitted + * @return splitted string values + */ + public static String[] split(String ids) { + return IdUtil.unescape(ids, IDS_SPLITOR_STR, ESCAPE_STR); + } + + /** + * Concat property values with NAME_SPLITOR + * @param values the property values to be concatted + * @return concatted string value + */ + public static String concatValues(List values) { + // Convert the object list to string array + int valuesSize = values.size(); + String[] parts = new String[valuesSize]; + for (int i = 0; i < valuesSize; i++) { + parts[i] = values.get(i).toString(); + } + return IdUtil.escape(NAME_SPLITOR, ESCAPE, parts); + } + + /** + * Concat property values with NAME_SPLITOR + * @param values the property values to be concatted + * @return concatted string value + */ + public static String concatValues(Object... values) { + return concatValues(Arrays.asList(values)); + } + + /** + * Concat multiple parts into a single id with ID_SPLITOR + * @param parts the string id values to be spliced + * @return spliced id object + */ + public static Id splicing(String... parts) { + String escaped = IdUtil.escape(ID_SPLITOR, ESCAPE, parts); + return IdGenerator.of(escaped); + } + + public static Id splicingWithNoEscape(String... parts) { + String escaped = String.join(ID_SPLITOR_STR, parts); + return IdGenerator.of(escaped); + } + + public static Id generateBinaryId(Id id) { + if (id instanceof BinaryId) { + return id; + } + BytesBuffer buffer = BytesBuffer.allocate(1 + id.length()); + BinaryId binaryId = new BinaryId(buffer.writeId(id).bytes(), id); + return binaryId; + } + + /** + * Parse a single id into multiple parts with ID_SPLITOR + * @param id the id object to be parsed + * @return parsed string id parts + */ + public static String[] parse(Id id) { + return IdUtil.unescape(id.asString(), ID_SPLITOR_STR, ESCAPE_STR); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/options/AuthOptions.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/options/AuthOptions.java new file mode 100644 index 000000000..3ae732e2e --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/options/AuthOptions.java @@ -0,0 +1,153 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.options; + +import org.apache.hugegraph.config.ConfigListOption; +import org.apache.hugegraph.config.ConfigOption; +import org.apache.hugegraph.config.OptionHolder; + +import java.security.SecureRandom; +import java.util.Base64; + +import static org.apache.hugegraph.config.OptionChecker.*; + +public class AuthOptions extends OptionHolder { + + private AuthOptions() { + super(); + } + + private static volatile AuthOptions instance; + + public static synchronized AuthOptions instance() { + if (instance == null) { + instance = new AuthOptions(); + instance.registerOptions(); + } + return instance; + } + + public static final ConfigOption AUTH_TOKEN_SECRET = + new ConfigOption<>( + "auth.token_secret", + "Secret key of HS256 algorithm.", + disallowEmpty(), + "FXQXbJtbCLxODc6tGci732pkH1cyf8Qg" + ); + + public static final ConfigOption AUTH_AUDIT_LOG_RATE = + new ConfigOption<>( + "auth.audit_log_rate", + "The max rate of audit log output per user, " + + "default value is 1000 records per second.", + rangeDouble(0.0, Double.MAX_VALUE), + 1000.0 + ); + + public static final ConfigOption AUTH_PROXY_CACHE_EXPIRE = + new ConfigOption<>( + "auth.proxy_cache_expire", + "The expiration time in seconds of auth cache in " + + "auth client.", + rangeInt(0L, Long.MAX_VALUE), + (1 * 60L) + ); + + public static final ConfigOption AUTH_CACHE_CAPACITY = + new ConfigOption<>( + "auth.cache_capacity", + "The max cache capacity of each auth cache item.", + rangeInt(0L, Long.MAX_VALUE), + (1024 * 10L) + ); + + public static final ConfigOption AUTHENTICATOR = + new ConfigOption<>( + "auth.authenticator", + "The class path of authenticator implementation. " + + "e.g., org.apache.hugegraph.auth.StandardAuthenticator, " + + "or org.apache.hugegraph.auth.ConfigAuthenticator.", + null, + "" + ); + + public static final ConfigOption AUTH_GRAPH_STORE = + new ConfigOption<>( + "auth.graph_store", + "The name of graph used to store authentication information, " + + "like users, only for org.apache.hugegraph.auth.StandardAuthenticator.", + disallowEmpty(), + "hugegraph" + ); + + public static final ConfigOption AUTH_ADMIN_TOKEN = + new ConfigOption<>( + "auth.admin_token", + "Token for administrator operations, " + + "only for org.apache.hugegraph.auth.ConfigAuthenticator.", + disallowEmpty(), + "162f7848-0b6d-4faf-b557-3a0797869c55" + ); + + public static final ConfigListOption AUTH_USER_TOKENS = + new ConfigListOption<>( + "auth.user_tokens", + "The map of user tokens with name and password, " + + "only for org.apache.hugegraph.auth.ConfigAuthenticator.", + disallowEmpty(), + "hugegraph:9fd95c9c-711b-415b-b85f-d4df46ba5c31" + ); + + public static final ConfigOption AUTH_REMOTE_URL = + new ConfigOption<>( + "auth.remote_url", + "If the address is empty, it provide auth service, " + + "otherwise it is auth client and also provide auth service " + + "through rpc forwarding. The remote url can be set to " + + "multiple addresses, which are concat by ','.", + null, + "" + ); + + public static final ConfigOption AUTH_CACHE_EXPIRE = + new ConfigOption<>( + "auth.cache_expire", + "The expiration time in seconds of auth cache in " + + "auth client and auth server.", + rangeInt(0L, Long.MAX_VALUE), + (60 * 10L) + ); + + public static final ConfigOption AUTH_TOKEN_EXPIRE = + new ConfigOption<>( + "auth.token_expire", + "The expiration time in seconds after token created", + rangeInt(0L, Long.MAX_VALUE), + (3600 * 24L) + ); + + private static String generateRandomBase64Key() { + SecureRandom random = new SecureRandom(); + // 32 bytes for HMAC-SHA256 + byte[] bytes = new byte[32]; + random.nextBytes(bytes); + return Base64.getEncoder().encodeToString(bytes); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/options/CoreOptions.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/options/CoreOptions.java new file mode 100644 index 000000000..849539419 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/options/CoreOptions.java @@ -0,0 +1,666 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.options; + +import org.apache.hugegraph.config.ConfigConvOption; +import org.apache.hugegraph.config.ConfigOption; +import org.apache.hugegraph.config.OptionHolder; +import org.apache.hugegraph.query.Query; +import org.apache.hugegraph.type.define.CollectionType; +import org.apache.hugegraph.util.Bytes; + +import static org.apache.hugegraph.config.OptionChecker.*; +import static org.apache.hugegraph.query.Query.COMMIT_BATCH; + +public class CoreOptions extends OptionHolder { + + public static final int CPUS = Runtime.getRuntime().availableProcessors(); + public static final ConfigOption GREMLIN_GRAPH = + new ConfigOption<>( + "gremlin.graph", + "Gremlin entrance to create graph.", + disallowEmpty(), + "org.apache.hugegraph.HugeFactory" + ); + public static final ConfigOption BACKEND = + new ConfigOption<>( + "backend", + "The data store type.", + disallowEmpty(), + "memory" + ); + public static final ConfigOption STORE = + new ConfigOption<>( + "store", + "The database name like Cassandra Keyspace.", + disallowEmpty(), + "hugegraph" + ); + public static final ConfigOption STORE_GRAPH = + new ConfigOption<>( + "store.graph", + "The graph table name, which store vertex, edge and property.", + disallowEmpty(), + "g" + ); + public static final ConfigOption ALIAS_NAME = + new ConfigOption<>( + "alias.graph.id", + "The graph alias id.", + "" + ); + public static final ConfigOption SERIALIZER = + new ConfigOption<>( + "serializer", + "The serializer for backend store, like: text/binary/cassandra.", + disallowEmpty(), + "text" + ); + public static final ConfigOption RAFT_MODE = + new ConfigOption<>( + "raft.mode", + "Whether the backend storage works in raft mode.", + disallowEmpty(), + false + ); + public static final ConfigOption RAFT_SAFE_READ = + new ConfigOption<>( + "raft.safe_read", + "Whether to use linearly consistent read.", + disallowEmpty(), + false + ); + public static final ConfigOption RAFT_PATH = + new ConfigOption<>( + "raft.path", + "The log path of current raft node.", + disallowEmpty(), + "./raftlog" + ); + public static final ConfigOption RAFT_REPLICATOR_PIPELINE = + new ConfigOption<>( + "raft.use_replicator_pipeline", + "Whether to use replicator line, when turned on it " + + "multiple logs can be sent in parallel, and the next log " + + "doesn't have to wait for the ack message of the current " + + "log to be sent.", + disallowEmpty(), + true + ); + public static final ConfigOption RAFT_ELECTION_TIMEOUT = + new ConfigOption<>( + "raft.election_timeout", + "Timeout in milliseconds to launch a round of election.", + rangeInt(0, Integer.MAX_VALUE), + 10000 + ); + public static final ConfigOption RAFT_SNAPSHOT_INTERVAL = + new ConfigOption<>( + "raft.snapshot_interval", + "The interval in seconds to trigger snapshot save.", + rangeInt(0, Integer.MAX_VALUE), + 3600 + ); + public static final ConfigOption RAFT_SNAPSHOT_THREADS = + new ConfigOption<>( + "raft.snapshot_threads", + "The thread number used to do snapshot.", + rangeInt(0, Integer.MAX_VALUE), + 4 + ); + public static final ConfigOption RAFT_SNAPSHOT_PARALLEL_COMPRESS = + new ConfigOption<>( + "raft.snapshot_parallel_compress", + "Whether to enable parallel compress.", + disallowEmpty(), + false + ); + public static final ConfigOption RAFT_SNAPSHOT_COMPRESS_THREADS = + new ConfigOption<>( + "raft.snapshot_compress_threads", + "The thread number used to do snapshot compress.", + rangeInt(0, Integer.MAX_VALUE), + 4 + ); + public static final ConfigOption RAFT_SNAPSHOT_DECOMPRESS_THREADS = + new ConfigOption<>( + "raft.snapshot_decompress_threads", + "The thread number used to do snapshot decompress.", + rangeInt(0, Integer.MAX_VALUE), + 4 + ); + public static final ConfigOption RAFT_BACKEND_THREADS = + new ConfigOption<>( + "raft.backend_threads", + "The thread number used to apply task to backend.", + rangeInt(0, Integer.MAX_VALUE), + CPUS + ); + public static final ConfigOption RAFT_READ_INDEX_THREADS = + new ConfigOption<>( + "raft.read_index_threads", + "The thread number used to execute reading index.", + rangeInt(0, Integer.MAX_VALUE), + 8 + ); + public static final ConfigOption RAFT_READ_STRATEGY = + new ConfigOption<>( + "raft.read_strategy", + "The linearizability of read strategy.", + allowValues("ReadOnlyLeaseBased", "ReadOnlySafe"), + "ReadOnlyLeaseBased" + ); + public static final ConfigOption RAFT_APPLY_BATCH = + new ConfigOption<>( + "raft.apply_batch", + "The apply batch size to trigger disruptor event handler.", + positiveInt(), + // jraft default value is 32 + 1 + ); + public static final ConfigOption RAFT_QUEUE_SIZE = + new ConfigOption<>( + "raft.queue_size", + "The disruptor buffers size for jraft RaftNode, " + + "StateMachine and LogManager.", + positiveInt(), + // jraft default value is 16384 + 16384 + ); + public static final ConfigOption RAFT_QUEUE_PUBLISH_TIMEOUT = + new ConfigOption<>( + "raft.queue_publish_timeout", + "The timeout in second when publish event into disruptor.", + positiveInt(), + // jraft default value is 10(sec) + 60 + ); + public static final ConfigOption RAFT_RPC_THREADS = + new ConfigOption<>( + "raft.rpc_threads", + "The rpc threads for jraft RPC layer", + positiveInt(), + // jraft default value is 80 + Math.max(CPUS * 2, 80) + ); + public static final ConfigOption RAFT_RPC_CONNECT_TIMEOUT = + new ConfigOption<>( + "raft.rpc_connect_timeout", + "The rpc connect timeout for jraft rpc.", + positiveInt(), + // jraft default value is 1000(ms) + 5000 + ); + public static final ConfigOption RAFT_RPC_TIMEOUT = + new ConfigOption<>( + "raft.rpc_timeout", + "The general rpc timeout in seconds for jraft rpc.", + positiveInt(), + // jraft default value is 5s + 60 + ); + public static final ConfigOption RAFT_INSTALL_SNAPSHOT_TIMEOUT = + new ConfigOption<>( + "raft.install_snapshot_rpc_timeout", + "The install snapshot rpc timeout in seconds for jraft rpc.", + positiveInt(), + // jraft default value is 5 minutes + 10 * 60 * 60 + ); + public static final ConfigOption RAFT_RPC_BUF_LOW_WATER_MARK = + new ConfigOption<>( + "raft.rpc_buf_low_water_mark", + "The ChannelOutboundBuffer's low water mark of netty, " + + "when buffer size less than this size, the method " + + "ChannelOutboundBuffer.isWritable() will return true, " + + "it means that low downstream pressure or good network.", + positiveInt(), + 10 * 1024 * 1024 + ); + public static final ConfigOption RAFT_RPC_BUF_HIGH_WATER_MARK = + new ConfigOption<>( + "raft.rpc_buf_high_water_mark", + "The ChannelOutboundBuffer's high water mark of netty, " + + "only when buffer size exceed this size, the method " + + "ChannelOutboundBuffer.isWritable() will return false, " + + "it means that the downstream pressure is too great to " + + "process the request or network is very congestion, " + + "upstream needs to limit rate at this time.", + positiveInt(), + 20 * 1024 * 1024 + ); + public static final ConfigOption RATE_LIMIT_WRITE = + new ConfigOption<>( + "rate_limit.write", + "The max rate(items/s) to add/update/delete vertices/edges.", + rangeInt(0, Integer.MAX_VALUE), + 0 + ); + public static final ConfigOption RATE_LIMIT_READ = + new ConfigOption<>( + "rate_limit.read", + "The max rate(times/s) to execute query of vertices/edges.", + rangeInt(0, Integer.MAX_VALUE), + 0 + ); + public static final ConfigOption TASK_SCHEDULE_PERIOD = + new ConfigOption<>( + "task.schedule_period", + "Period time when scheduler to schedule task", + rangeInt(0L, Long.MAX_VALUE), + 10L + ); + public static final ConfigOption TASK_WAIT_TIMEOUT = + new ConfigOption<>( + "task.wait_timeout", + "Timeout in seconds for waiting for the task to " + + "complete, such as when truncating or clearing the " + + "backend.", + rangeInt(0L, Long.MAX_VALUE), + 10L + ); + public static final ConfigOption TASK_INPUT_SIZE_LIMIT = + new ConfigOption<>( + "task.input_size_limit", + "The job input size limit in bytes.", + rangeInt(0L, Bytes.GB), + 16 * Bytes.MB + ); + public static final ConfigOption TASK_RESULT_SIZE_LIMIT = + new ConfigOption<>( + "task.result_size_limit", + "The job result size limit in bytes.", + rangeInt(0L, Bytes.GB), + 16 * Bytes.MB + ); + public static final ConfigOption TASK_TTL_DELETE_BATCH = + new ConfigOption<>( + "task.ttl_delete_batch", + "The batch size used to delete expired data.", + rangeInt(1, 500), + 1 + ); + public static final ConfigOption SCHEDULER_TYPE = + new ConfigOption<>( + "task.scheduler_type", + "The type of scheduler used in distribution system.", + allowValues("local", "distributed"), + "local" + ); + public static final ConfigOption TASK_SYNC_DELETION = + new ConfigOption<>( + "task.sync_deletion", + "Whether to delete schema or expired data synchronously.", + disallowEmpty(), + false + ); + public static final ConfigOption TASK_RETRY = + new ConfigOption<>( + "task.retry", + "Task retry times.", + rangeInt(0, 3), + 0 + ); + public static final ConfigOption STORE_CONN_DETECT_INTERVAL = + new ConfigOption<>( + "store.connection_detect_interval", + "The interval in seconds for detecting connections, " + + "if the idle time of a connection exceeds this value, " + + "detect it and reconnect if needed before using, " + + "value 0 means detecting every time.", + rangeInt(0L, Long.MAX_VALUE), + 600L + ); + public static final ConfigOption VERTEX_DEFAULT_LABEL = + new ConfigOption<>( + "vertex.default_label", + "The default vertex label.", + disallowEmpty(), + "vertex" + ); + public static final ConfigOption VERTEX_CHECK_CUSTOMIZED_ID_EXIST = + new ConfigOption<>( + "vertex.check_customized_id_exist", + "Whether to check the vertices exist for those using " + + "customized id strategy.", + disallowEmpty(), + false + ); + public static final ConfigOption VERTEX_REMOVE_LEFT_INDEX = + new ConfigOption<>( + "vertex.remove_left_index_at_overwrite", + "Whether remove left index at overwrite.", + disallowEmpty(), + false + ); + public static final ConfigOption VERTEX_ADJACENT_VERTEX_EXIST = + new ConfigOption<>( + "vertex.check_adjacent_vertex_exist", + "Whether to check the adjacent vertices of edges exist.", + disallowEmpty(), + false + ); + public static final ConfigOption VERTEX_ADJACENT_VERTEX_LAZY = + new ConfigOption<>( + "vertex.lazy_load_adjacent_vertex", + "Whether to lazy load adjacent vertices of edges.", + disallowEmpty(), + true + ); + public static final ConfigOption VERTEX_PART_EDGE_COMMIT_SIZE = + new ConfigOption<>( + "vertex.part_edge_commit_size", + "Whether to enable the mode to commit part of edges of " + + "vertex, enabled if commit size > 0, 0 meas disabled.", + rangeInt(0, (int) Query.DEFAULT_CAPACITY), + 5000 + ); + public static final ConfigOption VERTEX_ENCODE_PK_NUMBER = + new ConfigOption<>( + "vertex.encode_primary_key_number", + "Whether to encode number value of primary key " + + "in vertex id.", + disallowEmpty(), + true + ); + public static final ConfigOption VERTEX_TX_CAPACITY = + new ConfigOption<>( + "vertex.tx_capacity", + "The max size(items) of vertices(uncommitted) in " + + "transaction.", + rangeInt((int) COMMIT_BATCH, 1000000), + 10000 + ); + public static final ConfigOption QUERY_IGNORE_INVALID_DATA = + new ConfigOption<>( + "query.ignore_invalid_data", + "Whether to ignore invalid data of vertex or edge.", + disallowEmpty(), + true + ); + public static final ConfigOption QUERY_OPTIMIZE_AGGR_BY_INDEX = + new ConfigOption<>( + "query.optimize_aggregate_by_index", + "Whether to optimize aggregate query(like count) by index.", + disallowEmpty(), + false + ); + public static final ConfigOption QUERY_BATCH_SIZE = + new ConfigOption<>( + "query.batch_size", + "The size of each batch when querying by batch.", + rangeInt(1, (int) Query.DEFAULT_CAPACITY), + 1000 + ); + public static final ConfigOption QUERY_PAGE_SIZE = + new ConfigOption<>( + "query.page_size", + "The size of each page when querying by paging.", + rangeInt(1, (int) Query.DEFAULT_CAPACITY), + 500 + ); + public static final ConfigOption QUERY_INDEX_INTERSECT_THRESHOLD = + new ConfigOption<>( + "query.index_intersect_threshold", + "The maximum number of intermediate results to " + + "intersect indexes when querying by multiple single " + + "index properties.", + rangeInt(1, (int) Query.DEFAULT_CAPACITY), + 1000 + ); + public static final ConfigOption SCHEMA_INIT_TEMPLATE = + new ConfigOption<>( + "schema.init_template", + "The template schema used to init graph", + null, + "" + ); + public static final ConfigOption SCHEMA_INDEX_REBUILD_USING_PUSHDOWN = + new ConfigOption<>( + "schema.index_rebuild_using_pushdown", + "Whether to use pushdown when to create/rebuid index.", + true + ); + public static final ConfigOption QUERY_RAMTABLE_ENABLE = + new ConfigOption<>( + "query.ramtable_enable", + "Whether to enable ramtable for query of adjacent edges.", + disallowEmpty(), + false + ); + public static final ConfigOption QUERY_RAMTABLE_VERTICES_CAPACITY = + new ConfigOption<>( + "query.ramtable_vertices_capacity", + "The maximum number of vertices in ramtable, " + + "generally the largest vertex id is used as capacity.", + rangeInt(1L, Integer.MAX_VALUE * 2L), + 10000000L + ); + public static final ConfigOption QUERY_RAMTABLE_EDGES_CAPACITY = + new ConfigOption<>( + "query.ramtable_edges_capacity", + "The maximum number of edges in ramtable, " + + "include OUT and IN edges.", + rangeInt(1, Integer.MAX_VALUE), + 20000000 + ); + /** + * The schema name rule: + * 1. Not allowed end with spaces + * 2. Not allowed start with '~' + */ + public static final ConfigOption SCHEMA_ILLEGAL_NAME_REGEX = + new ConfigOption<>( + "schema.illegal_name_regex", + "The regex specified the illegal format for schema name.", + disallowEmpty(), + ".*\\s+$|~.*" + ); + public static final ConfigOption SCHEMA_CACHE_CAPACITY = + new ConfigOption<>( + "schema.cache_capacity", + "The max cache size(items) of schema cache.", + rangeInt(0L, Long.MAX_VALUE), + 10000L + ); + public static final ConfigOption VERTEX_CACHE_TYPE = + new ConfigOption<>( + "vertex.cache_type", + "The type of vertex cache, allowed values are [l1, l2].", + allowValues("l1", "l2"), + "l2" + ); + public static final ConfigOption VERTEX_CACHE_CAPACITY = + new ConfigOption<>( + "vertex.cache_capacity", + "The max cache size(items) of vertex cache.", + rangeInt(0L, Long.MAX_VALUE), + (1000 * 1000 * 10L) + ); + public static final ConfigOption VERTEX_CACHE_EXPIRE = + new ConfigOption<>( + "vertex.cache_expire", + "The expiration time in seconds of vertex cache.", + rangeInt(0, Integer.MAX_VALUE), + (60 * 10) + ); + public static final ConfigOption EDGE_CACHE_TYPE = + new ConfigOption<>( + "edge.cache_type", + "The type of edge cache, allowed values are [l1, l2].", + allowValues("l1", "l2"), + "l2" + ); + public static final ConfigOption EDGE_CACHE_CAPACITY = + new ConfigOption<>( + "edge.cache_capacity", + "The max cache size(items) of edge cache.", + rangeInt(0L, Long.MAX_VALUE), + ((long) 1000 * 1000) + ); + public static final ConfigOption EDGE_CACHE_EXPIRE = + new ConfigOption<>( + "edge.cache_expire", + "The expiration time in seconds of edge cache.", + rangeInt(0, Integer.MAX_VALUE), + (60 * 10) + ); + public static final ConfigOption SNOWFLAKE_WORKER_ID = + new ConfigOption<>( + "snowflake.worker_id", + "The worker id of snowflake id generator.", + disallowEmpty(), + 0L + ); + public static final ConfigOption SNOWFLAKE_DATACENTER_ID = + new ConfigOption<>( + "snowflake.datacenter_id", + "The datacenter id of snowflake id generator.", + disallowEmpty(), + 0L + ); + public static final ConfigOption SNOWFLAKE_FORCE_STRING = + new ConfigOption<>( + "snowflake.force_string", + "Whether to force the snowflake long id to be a string.", + disallowEmpty(), + false + ); + public static final ConfigOption TEXT_ANALYZER = + new ConfigOption<>( + "search.text_analyzer", + "Choose a text analyzer for searching the " + + "vertex/edge properties, available type are " + + "[ansj, hanlp, smartcn, jieba, jcseg, " + + "mmseg4j, ikanalyzer].", + disallowEmpty(), + "ikanalyzer" + ); + public static final ConfigOption TEXT_ANALYZER_MODE = + new ConfigOption<>( + "search.text_analyzer_mode", + "Specify the mode for the text analyzer, " + + "the available mode of analyzer are " + + "ansj: [BaseAnalysis, IndexAnalysis, ToAnalysis, " + + "NlpAnalysis], " + + "hanlp: [standard, nlp, index, nShort, shortest, speed], " + + "smartcn: [], " + + "jieba: [SEARCH, INDEX], " + + "jcseg: [Simple, Complex], " + + "mmseg4j: [Simple, Complex, MaxWord], " + + "ikanalyzer: [smart, max_word]" + + "}.", + disallowEmpty(), + "smart" + ); + public static final ConfigOption COMPUTER_CONFIG = + new ConfigOption<>( + "computer.config", + "The config file path of computer job.", + disallowEmpty(), + "./conf/computer.yaml" + ); + public static final ConfigOption K8S_OPERATOR_TEMPLATE = + new ConfigOption<>( + "k8s.operator_template", + "the path of operator container template.", + disallowEmpty(), + "./conf/operator-template.yaml" + ); + public static final ConfigOption K8S_QUOTA_TEMPLATE = + new ConfigOption<>( + "k8s.quota_template", + "the path of resource quota template.", + disallowEmpty(), + "./conf/resource-quota-template.yaml" + ); + public static final ConfigOption OLTP_CONCURRENT_THREADS = + new ConfigOption<>( + "oltp.concurrent_threads", + "Thread number to concurrently execute oltp algorithm.", + rangeInt(0, 65535), + 10 + ); + public static final ConfigOption OLTP_CONCURRENT_DEPTH = + new ConfigOption<>( + "oltp.concurrent_depth", + "The min depth to enable concurrent oltp algorithm.", + rangeInt(0, 65535), + 10 + ); + public static final ConfigConvOption OLTP_COLLECTION_TYPE = + new ConfigConvOption<>( + "oltp.collection_type", + "The implementation type of collections " + + "used in oltp algorithm.", + allowValues("JCF", "EC", "FU"), + CollectionType::valueOf, + "EC" + ); + public static final ConfigOption PD_PEERS = new ConfigOption<>( + "pd.peers", + "The addresses of pd nodes, separated with commas.", + disallowEmpty(), + "127.0.0.1:8686" + ); + public static final ConfigOption MEMORY_MODE = new ConfigOption<>( + "memory.mode", + "The memory mode used for query in HugeGraph.", + disallowEmpty(), + "off-heap" + ); + public static final ConfigOption MAX_MEMORY_CAPACITY = new ConfigOption<>( + "memory.max_capacity", + "The maximum memory capacity that can be managed for all queries in HugeGraph.", + nonNegativeInt(), + Bytes.GB + ); + public static final ConfigOption ONE_QUERY_MAX_MEMORY_CAPACITY = new ConfigOption<>( + "memory.one_query_max_capacity", + "The maximum memory capacity that can be managed for a query in HugeGraph.", + nonNegativeInt(), + Bytes.MB * 100 + ); + public static final ConfigOption MEMORY_ALIGNMENT = new ConfigOption<>( + "memory.alignment", + "The alignment used for round memory size.", + nonNegativeInt(), + 8L + ); + public static final ConfigOption GRAPH_SPACE = + new ConfigOption<>( + "graphspace", + "The graph space name.", + null, + "DEFAULT" + ); + private static volatile CoreOptions instance; + + private CoreOptions() { + super(); + } + + public static synchronized CoreOptions instance() { + if (instance == null) { + instance = new CoreOptions(); + // Should initialize all static members first, then register. + instance.registerOptions(); + } + return instance; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Aggregate.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Aggregate.java new file mode 100644 index 000000000..38f1365f6 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Aggregate.java @@ -0,0 +1,61 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.query; + +import java.util.Iterator; + +@Deprecated +public class Aggregate

{ + + private final AggregateFuncDefine

func; + private final String column; + + public Aggregate(AggregateFuncDefine func, String column) { + this.func = func; + this.column = column; + } + + public AggregateFuncDefine func() { + return this.func; + } + + public String column() { + return this.column; + } + + public boolean countAll() { + return this.func.countAll() && this.column == null; + } + + public P reduce(Iterator

results) { + return this.func.reduce(results); + } + + public P defaultValue() { + return this.func.defaultValue(); + } + + @Override + public String toString() { + return String.format("%s(%s)", this.func.string(), + this.column == null ? "*" : this.column); + } + +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/AggregateFuncDefine.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/AggregateFuncDefine.java new file mode 100644 index 000000000..d883aab49 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/AggregateFuncDefine.java @@ -0,0 +1,37 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.query; + +import java.util.Iterator; + +/** + * Definition of aggregation method + * + * @param

+ */ +public interface AggregateFuncDefine

{ + String string(); + + P defaultValue(); + + P reduce(Iterator

results); + + boolean countAll(); +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Condition.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Condition.java new file mode 100644 index 000000000..5c7d3e221 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Condition.java @@ -0,0 +1,1040 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.query; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.text.similarity.LevenshteinDistance; +import org.apache.hugegraph.backend.Shard; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.structure.BaseElement; +import org.apache.hugegraph.structure.BaseProperty; +import org.apache.hugegraph.type.define.HugeKeys; +import org.apache.hugegraph.util.Bytes; +import org.apache.hugegraph.util.DateUtil; +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.NumericUtil; + +import java.util.*; +import java.util.function.BiFunction; +import java.util.function.BiPredicate; +import java.util.regex.Pattern; + +public abstract class Condition { + + public Condition() { + + } + + public static Condition and(Condition left, Condition right) { + return new And(left, right); + } + + public static Condition or(Condition left, Condition right) { + return new Or(left, right); + } + + public static Condition not(Condition condition) { + return new Not(condition); + } + + public static Relation eq(HugeKeys key, Object value) { + return new SyspropRelation(key, RelationType.EQ, value); + } + + public static Relation gt(HugeKeys key, Object value) { + return new SyspropRelation(key, RelationType.GT, value); + } + + public static Relation gte(HugeKeys key, Object value) { + return new SyspropRelation(key, RelationType.GTE, value); + } + + public static Relation lt(HugeKeys key, Object value) { + return new SyspropRelation(key, RelationType.LT, value); + } + + public static Relation lte(HugeKeys key, Object value) { + return new SyspropRelation(key, RelationType.LTE, value); + } + + public static Relation neq(HugeKeys key, Object value) { + return new SyspropRelation(key, RelationType.NEQ, value); + } + + public static Condition in(HugeKeys key, List value) { + return new SyspropRelation(key, RelationType.IN, value); + } + + public static Condition nin(HugeKeys key, List value) { + return new SyspropRelation(key, RelationType.NOT_IN, value); + } + + public static Condition prefix(HugeKeys key, Id value) { + return new SyspropRelation(key, RelationType.PREFIX, value); + } + + public static Condition containsValue(HugeKeys key, Object value) { + return new SyspropRelation(key, RelationType.CONTAINS_VALUE, value); + } + + public static Condition containsKey(HugeKeys key, Object value) { + return new SyspropRelation(key, RelationType.CONTAINS_KEY, value); + } + + public static Condition contains(HugeKeys key, Object value) { + return new SyspropRelation(key, RelationType.CONTAINS, value); + } + + public static Condition scan(String start, String end) { + Shard value = new Shard(start, end, 0); + return new SyspropRelation(HugeKeys.ID, RelationType.SCAN, value); + } + + public static Relation eq(Id key, Object value) { + return new UserpropRelation(key, RelationType.EQ, value); + } + + public static Relation gt(Id key, Object value) { + return new UserpropRelation(key, RelationType.GT, value); + } + + public static Relation gte(Id key, Object value) { + return new UserpropRelation(key, RelationType.GTE, value); + } + + public static Relation lt(Id key, Object value) { + return new UserpropRelation(key, RelationType.LT, value); + } + + public static Relation lte(Id key, Object value) { + return new UserpropRelation(key, RelationType.LTE, value); + } + + public static Relation neq(Id key, Object value) { + return new UserpropRelation(key, RelationType.NEQ, value); + } + + public static Relation in(Id key, List value) { + return new UserpropRelation(key, RelationType.IN, value); + } + + public static Relation nin(Id key, List value) { + return new UserpropRelation(key, RelationType.NOT_IN, value); + } + + public static Relation textContains(Id key, String word) { + return new UserpropRelation(key, RelationType.TEXT_CONTAINS, word); + } + + public static Relation textContainsAny(Id key, Set words) { + return new UserpropRelation(key, RelationType.TEXT_CONTAINS_ANY, words); + } + + public static Condition contains(Id key, Object value) { + return new UserpropRelation(key, RelationType.CONTAINS, value); + } + + public abstract ConditionType type(); + + public abstract boolean isSysprop(); + + public abstract List relations(); + + public abstract boolean test(Object value); + + public abstract boolean test(BaseElement element); + + public abstract Condition copy(); + + public abstract Condition replace(Relation from, Relation to); + + public Condition and(Condition other) { + return new And(this, other); + } + + public Condition or(Condition other) { + return new Or(this, other); + } + + public Condition not() { + return new Not(this); + } + + public boolean isRelation() { + return this.type() == ConditionType.RELATION; + } + + public boolean isLogic() { + return this.type() == ConditionType.AND || + this.type() == ConditionType.OR || + this.type() == ConditionType.NOT; + } + + public boolean isFlattened() { + return this.isRelation(); + } + + public enum ConditionType { + NONE, + RELATION, + AND, + OR, + NOT + } + + public enum RelationType implements BiPredicate { + + EQ("==", RelationType::equals), + + GT(">", (v1, v2) -> { + return compare(v1, v2) > 0; + }), + + GTE(">=", (v1, v2) -> { + return compare(v1, v2) >= 0; + }), + + LT("<", (v1, v2) -> { + return compare(v1, v2) < 0; + }), + + LTE("<=", (v1, v2) -> { + return compare(v1, v2) <= 0; + }), + + NEQ("!=", (v1, v2) -> { + return compare(v1, v2) != 0; + }), + + IN("in", null, Collection.class, (v1, v2) -> { + assert v2 != null; + return ((Collection) v2).contains(v1); + }), + + NOT_IN("notin", null, Collection.class, (v1, v2) -> { + assert v2 != null; + return !((Collection) v2).contains(v1); + }), + + PREFIX("prefix", Id.class, Id.class, (v1, v2) -> { + assert v2 != null; + return v1 != null && Bytes.prefixWith(((Id) v2).asBytes(), + ((Id) v1).asBytes()); + }), + + TEXT_ANALYZER_CONTAINS("analyzercontains", String.class, + String.class, (v1, v2) -> { + return v1 != null && + ((String) v1).toLowerCase().contains(((String) v2).toLowerCase()); + }), + + TEXT_CONTAINS("textcontains", String.class, String.class, (v1, v2) -> { + // TODO: support collection-property textcontains + return v1 != null && ((String) v1).contains((String) v2); + }), + TEXT_MATCH_REGEX("textmatchregex", String.class, String.class, + (v1, v2) -> { + return Pattern.matches((String) v2, (String) v1); + }), + + TEXT_MATCH_EDIT_DISTANCE("texteditdistance", String.class, + String.class, (v1, v2) -> { + String content = (String) v2; + String distanceStr = content.substring(0, content.indexOf("#")); + int distance = Integer.valueOf(distanceStr); + String target = content.substring(content.indexOf("#") + 1); + return minEditDistance((String) v1, target) <= distance; + }), + TEXT_NOT_CONTAINS("textnotcontains", String.class, + String.class, (v1, v2) -> { + return v1 == null && v2 != null || + !((String) v1).toLowerCase().contains(((String) v2).toLowerCase()); + }), + TEXT_PREFIX("textprefix", String.class, String.class, (v1, v2) -> { + return ((String) v1).startsWith((String) v2); + }), + TEXT_NOT_PREFIX("textnotprefix", String.class, + String.class, (v1, v2) -> { + return !((String) v1).startsWith((String) v2); + }), + TEXT_SUFFIX("textsuffix", String.class, String.class, (v1, v2) -> { + return ((String) v1).endsWith((String) v2); + }), + TEXT_NOT_SUFFIX("textnotsuffix", String.class, + String.class, (v1, v2) -> { + return !((String) v1).endsWith((String) v2); + }), + + TEXT_CONTAINS_ANY("textcontainsany", String.class, Collection.class, (v1, v2) -> { + assert v2 != null; + if (v1 == null) { + return false; + } + + @SuppressWarnings("unchecked") + Collection words = (Collection) v2; + + for (String word : words) { + if (((String) v1).contains(word)) { + return true; + } + } + return false; + }), + + CONTAINS("contains", Collection.class, null, (v1, v2) -> { + assert v2 != null; + return v1 != null && ((Collection) v1).contains(v2); + }), + + CONTAINS_VALUE("containsv", Map.class, null, (v1, v2) -> { + assert v2 != null; + return v1 != null && ((Map) v1).containsValue(v2); + }), + + CONTAINS_KEY("containsk", Map.class, null, (v1, v2) -> { + assert v2 != null; + return v1 != null && ((Map) v1).containsKey(v2); + }), + + TEXT_CONTAINS_FUZZY("textcontainsfuzzy", String.class, + String.class, (v1, v2) -> { + for (String token : tokenize(((String) v1).toLowerCase())) { + if (isFuzzy(((String) v2).toLowerCase(), token)) { + return true; + } + } + return false; + }), + TEXT_FUZZY("textfuzzy", String.class, String.class, (v1, v2) -> { + return isFuzzy((String) v2, (String) v1); + }), + TEXT_CONTAINS_REGEX("textcontainsregex", String.class, + String.class, (v1, v2) -> { + for (String token : tokenize(((String) v1).toLowerCase())) { + if (token.matches((String) v2)) { + return true; + } + } + return false; + }), + TEXT_REGEX("textregex", String.class, String.class, (v1, v2) -> { + return ((String) v1).matches((String) v2); + }), + + SCAN("scan", (v1, v2) -> { + assert v2 != null; + /* + * TODO: we still have no way to determine accurately, since + * some backends may scan with token(column) like cassandra. + */ + return true; + }); + + private static final LevenshteinDistance ONE_LEVENSHTEIN_DISTANCE = + new LevenshteinDistance(1); + private static final LevenshteinDistance TWO_LEVENSHTEIN_DISTANCE = + new LevenshteinDistance(2); + private final String operator; + private final BiFunction tester; + private final Class v1Class; + private final Class v2Class; + + RelationType(String op, + BiFunction tester) { + this(op, null, null, tester); + } + + RelationType(String op, Class v1Class, Class v2Class, + BiFunction tester) { + this.operator = op; + this.tester = tester; + this.v1Class = v1Class; + this.v2Class = v2Class; + } + + private static int minEditDistance(String source, String target) { + E.checkArgument(source != null, "The source could not be null"); + E.checkArgument(target != null, "The target could not be null"); + + int sourceLen = source.length(); + int targetLen = target.length(); + if (sourceLen == 0) { + return targetLen; + } + if (targetLen == 0) { + return sourceLen; + } + + int[][] arr = new int[sourceLen + 1][targetLen + 1]; + for (int i = 0; i < sourceLen + 1; i++) { + arr[i][0] = i; + } + for (int j = 0; j < targetLen + 1; j++) { + arr[0][j] = j; + } + Character sourceChar = null; + Character targetChar = null; + for (int i = 1; i < sourceLen + 1; i++) { + sourceChar = source.charAt(i - 1); + for (int j = 1; j < targetLen + 1; j++) { + targetChar = target.charAt(j - 1); + if (sourceChar.equals(targetChar)) { + arr[i][j] = arr[i - 1][j - 1]; + } else { + arr[i][j] = (Math.min(Math.min(arr[i - 1][j], + arr[i][j - 1]), arr[i - 1][j - 1])) + 1; + } + } + } + return arr[sourceLen][targetLen]; + } + + /** + * Determine two values of any type equal + * + * @param first is actual value + * @param second is value in query condition + * @return true if equal, otherwise false + */ + private static boolean equals(final Object first, + final Object second) { + assert second != null; + if (first instanceof Id) { + if (second instanceof String) { + return second.equals(((Id) first).asString()); + } else if (second instanceof Long) { + return second.equals(((Id) first).asLong()); + } + } else if (second instanceof Number) { + return compare(first, second) == 0; + } else if (second.getClass().isArray()) { + return ArrayUtils.isEquals(first, second); + } + + return Objects.equals(first, second); + } + + /** + * Determine two numbers equal + * + * @param first is actual value, might be Number/Date or String, It is + * probably that the `first` is serialized to String. + * @param second is value in query condition, must be Number/Date + * @return the value 0 if first is numerically equal to second; + * a value less than 0 if first is numerically less than + * second; and a value greater than 0 if first is + * numerically greater than second. + */ + private static int compare(final Object first, final Object second) { + assert second != null; + if (second instanceof Number) { + return NumericUtil.compareNumber(first == null ? 0 : first, + (Number) second); + } else if (second instanceof Date) { + return compareDate(first, (Date) second); + } + + throw new IllegalArgumentException(String.format( + "Can't compare between %s(%s) and %s(%s)", first, + first == null ? null : first.getClass().getSimpleName(), + second, second.getClass().getSimpleName())); + } + + private static int compareDate(Object first, Date second) { + if (first == null) { + first = DateUtil.DATE_ZERO; + } + if (first instanceof Date) { + return ((Date) first).compareTo(second); + } + + throw new IllegalArgumentException(String.format( + "Can't compare between %s(%s) and %s(%s)", + first, first.getClass().getSimpleName(), + second, second.getClass().getSimpleName())); + } + + public static List tokenize(String str) { + final ArrayList tokens = new ArrayList<>(); + int previous = 0; + for (int p = 0; p < str.length(); p++) { + if (!Character.isLetterOrDigit(str.charAt(p))) { + if (p > previous + 1) { + tokens.add(str.substring(previous, p)); + } + previous = p + 1; + } + } + if (previous + 1 < str.length()) { + tokens.add(str.substring(previous)); + } + return tokens; + } + + private static boolean isFuzzy(String term, String value) { + int distance; + term = term.trim(); + int length = term.length(); + if (length < 3) { + return term.equals(value); + } else if (length < 6) { + distance = ONE_LEVENSHTEIN_DISTANCE.apply(value, term); + return distance <= 1 && distance >= 0; + } else { + distance = TWO_LEVENSHTEIN_DISTANCE.apply(value, term); + return distance <= 2 && distance >= 0; + } + } + + public String string() { + return this.operator; + } + + private void checkBaseType(Object value, Class clazz) { + if (!clazz.isInstance(value)) { + String valueClass = value == null ? "null" : + value.getClass().getSimpleName(); + E.checkArgument(false, + "Can't execute `%s` on type %s, expect %s", + this.operator, valueClass, + clazz.getSimpleName()); + } + } + + private void checkValueType(Object value, Class clazz) { + if (!clazz.isInstance(value)) { + String valueClass = value == null ? "null" : + value.getClass().getSimpleName(); + E.checkArgument(false, + "Can't test '%s'(%s) for `%s`, expect %s", + value, valueClass, this.operator, + clazz.getSimpleName()); + } + } + + @Override + public boolean test(Object first, Object second) { + E.checkState(this.tester != null, "Can't test %s", this.name()); + E.checkArgument(second != null, + "Can't test null value for `%s`", this.operator); + if (this.v1Class != null) { + this.checkBaseType(first, this.v1Class); + } + if (this.v2Class != null) { + this.checkValueType(second, this.v2Class); + } + return this.tester.apply(first, second); + } + + public boolean isFuzzyType() { + return this == TEXT_CONTAINS || this == TEXT_NOT_CONTAINS || + this == TEXT_NOT_PREFIX || this == TEXT_PREFIX || + this == TEXT_SUFFIX || this == TEXT_NOT_SUFFIX || + this == TEXT_CONTAINS_FUZZY || this == TEXT_FUZZY || + this == TEXT_CONTAINS_REGEX || this == TEXT_REGEX || + this == TEXT_CONTAINS_ANY || this == TEXT_MATCH_REGEX || + this == TEXT_MATCH_EDIT_DISTANCE; + } + + public boolean isRangeType() { + return ImmutableSet.of(GT, GTE, LT, LTE).contains(this); + } + + public boolean isSearchType() { + return this == TEXT_CONTAINS || this == TEXT_CONTAINS_ANY; + } + + public boolean isSecondaryType() { + return this == EQ; + } + } + + /** + * Condition defines + */ + public abstract static class BinCondition extends Condition { + + private Condition left; + private Condition right; + + public BinCondition(Condition left, Condition right) { + E.checkNotNull(left, "left condition"); + E.checkNotNull(right, "right condition"); + this.left = left; + this.right = right; + } + + public Condition left() { + return this.left; + } + + public Condition right() { + return this.right; + } + + @Override + public boolean isSysprop() { + return this.left.isSysprop() && this.right.isSysprop(); + } + + @Override + public List relations() { + List list = new ArrayList<>(this.left.relations()); + list.addAll(this.right.relations()); + return list; + } + + @Override + public Condition replace(Relation from, Relation to) { + this.left = this.left.replace(from, to); + this.right = this.right.replace(from, to); + return this; + } + + @Override + public String toString() { + String sb = String.valueOf(this.left) + ' ' + + this.type().name() + ' ' + + this.right; + return sb; + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof BinCondition)) { + return false; + } + BinCondition other = (BinCondition) object; + return this.type().equals(other.type()) && + this.left().equals(other.left()) && + this.right().equals(other.right()); + } + + @Override + public int hashCode() { + return this.type().hashCode() ^ + this.left().hashCode() ^ + this.right().hashCode(); + } + } + + public static class And extends BinCondition { + + public And(Condition left, Condition right) { + super(left, right); + } + + @Override + public ConditionType type() { + return ConditionType.AND; + } + + @Override + public boolean test(Object value) { + return this.left().test(value) && this.right().test(value); + } + + @Override + public boolean test(BaseElement element) { + return this.left().test(element) && this.right().test(element); + } + + @Override + public Condition copy() { + return new And(this.left().copy(), this.right().copy()); + } + } + + public static class Or extends BinCondition { + + public Or(Condition left, Condition right) { + super(left, right); + } + + @Override + public ConditionType type() { + return ConditionType.OR; + } + + @Override + public boolean test(Object value) { + return this.left().test(value) || this.right().test(value); + } + + @Override + public boolean test(BaseElement element) { + return this.left().test(element) || this.right().test(element); + } + + @Override + public Condition copy() { + return new Or(this.left().copy(), this.right().copy()); + } + } + + public static class Not extends Condition { + + Condition condition; + + public Not(Condition condition) { + super(); + this.condition = condition; + } + + public Condition condition() { + return condition; + } + + @Override + public ConditionType type() { + return ConditionType.NOT; + } + + @Override + public boolean test(Object value) { + return !this.condition.test(value); + } + + @Override + public boolean test(BaseElement element) { + return !this.condition.test(element); + } + + @Override + public Condition copy() { + return new Not(this.condition.copy()); + } + + @Override + public boolean isSysprop() { + return this.condition.isSysprop(); + } + + @Override + public List relations() { + return new ArrayList(this.condition.relations()); + } + + @Override + public Condition replace(Relation from, Relation to) { + this.condition = this.condition.replace(from, to); + return this; + } + + @Override + public String toString() { + String sb = this.type().name() + ' ' + + this.condition; + return sb; + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof Not)) { + return false; + } + Not other = (Not) object; + return this.type().equals(other.type()) && + this.condition.equals(other.condition()); + } + + @Override + public int hashCode() { + return this.type().hashCode() ^ + this.condition.hashCode(); + } + } + + public abstract static class Relation extends Condition { + + protected static final Set UNFLATTEN_RELATION_TYPES = + ImmutableSet.of(RelationType.IN, RelationType.NOT_IN, + RelationType.TEXT_CONTAINS_ANY); + // Relational operator (like: =, >, <, in, ...) + protected RelationType relation; + // Single-type value or a list of single-type value + protected Object value; + // The key serialized(code/string) by backend store. + protected Object serialKey; + // The value serialized(code/string) by backend store. + protected Object serialValue; + + @Override + public ConditionType type() { + return ConditionType.RELATION; + } + + public RelationType relation() { + return this.relation; + } + + public Object value() { + return this.value; + } + + public void value(Object value) { + this.value = value; + } + + public void serialKey(Object key) { + this.serialKey = key; + } + + public Object serialKey() { + return this.serialKey != null ? this.serialKey : this.key(); + } + + public void serialValue(Object value) { + this.serialValue = value; + } + + public Object serialValue() { + return this.serialValue != null ? this.serialValue : this.value(); + } + + @Override + public boolean test(Object value) { + return this.relation.test(value, this.value()); + } + + @Override + public boolean isFlattened() { + return !UNFLATTEN_RELATION_TYPES.contains(this.relation); + } + + @Override + public List relations() { + return ImmutableList.of(this); + } + + @Override + public Condition replace(Relation from, Relation to) { + if (this == from) { + return to; + } else { + return this; + } + } + + @Override + public String toString() { + String sb = String.valueOf(this.key()) + ' ' + + this.relation.string() + ' ' + + this.value; + return sb; + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof Relation)) { + return false; + } + Relation other = (Relation) object; + return this.relation().equals(other.relation()) && + this.key().equals(other.key()) && + this.value().equals(other.value()); + } + + @Override + public int hashCode() { + return this.type().hashCode() ^ + this.relation().hashCode() ^ + this.key().hashCode() ^ + this.value().hashCode(); + } + + @Override + public abstract boolean isSysprop(); + + public abstract Object key(); + + @Override + public abstract Relation copy(); + } + + public static class SyspropRelation extends Relation { + + private final HugeKeys key; + + public SyspropRelation(HugeKeys key, Object value) { + this(key, RelationType.EQ, value); + } + + public SyspropRelation(HugeKeys key, RelationType op, Object value) { + E.checkNotNull(op, "relation type"); + this.key = key; + this.relation = op; + this.value = value; + } + + @Override + public HugeKeys key() { + return this.key; + } + + @Override + public boolean isSysprop() { + return true; + } + + @Override + public boolean test(BaseElement element) { + E.checkNotNull(element, "element"); + Object value = element.sysprop(this.key); + return this.relation.test(value, this.value()); + } + + @Override + public Relation copy() { + Relation clone = new SyspropRelation(this.key, this.relation(), + this.value); + clone.serialKey(this.serialKey); + clone.serialValue(this.serialValue); + return clone; + } + } + + public static class FlattenSyspropRelation extends SyspropRelation { + + public FlattenSyspropRelation(SyspropRelation relation) { + super(relation.key(), relation.relation(), relation.value()); + } + + @Override + public boolean isFlattened() { + return true; + } + } + + public static class UserpropRelation extends Relation { + + // Id of property key + private final Id key; + + public UserpropRelation(Id key, Object value) { + this(key, RelationType.EQ, value); + } + + public UserpropRelation(Id key, RelationType op, Object value) { + E.checkNotNull(op, "relation type"); + this.key = key; + this.relation = op; + this.value = value; + } + + @Override + public Id key() { + return this.key; + } + + @Override + public boolean isSysprop() { + return false; + } + + @Override + public boolean test(BaseElement element) { + BaseProperty prop = element.getProperty(this.key); + Object value = prop != null ? prop.value() : null; + if (value == null) { + /* + * Fix #611 + * TODO: It's possible some scenes can't be returned false + * directly, such as: EQ with p1 == null, it should be returned + * true, but the query has(p, null) is not allowed by + * TraversalUtil.validPredicateValue(). + */ + return false; + } + return this.relation.test(value, this.value()); + } + + @Override + public Relation copy() { + Relation clone = new UserpropRelation(this.key, this.relation(), + this.value); + clone.serialKey(this.serialKey); + clone.serialValue(this.serialValue); + return clone; + } + } + + public static class RangeConditions { + + private Object keyEq = null; + private Object keyMin = null; + private boolean keyMinEq = false; + private Object keyMax = null; + private boolean keyMaxEq = false; + + public RangeConditions(List conditions) { + for (Condition c : conditions) { + Relation r = (Relation) c; + switch (r.relation()) { + case EQ: + this.keyEq = r.value(); + break; + case GTE: + this.keyMinEq = true; + this.keyMin = r.value(); + break; + case GT: + this.keyMin = r.value(); + break; + case LTE: + this.keyMaxEq = true; + this.keyMax = r.value(); + break; + case LT: + this.keyMax = r.value(); + break; + default: + E.checkArgument(false, "Unsupported relation '%s'", + r.relation()); + } + } + } + + public Object keyEq() { + return this.keyEq; + } + + public Object keyMin() { + return this.keyMin; + } + + public Object keyMax() { + return this.keyMax; + } + + public boolean keyMinEq() { + return this.keyMinEq; + } + + public boolean keyMaxEq() { + return this.keyMaxEq; + } + + public boolean hasRange() { + return this.keyMin != null || this.keyMax != null; + } + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/ConditionQuery.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/ConditionQuery.java new file mode 100644 index 000000000..553fec9b8 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/ConditionQuery.java @@ -0,0 +1,1217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.query; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.hugegraph.exception.BackendException; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.SplicingIdGenerator; +import org.apache.hugegraph.perf.PerfUtil.Watched; +import org.apache.hugegraph.query.Condition.Relation; +import org.apache.hugegraph.query.Condition.RelationType; +import org.apache.hugegraph.query.serializer.QueryAdapter; +import org.apache.hugegraph.query.serializer.QueryIdAdapter; +import org.apache.hugegraph.structure.BaseElement; +import org.apache.hugegraph.structure.BaseProperty; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.CollectionType; +import org.apache.hugegraph.type.define.HugeKeys; +import org.apache.hugegraph.util.*; +import org.apache.hugegraph.util.collection.CollectionFactory; + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.*; + +public class ConditionQuery extends IdQuery { + + public static final char INDEX_SYM_MIN = '\u0000'; + public static final String INDEX_SYM_ENDING = "\u0000"; + public static final String INDEX_SYM_NULL = "\u0001"; + public static final String INDEX_SYM_EMPTY = "\u0002"; + public static final char INDEX_SYM_MAX = '\u0003'; + // Note: here we use "new String" to distinguish normal string code + public static final String INDEX_VALUE_NULL = ""; + public static final String INDEX_VALUE_EMPTY = ""; + public static final Set IGNORE_SYM_SET; + private static final List EMPTY_CONDITIONS = ImmutableList.of(); + private static final Gson gson = new GsonBuilder() + .registerTypeAdapter(Condition.class, new QueryAdapter()) + .registerTypeAdapter(Id.class, new QueryIdAdapter()) + .setDateFormat("yyyy-MM-dd HH:mm:ss.SSS") + .create(); + private static final int indexStringValueLength = 20; + + static { + List list = new ArrayList<>(INDEX_SYM_MAX - INDEX_SYM_MIN); + for (char ch = INDEX_SYM_MIN; ch <= INDEX_SYM_MAX; ch++) { + list.add(String.valueOf(ch)); + } + IGNORE_SYM_SET = ImmutableSet.copyOf(list); + } + + // Conditions will be contacted with `and` by default + private List conditions = EMPTY_CONDITIONS; + + private OptimizedType optimizedType = OptimizedType.NONE; + + private ResultsFilter resultsFilter = null; + // 2023-03-30 + // Condition query sinking, no need to serialize this field + private transient Element2IndexValueMap element2IndexValueMap = null; + private boolean shard; + + // Store the index hit by current ConditionQuery + private transient MatchedIndex matchedIndex; + + public ConditionQuery(HugeType resultType) { + super(resultType); + } + + public ConditionQuery(HugeType resultType, Query originQuery) { + super(resultType, originQuery); + } + + /** + * Index and composite index interception + * + * @param values + * @return + */ + public static String concatValuesLimitLength(List values) { + List newValues = new ArrayList<>(values.size()); + for (Object v : values) { + v = convertLargeValue(v); + newValues.add(convertNumberIfNeeded(v)); + } + return SplicingIdGenerator.concatValues(newValues); + } + + /** + * Index and composite index interception + * + * @param value + * @return + */ + public static String concatValuesLimitLength(Object value) { + if (value instanceof List) { + return concatValuesLimitLength((List) value); + } + + if (needConvertNumber(value)) { + return LongEncoding.encodeNumber(value); + } + value = convertLargeValue(value); + return value.toString(); + } + + public static int getIndexStringValueLength() { + return indexStringValueLength; + } + + /** + * Extract the String value + * + * @param v + * @return + */ + private static Object convertLargeValue(Object v) { + + if (Objects.nonNull(v) && v instanceof String && + ((String) v).length() > getIndexStringValueLength()) { + + v = ((String) v).substring(0, getIndexStringValueLength()); + + } + + return v; + } + + private static Object convertNumberIfNeeded(Object value) { + if (needConvertNumber(value)) { + return LongEncoding.encodeNumber(value); + } + return value; + } + + private static boolean removeValue(Set values, Object value) { + for (Object compareValue : values) { + if (numberEquals(compareValue, value)) { + values.remove(compareValue); + return true; + } + } + return false; + } + + private static boolean numberEquals(Object number1, Object number2) { + // Same class compare directly + if (number1.getClass().equals(number2.getClass())) { + return number1.equals(number2); + } + // Otherwise convert to BigDecimal to make two numbers comparable + Number n1 = NumericUtil.convertToNumber(number1); + Number n2 = NumericUtil.convertToNumber(number2); + BigDecimal b1 = BigDecimal.valueOf(n1.doubleValue()); + BigDecimal b2 = BigDecimal.valueOf(n2.doubleValue()); + return b1.compareTo(b2) == 0; + } + + public static String concatValues(List values) { + assert !values.isEmpty(); + List newValues = new ArrayList<>(values.size()); + for (Object v : values) { + newValues.add(concatValues(v)); + } + return SplicingIdGenerator.concatValues(newValues); + } + + public static String concatValues(Object value) { + if (value instanceof String) { + return escapeSpecialValueIfNeeded((String) value); + } + if (value instanceof List) { + return concatValues((List) value); + } else if (needConvertNumber(value)) { + return LongEncoding.encodeNumber(value); + } else { + return escapeSpecialValueIfNeeded(value.toString()); + } + } + + public static ConditionQuery fromBytes(byte[] bytes) { + Gson gson = new GsonBuilder() + .registerTypeAdapter(Condition.class, new QueryAdapter()) + .registerTypeAdapter(Id.class, new QueryIdAdapter()) + .setDateFormat("yyyy-MM-dd HH:mm:ss.SSS") + .create(); + String cqs = new String(bytes, StandardCharsets.UTF_8); + ConditionQuery conditionQuery = gson.fromJson(cqs, ConditionQuery.class); + + return conditionQuery; + } + + private static boolean needConvertNumber(Object value) { + // Numeric or date values should be converted to number from string + return NumericUtil.isNumber(value) || value instanceof Date; + } + + private static String escapeSpecialValueIfNeeded(String value) { + if (value.isEmpty()) { + // Escape empty String to INDEX_SYM_EMPTY (char `\u0002`) + value = INDEX_SYM_EMPTY; + } else if (value == INDEX_VALUE_EMPTY) { + value = ""; + } else if (value == INDEX_VALUE_NULL) { + value = INDEX_SYM_NULL; + } else { + char ch = value.charAt(0); + if (ch <= INDEX_SYM_MAX) { + /* + * Special symbols can't be used due to impossible to parse, + * and treat it as illegal value for the origin text property. + * TODO: escape special symbols + */ + E.checkArgument(false, + "Illegal leading char '\\u%s' " + + "in index property: '%s'", + (int) ch, value); + } + } + return value; + } + + public MatchedIndex matchedIndex() { + return matchedIndex; + } + + public void matchedIndex(MatchedIndex matchedIndex) { + this.matchedIndex = matchedIndex; + } + + public void shard(boolean shard) { + this.shard = shard; + } + + public boolean shard() { + return this.shard; + } + + private void ensureElement2IndexValueMap() { + if (this.element2IndexValueMap == null) { + this.element2IndexValueMap = new Element2IndexValueMap(); + } + } + + public ConditionQuery query(Condition condition) { + // Query by id (HugeGraph-259) + if (condition instanceof Relation) { + Relation relation = (Relation) condition; + if (relation.key().equals(HugeKeys.ID) && + relation.relation() == RelationType.EQ) { + E.checkArgument(relation.value() instanceof Id, + "Invalid id value '%s'", relation.value()); + super.query((Id) relation.value()); + return this; + } + } + + if (this.conditions == EMPTY_CONDITIONS) { + this.conditions = InsertionOrderUtil.newList(); + } + this.conditions.add(condition); + return this; + } + + public ConditionQuery query(List conditions) { + for (Condition condition : conditions) { + this.query(condition); + } + return this; + } + + public ConditionQuery eq(HugeKeys key, Object value) { + // Filter value by key + return this.query(Condition.eq(key, value)); + } + + public ConditionQuery gt(HugeKeys key, Object value) { + return this.query(Condition.gt(key, value)); + } + + public ConditionQuery gte(HugeKeys key, Object value) { + return this.query(Condition.gte(key, value)); + } + + public ConditionQuery lt(HugeKeys key, Object value) { + return this.query(Condition.lt(key, value)); + } + + public ConditionQuery lte(HugeKeys key, Object value) { + return this.query(Condition.lte(key, value)); + } + + public ConditionQuery neq(HugeKeys key, Object value) { + return this.query(Condition.neq(key, value)); + } + + public ConditionQuery prefix(HugeKeys key, Id value) { + return this.query(Condition.prefix(key, value)); + } + + public ConditionQuery key(HugeKeys key, Object value) { + return this.query(Condition.containsKey(key, value)); + } + + public ConditionQuery scan(String start, String end) { + return this.query(Condition.scan(start, end)); + } + + @Override + public int conditionsSize() { + return this.conditions.size(); + } + + @Override + public Collection conditions() { + return Collections.unmodifiableList(this.conditions); + } + + public void resetConditions(List conditions) { + this.conditions = conditions; + } + + public void resetConditions() { + this.conditions = EMPTY_CONDITIONS; + } + + public void recordIndexValue(Id propertyId, Id id, Object indexValue) { + this.ensureElement2IndexValueMap(); + this.element2IndexValueMap().addIndexValue(propertyId, id, indexValue); + } + + public void selectedIndexField(Id indexField) { + this.ensureElement2IndexValueMap(); + this.element2IndexValueMap().selectedIndexField(indexField); + } + + public Set getElementLeftIndex(Id elementId) { + if (this.element2IndexValueMap == null) { + return null; + } + return this.element2IndexValueMap.getLeftIndex(elementId); + } + + public void removeElementLeftIndex(Id elementId) { + if (this.element2IndexValueMap == null) { + return; + } + this.element2IndexValueMap.removeElementLeftIndex(elementId); + } + + public ConditionQuery removeSysproCondition(HugeKeys sysproKey) { + for (Condition c : this.syspropConditions(sysproKey)) { + this.removeCondition(c); + } + return this; + } + + public ConditionQuery removeUserproCondition(Id key) { + for (Condition c : this.userpropConditions(key)) { + this.removeCondition(c); + } + return this; + } + + public ConditionQuery removeCondition(Condition condition) { + this.conditions.remove(condition); + return this; + } + + public boolean existLeftIndex(Id elementId) { + return this.getLeftIndexOfElement(elementId) != null; + } + + public Set getLeftIndexOfElement(Id elementId) { + if (this.element2IndexValueMap == null) { + return null; + } + return this.element2IndexValueMap.getLeftIndex(elementId); + } + + private Element2IndexValueMap element2IndexValueMap() { + if (this.element2IndexValueMap == null) { + this.element2IndexValueMap = new Element2IndexValueMap(); + } + return this.element2IndexValueMap; + } + + public List relations() { + List relations = new ArrayList<>(); + for (Condition c : this.conditions) { + relations.addAll(c.relations()); + } + return relations; + } + + public Relation relation(Id key) { + for (Relation r : this.relations()) { + if (r.key().equals(key)) { + return r; + } + } + return null; + } + + public Relation relation(HugeKeys key) { + for (Condition c : this.conditions) { + if (c.isRelation()) { + Condition.Relation r = (Condition.Relation) c; + if (r.key().equals(key)) { + return r; + } + } + } + return null; + } + + public boolean containsLabelOrUserpropRelation() { + for (Condition c : this.conditions) { + while (c instanceof Condition.Not) { + c = ((Condition.Not) c).condition(); + } + if (c.isLogic()) { + Condition.BinCondition binCondition = + (Condition.BinCondition) c; + ConditionQuery query = new ConditionQuery(HugeType.EDGE); + query.query(binCondition.left()); + query.query(binCondition.right()); + if (query.containsLabelOrUserpropRelation()) { + return true; + } + } else { + Condition.Relation r = (Condition.Relation) c; + if (r.key().equals(HugeKeys.LABEL) || + c instanceof Condition.UserpropRelation) { + return true; + } + } + } + return false; + } + + @Watched + public T condition(Object key) { + List valuesEQ = InsertionOrderUtil.newList(); + List valuesIN = InsertionOrderUtil.newList(); + for (Condition c : this.conditions) { + if (c.isRelation()) { + Condition.Relation r = (Condition.Relation) c; + if (r.key().equals(key)) { + if (r.relation() == RelationType.EQ) { + valuesEQ.add(r.value()); + } else if (r.relation() == RelationType.IN) { + Object value = r.value(); + assert value instanceof List; + valuesIN.add(value); + } + } + } + } + if (valuesEQ.isEmpty() && valuesIN.isEmpty()) { + return null; + } + if (valuesEQ.size() == 1 && valuesIN.isEmpty()) { + @SuppressWarnings("unchecked") + T value = (T) valuesEQ.get(0); + return value; + } + if (valuesEQ.isEmpty() && valuesIN.size() == 1) { + @SuppressWarnings("unchecked") + T value = (T) valuesIN.get(0); + return value; + } + + Set intersectValues = InsertionOrderUtil.newSet(); + for (Object value : valuesEQ) { + List valueAsList = ImmutableList.of(value); + if (intersectValues.isEmpty()) { + intersectValues.addAll(valueAsList); + } else { + CollectionUtil.intersectWithModify(intersectValues, + valueAsList); + } + } + for (Object value : valuesIN) { + @SuppressWarnings("unchecked") + List valueAsList = (List) value; + if (intersectValues.isEmpty()) { + intersectValues.addAll(valueAsList); + } else { + CollectionUtil.intersectWithModify(intersectValues, + valueAsList); + } + } + + if (intersectValues.isEmpty()) { + return null; + } + E.checkState(intersectValues.size() == 1, + "Illegal key '%s' with more than one value: %s", + key, intersectValues); + @SuppressWarnings("unchecked") + T value = (T) intersectValues.iterator().next(); + return value; + } + + public void unsetCondition(Object key) { + this.conditions.removeIf(c -> c.isRelation() && ((Relation) c).key().equals(key)); + } + + public boolean containsCondition(HugeKeys key) { + for (Condition c : this.conditions) { + if (c.isRelation()) { + Condition.Relation r = (Condition.Relation) c; + if (r.key().equals(key)) { + return true; + } + } + } + return false; + } + + public boolean containsCondition(Condition.RelationType type) { + for (Relation r : this.relations()) { + if (r.relation().equals(type)) { + return true; + } + } + return false; + } + + public boolean containsScanCondition() { + return this.containsCondition(Condition.RelationType.SCAN); + } + + public boolean containsRelation(HugeKeys key, Condition.RelationType type) { + for (Relation r : this.relations()) { + if (r.key().equals(key) && r.relation().equals(type)) { + return true; + } + } + return false; + } + + public boolean containsRelation(Condition.RelationType type) { + for (Relation r : this.relations()) { + if (r.relation().equals(type)) { + return true; + } + } + return false; + } + + public boolean containsScanRelation() { + return this.containsRelation(Condition.RelationType.SCAN); + } + + public boolean containsContainsCondition(Id key) { + for (Relation r : this.relations()) { + if (r.key().equals(key)) { + return r.relation().equals(RelationType.CONTAINS) || + r.relation().equals(RelationType.TEXT_CONTAINS); + } + } + return false; + } + + public boolean allSysprop() { + for (Condition c : this.conditions) { + if (!c.isSysprop()) { + return false; + } + } + return true; + } + + public boolean allRelation() { + for (Condition c : this.conditions) { + if (!c.isRelation()) { + return false; + } + } + return true; + } + + public List syspropConditions() { + this.checkFlattened(); + List conds = new ArrayList<>(); + for (Condition c : this.conditions) { + if (c.isSysprop()) { + conds.add(c); + } + } + return conds; + } + + public List syspropConditions(HugeKeys key) { + this.checkFlattened(); + List conditions = new ArrayList<>(); + for (Condition condition : this.conditions) { + Relation relation = (Relation) condition; + if (relation.key().equals(key)) { + conditions.add(relation); + } + } + return conditions; + } + + public List userpropConditions() { + this.checkFlattened(); + List conds = new ArrayList<>(); + for (Condition c : this.conditions) { + if (!c.isSysprop()) { + conds.add(c); + } + } + return conds; + } + + public List userpropConditions(Id key) { + this.checkFlattened(); + List conditions = new ArrayList<>(); + for (Condition condition : this.conditions) { + Relation relation = (Relation) condition; + if (relation.key().equals(key)) { + conditions.add(relation); + } + } + return conditions; + } + + public List userpropRelations() { + List relations = new ArrayList<>(); + for (Relation r : this.relations()) { + if (!r.isSysprop()) { + relations.add(r); + } + } + return relations; + } + + public void resetUserpropConditions() { + this.conditions.removeIf(condition -> !condition.isSysprop()); + } + + public Set userpropKeys() { + Set keys = new LinkedHashSet<>(); + for (Relation r : this.relations()) { + if (!r.isSysprop()) { + Condition.UserpropRelation ur = (Condition.UserpropRelation) r; + keys.add(ur.key()); + } + } + return keys; + } + + /** + * This method is only used for secondary index scenario, + * its relation must be EQ + * + * @param fields the user property fields + * @return the corresponding user property serial values of fields + */ + public String userpropValuesString(List fields) { + List values = new ArrayList<>(fields.size()); + for (Id field : fields) { + boolean got = false; + for (Relation r : this.userpropRelations()) { + if (r.key().equals(field) && !r.isSysprop()) { + E.checkState(r.relation == RelationType.EQ || + r.relation == RelationType.CONTAINS, + "Method userpropValues(List) only " + + "used for secondary index, " + + "relation must be EQ or CONTAINS, but got %s", + r.relation()); + values.add(r.serialValue()); + got = true; + } + } + if (!got) { + throw new BackendException( + "No such userprop named '%s' in the query '%s'", + field, this); + } + } + return concatValues(values); + } + + public String userpropValuesStringForIndex(List fields) { + List values = new ArrayList<>(fields.size()); + for (Id field : fields) { + boolean got = false; + for (Relation r : this.userpropRelations()) { + if (r.key().equals(field) && !r.isSysprop()) { + E.checkState(r.relation() == RelationType.EQ || + r.relation() == RelationType.CONTAINS, + "Method userpropValues(List) only " + + "used for secondary index, " + + "relation must be EQ or CONTAINS, but got %s", + r.relation()); + values.add(r.serialValue()); + got = true; + } + } + if (!got) { + throw new BackendException( + "No such userprop named '%s' in the query '%s'", + field, this); + } + } + return concatValuesLimitLength(values); + } + + public Set userpropValues(Id field) { + Set values = new HashSet<>(); + for (Relation r : this.userpropRelations()) { + if (r.key().equals(field)) { + values.add(r.serialValue()); + } + } + return values; + } + + public Object userpropValue(Id field) { + Set values = this.userpropValues(field); + if (values.isEmpty()) { + return null; + } + E.checkState(values.size() == 1, + "Expect one user-property value of field '%s', " + + "but got '%s'", field, values.size()); + return values.iterator().next(); + } + + public boolean hasRangeCondition() { + // NOTE: we need to judge all the conditions, including the nested + for (Condition.Relation r : this.relations()) { + if (r.relation().isRangeType()) { + return true; + } + } + return false; + } + + public boolean hasShardCondition() { + return this.shard; + } + + public boolean hasSearchCondition() { + // NOTE: we need to judge all the conditions, including the nested + for (Condition.Relation r : this.relations()) { + if (r.relation().isSearchType()) { + return true; + } + } + return false; + } + + public boolean hasSecondaryCondition() { + // NOTE: we need to judge all the conditions, including the nested + for (Condition.Relation r : this.relations()) { + if (r.relation().isSecondaryType()) { + return true; + } + } + return false; + } + + public boolean hasNeqCondition() { + // NOTE: we need to judge all the conditions, including the nested + for (Condition.Relation r : this.relations()) { + if (r.relation() == RelationType.NEQ) { + return true; + } + } + return false; + } + + public boolean matchUserpropKeys(List keys) { + Set conditionKeys = this.userpropKeys(); + return !keys.isEmpty() && conditionKeys.containsAll(keys); + } + + @Override + public ConditionQuery copy() { + ConditionQuery query = (ConditionQuery) super.copy(); + query.originQuery(this); + if (query.conditions != EMPTY_CONDITIONS) { + query.conditions = InsertionOrderUtil.newList(this.conditions); + } + query.optimizedType = OptimizedType.NONE; + query.resultsFilter = null; + + return query; + } + + public ConditionQuery deepCopy() { + ConditionQuery query = (ConditionQuery) super.copy(); + query.originQuery(this); + + List newConds = CollectionFactory.newList(CollectionType.EC); + for (Condition c : this.conditions) { + newConds.add(c); + } + query.resetConditions(newConds); + + query.optimizedType = OptimizedType.NONE; + query.resultsFilter = null; + + return query; + } + + public ConditionQuery copyAndResetUnshared() { + ConditionQuery query = this.copy(); + // These fields should not be shared by multiple sub-query + query.optimizedType = OptimizedType.NONE; + query.resultsFilter = null; + return query; + } + + public Condition.Relation copyRelationAndUpdateQuery(Object key) { + Condition.Relation copyRes = null; + for (int i = 0; i < this.conditions.size(); i++) { + Condition c = this.conditions.get(i); + if (c.isRelation()) { + Condition.Relation r = (Condition.Relation) c; + if (r.key().equals(key)) { + copyRes = r.copy(); + this.conditions.set(i, copyRes); + break; + } + } + } + E.checkArgument(copyRes != null, "Failed to copy Condition.Relation: %s", key); + return copyRes; + } + + @Override + public boolean test(BaseElement element) { + if (!this.ids().isEmpty() && !super.test(element)) { + return false; + } + + /* + * Currently results-filter is used to filter unmatched results returned + * by search index, and there may be multiple results-filter for every + * sub-query like within() + Text.contains(). + * We can't use sub-query results-filter here for fresh element which is + * not committed to backend store, because it's not from a sub-query. + */ + if (this.resultsFilter != null && !element.fresh()) { + return this.resultsFilter.test(element); + } + + /* + * NOTE: seems need to keep call checkRangeIndex() for each condition, + * so don't break early even if test() return false. + */ + boolean valid = true; + for (Condition cond : this.conditions) { + valid &= cond.test(element); + valid &= this.element2IndexValueMap == null || + this.element2IndexValueMap.checkRangeIndex(element, cond); + } + return valid; + } + + public void checkFlattened() { + E.checkState(this.isFlattened(), + "Query has none-flatten condition: %s", this); + } + + public boolean isFlattened() { + for (Condition condition : this.conditions) { + if (!condition.isFlattened()) { + return false; + } + } + return true; + } + + public boolean mayHasDupKeys(Set keys) { + Map keyCounts = new HashMap<>(); + for (Condition condition : this.conditions) { + if (!condition.isRelation()) { + // Assume may exist duplicate keys when has nested conditions + return true; + } + Relation relation = (Relation) condition; + if (keys.contains(relation.key())) { + int keyCount = keyCounts.getOrDefault(relation.key(), 0); + if (++keyCount > 1) { + return true; + } + keyCounts.put((HugeKeys) relation.key(), keyCount); + } + } + return false; + } + + public void optimized(OptimizedType optimizedType) { + assert this.optimizedType.ordinal() <= optimizedType.ordinal() : + this.optimizedType + " !<= " + optimizedType; + this.optimizedType = optimizedType; + + Query originQuery = this.originQuery(); + if (originQuery instanceof ConditionQuery) { + ConditionQuery cq = (ConditionQuery) originQuery; + /* + * Two sub-query(flatten) will both set optimized of originQuery, + * here we just keep the higher one, this may not be a perfect way + */ + if (optimizedType.ordinal() > cq.optimized().ordinal()) { + cq.optimized(optimizedType); + } + } + } + + public OptimizedType optimized() { + return this.optimizedType; + } + + public void registerResultsFilter(ResultsFilter filter) { + assert this.resultsFilter == null; + this.resultsFilter = filter; + } + + public void updateResultsFilter() { + Query originQuery = this.originQuery(); + if (originQuery instanceof ConditionQuery) { + ConditionQuery originCQ = (ConditionQuery) originQuery; + if (this.resultsFilter != null) { + originCQ.updateResultsFilter(this.resultsFilter); + } else { + originCQ.updateResultsFilter(); + } + } + } + + protected void updateResultsFilter(ResultsFilter filter) { + this.resultsFilter = filter; + Query originQuery = this.originQuery(); + if (originQuery instanceof ConditionQuery) { + ConditionQuery originCQ = (ConditionQuery) originQuery; + originCQ.updateResultsFilter(filter); + } + } + + public ConditionQuery originConditionQuery() { + Query originQuery = this.originQuery(); + if (!(originQuery instanceof ConditionQuery)) { + return null; + } + + while (originQuery.originQuery() instanceof ConditionQuery) { + originQuery = originQuery.originQuery(); + } + return (ConditionQuery) originQuery; + } + + public byte[] bytes() { + String cqs = gson.toJson(this); + return cqs.getBytes(StandardCharsets.UTF_8); + } + + public enum OptimizedType { + NONE, + PRIMARY_KEY, + SORT_KEYS, + INDEX, + INDEX_FILTER + } + + public interface ResultsFilter { + + boolean test(BaseElement element); + } + + public static final class Element2IndexValueMap { + + private final Map> leftIndexMap; + private final Map>> filed2IndexValues; + private Id selectedIndexField; + + public Element2IndexValueMap() { + this.filed2IndexValues = new HashMap<>(); + this.leftIndexMap = new HashMap<>(); + } + + private static boolean removeFieldValue(Set values, + Object value) { + for (Object elem : values) { + if (numberEquals(elem, value)) { + values.remove(elem); + return true; + } + } + return false; + } + + private static boolean removeValue(Set values, Object value) { + for (Object compareValue : values) { + if (numberEquals(compareValue, value)) { + values.remove(compareValue); + return true; + } + } + return false; + } + + private static boolean numberEquals(Object number1, Object number2) { + // Same class compare directly + if (number1.getClass().equals(number2.getClass())) { + return number1.equals(number2); + } + + // Otherwise convert to BigDecimal to make two numbers comparable + Number n1 = NumericUtil.convertToNumber(number1); + Number n2 = NumericUtil.convertToNumber(number2); + BigDecimal b1 = BigDecimal.valueOf(n1.doubleValue()); + BigDecimal b2 = BigDecimal.valueOf(n2.doubleValue()); + return b1.compareTo(b2) == 0; + } + + public void addIndexValue(Id indexField, Id elementId, + Object indexValue) { + if (!this.filed2IndexValues.containsKey(indexField)) { + this.filed2IndexValues.putIfAbsent(indexField, new HashMap<>()); + } + Map> element2IndexValueMap = + this.filed2IndexValues.get(indexField); + if (element2IndexValueMap.containsKey(elementId)) { + element2IndexValueMap.get(elementId).add(indexValue); + } else { + element2IndexValueMap.put(elementId, + Sets.newHashSet(indexValue)); + } + } + + public void selectedIndexField(Id indexField) { + this.selectedIndexField = indexField; + } + + public Set toRemoveIndexValues(Id indexField, Id elementId) { + if (!this.filed2IndexValues.containsKey(indexField)) { + return null; + } + return this.filed2IndexValues.get(indexField).get(elementId); + } + + public Set removeIndexValues(Id indexField, Id elementId) { + if (!this.filed2IndexValues.containsKey(indexField)) { + return null; + } + return this.filed2IndexValues.get(indexField).get(elementId); + } + + public void addLeftIndex(Id elementId, Id indexField, + Set indexValues) { + LeftIndex leftIndex = new LeftIndex(indexValues, indexField); + if (this.leftIndexMap.containsKey(elementId)) { + this.leftIndexMap.get(elementId).add(leftIndex); + } else { + this.leftIndexMap.put(elementId, Sets.newHashSet(leftIndex)); + } + } + + public Set getLeftIndex(Id elementId) { + return this.leftIndexMap.get(elementId); + } + + public void addLeftIndex(Id indexField, Set indexValues, + Id elementId) { + LeftIndex leftIndex = new LeftIndex(indexValues, indexField); + if (this.leftIndexMap.containsKey(elementId)) { + this.leftIndexMap.get(elementId).add(leftIndex); + } else { + this.leftIndexMap.put(elementId, Sets.newHashSet(leftIndex)); + } + } + + public void removeElementLeftIndex(Id elementId) { + this.leftIndexMap.remove(elementId); + } + + public boolean checkRangeIndex(BaseElement element, Condition cond) { + // Not UserpropRelation + if (!(cond instanceof Condition.UserpropRelation)) { + return true; + } + + Condition.UserpropRelation propRelation = (Condition.UserpropRelation) cond; + Id propId = propRelation.key(); + Set fieldValues = this.toRemoveIndexValues(propId, + element.id()); + if (fieldValues == null) { + // Not range index + return true; + } + + BaseProperty property = element.getProperty(propId); + if (property == null) { + // Property value has been deleted, so it's not matched + this.addLeftIndex(element.id(), propId, fieldValues); + return false; + } + + /* + * NOTE: If removing successfully means there is correct index, + * else we should add left-index values to left index map to + * wait the left-index to be removed. + */ + boolean hasRightValue = removeFieldValue(fieldValues, + property.value()); + if (!fieldValues.isEmpty()) { + this.addLeftIndex(element.id(), propId, fieldValues); + } + + /* + * NOTE: When query by more than one range index field, + * if current field is not the selected one, it can only be used to + * determine whether the index values matched, can't determine + * the element is valid or not. + */ + if (this.selectedIndexField != null) { + return !propId.equals(this.selectedIndexField) || hasRightValue; + } + + return hasRightValue; + } + + public boolean validRangeIndex(BaseElement element, Condition cond) { + // Not UserpropRelation + if (!(cond instanceof Condition.UserpropRelation)) { + return true; + } + + Condition.UserpropRelation propRelation = (Condition.UserpropRelation) cond; + Id propId = propRelation.key(); + Set fieldValues = this.removeIndexValues(propId, + element.id()); + if (fieldValues == null) { + // Not range index + return true; + } + + BaseProperty hugeProperty = element.getProperty(propId); + if (hugeProperty == null) { + // Property value has been deleted + this.addLeftIndex(propId, fieldValues, element.id()); + return false; + } + + /* + * NOTE: If success remove means has correct index, + * we should add left index values to left index map + * waiting to be removed + */ + boolean hasRightValue = removeValue(fieldValues, hugeProperty.value()); + if (fieldValues.size() > 0) { + this.addLeftIndex(propId, fieldValues, element.id()); + } + + /* + * NOTE: When query by more than one range index field, + * if current field is not the selected one, it can only be used to + * determine whether the index values matched, can't determine + * the element is valid or not + */ + if (this.selectedIndexField != null) { + return !propId.equals(this.selectedIndexField) || hasRightValue; + } + + return hasRightValue; + } + } + + public static final class LeftIndex { + + private final Set indexFieldValues; + private final Id indexField; + + public LeftIndex(Set indexFieldValues, Id indexField) { + this.indexFieldValues = indexFieldValues; + this.indexField = indexField; + } + + public Set indexFieldValues() { + return this.indexFieldValues; + } + + public Id indexField() { + return this.indexField; + } + } + + +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/IdQuery.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/IdQuery.java new file mode 100644 index 000000000..1235dfebc --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/IdQuery.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.query; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.structure.BaseElement; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.InsertionOrderUtil; + +import com.google.common.collect.ImmutableList; + +public class IdQuery extends Query { + + private static final List EMPTY_IDS = ImmutableList.of(); + + // The id(s) will be concated with `or` + private List ids = EMPTY_IDS; + private boolean mustSortByInput = true; + + public IdQuery(HugeType resultType) { + super(resultType); + } + + public IdQuery(HugeType resultType, Query originQuery) { + super(resultType, originQuery); + } + + public IdQuery(HugeType resultType, Set ids) { + this(resultType); + this.query(ids); + } + + public IdQuery(HugeType resultType, Id id) { + this(resultType); + this.query(id); + } + + public IdQuery(Query originQuery, Id id) { + this(originQuery.resultType(), originQuery); + this.query(id); + } + + public IdQuery(Query originQuery, Set ids) { + this(originQuery.resultType(), originQuery); + this.query(ids); + } + + public boolean mustSortByInput() { + return this.mustSortByInput; + } + + public void mustSortByInput(boolean mustSortedByInput) { + this.mustSortByInput = mustSortedByInput; + } + + @Override + public int idsSize() { + return this.ids.size(); + } + + @Override + public Collection ids() { + return Collections.unmodifiableList(this.ids); + } + + public void resetIds() { + this.ids = EMPTY_IDS; + } + + public IdQuery query(Id id) { + E.checkArgumentNotNull(id, "Query id can't be null"); + if (this.ids == EMPTY_IDS) { + this.ids = InsertionOrderUtil.newList(); + } + + int last = this.ids.size() - 1; + if (last >= 0 && id.equals(this.ids.get(last))) { + // The same id as the previous one, just ignore it + return this; + } + + this.ids.add(id); + this.checkCapacity(this.ids.size()); + return this; + } + + public IdQuery query(Set ids) { + for (Id id : ids) { + this.query(id); + } + return this; + } + + @Override + public boolean test(BaseElement element) { + return this.ids.contains(element.id()); + } + + @Override + public IdQuery copy() { + IdQuery query = (IdQuery) super.copy(); + query.ids = this.ids == EMPTY_IDS ? EMPTY_IDS : + InsertionOrderUtil.newList(this.ids); + return query; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/MatchedIndex.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/MatchedIndex.java new file mode 100644 index 000000000..6d63114ed --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/MatchedIndex.java @@ -0,0 +1,81 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.query; + +import java.util.Collections; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.hugegraph.schema.IndexLabel; +import org.apache.hugegraph.schema.SchemaLabel; + +public class MatchedIndex { + + private final SchemaLabel schemaLabel; + private final Set indexLabels; + + public MatchedIndex(SchemaLabel schemaLabel, + Set indexLabels) { + this.schemaLabel = schemaLabel; + this.indexLabels = indexLabels; + } + + public SchemaLabel schemaLabel() { + return this.schemaLabel; + } + + public Set indexLabels() { + return Collections.unmodifiableSet(this.indexLabels); + } + + + public boolean containsSearchIndex() { + for (IndexLabel il : this.indexLabels) { + if (il.indexType().isSearch()) { + return true; + } + } + return false; + } + + @Override + public int hashCode() { + return indexLabels.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof MatchedIndex)) { + return false; + } + Set indexLabels = ((MatchedIndex) other).indexLabels; + return Objects.equals(this.indexLabels, indexLabels); + } + + @Override + public String toString() { + String strIndexLabels = + indexLabels.stream().map(i -> i.name()).collect(Collectors.joining(",")); + + return "MatchedIndex{schemaLabel=" + schemaLabel.name() + + ", indexLabels=" + strIndexLabels + '}'; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Query.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Query.java new file mode 100644 index 000000000..2151cd6d0 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/Query.java @@ -0,0 +1,720 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.query; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import com.google.common.base.Joiner; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.hugegraph.exception.BackendException; +import org.apache.hugegraph.exception.LimitExceedException; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.structure.BaseElement; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.CollectionType; +import org.apache.hugegraph.type.define.HugeKeys; +import org.apache.hugegraph.util.CollectionUtil; +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.InsertionOrderUtil; +import org.apache.hugegraph.util.Log; +import org.apache.hugegraph.util.collection.IdSet; +import org.slf4j.Logger; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +public class Query implements Cloneable { + + private static final Logger LOG = Log.logger(Query.class); + // TODO: we should better not use Long.Max as the unify limit number + public static final long NO_LIMIT = Long.MAX_VALUE; + + public static final long COMMIT_BATCH = 500L; + public static final long QUERY_BATCH = 100L; + + public static final long NO_CAPACITY = -1L; + public static final long DEFAULT_CAPACITY = 800000L; // HugeGraph-777 + + private static final ThreadLocal CAPACITY_CONTEXT = new ThreadLocal<>(); + + protected static final Query NONE = new Query(HugeType.UNKNOWN); + + private static final Set EMPTY_OLAP_PKS = ImmutableSet.of(); + + private HugeType resultType; + private Map orders; + private long offset; + private long actualOffset; + private long actualStoreOffset; + private long limit; + private long skipDegree; + private String page; + private long capacity; + private boolean showHidden; + private boolean showDeleting; + private boolean showExpired; + private boolean olap; + private boolean withProperties; + private OrderType orderType; + private Set olapPks; + + private List selects = InsertionOrderUtil.newList(); + + @Deprecated + private transient Aggregate aggregate; + + private Query originQuery; + + private List groups = InsertionOrderUtil.newList(); + private boolean groupByLabel = false; + + // V3.7 aggs + private List> aggs = + InsertionOrderUtil.newList(); + + public Query() { + + } + + private static final ThreadLocal capacityContext = new ThreadLocal<>(); + + private static int indexStringValueLength = 20; + + public Query(HugeType resultType) { + this(resultType, null); + } + + public Query(HugeType resultType, Query originQuery) { + this.resultType = resultType; + this.originQuery = originQuery; + + this.orders = null; + + this.offset = 0L; + this.actualOffset = 0L; + this.actualStoreOffset = 0L; + this.limit = NO_LIMIT; + this.skipDegree = NO_LIMIT; + this.page = null; + + this.capacity = defaultCapacity(); + + this.showHidden = false; + this.showDeleting = false; + + this.withProperties = true; + this.orderType = OrderType.ORDER_STRICT; + + this.aggregate = null; + this.showExpired = false; + this.olap = false; + this.olapPks = EMPTY_OLAP_PKS; + } + + public void copyBasic(Query query) { + E.checkNotNull(query, "query"); + this.offset = query.offset(); + this.limit = query.limit(); + this.skipDegree = query.skipDegree(); + this.page = query.page(); + this.capacity = query.capacity(); + this.showHidden = query.showHidden(); + this.showDeleting = query.showDeleting(); + this.withProperties = query.withProperties(); + this.orderType = query.orderType(); + this.aggregate = query.aggregate(); + this.showExpired = query.showExpired(); + this.olap = query.olap(); + if (query.orders != null) { + this.orders(query.orders); + } + } + + public HugeType resultType() { + return this.resultType; + } + + public void resultType(HugeType resultType) { + this.resultType = resultType; + } + + public Query originQuery() { + return this.originQuery; + } + + public void setOriginQuery(Query query) { + this.originQuery = query; + } + + public Query rootOriginQuery() { + Query root = this; + while (root.originQuery != null) { + root = root.originQuery; + } + return root; + } + + protected void originQuery(Query originQuery) { + this.originQuery = originQuery; + } + + public Map orders() { + return Collections.unmodifiableMap(this.getOrNewOrders()); + } + + public void orders(Map orders) { + this.orders = InsertionOrderUtil.newMap(orders); + } + + public void order(HugeKeys key, Order order) { + this.getOrNewOrders().put(key, order); + } + + protected Map getOrNewOrders() { + if (this.orders != null) { + return this.orders; + } + this.orders = InsertionOrderUtil.newMap(); + return this.orders; + } + + public long offset() { + return this.offset; + } + + public void offset(long offset) { + E.checkArgument(offset >= 0L, "Invalid offset %s", offset); + this.offset = offset; + } + + public void copyOffset(Query parent) { + assert this.offset == 0L || this.offset == parent.offset; + assert this.actualOffset == 0L || + this.actualOffset == parent.actualOffset; + this.offset = parent.offset; + this.actualOffset = parent.actualOffset; + } + + public long actualOffset() { + return this.actualOffset; + } + + public void resetActualOffset() { + this.actualOffset = 0L; + this.actualStoreOffset = 0L; + } + + public long goOffset(long offset) { + E.checkArgument(offset >= 0L, "Invalid offset value: %s", offset); + if (this.originQuery != null) { + this.goParentOffset(offset); + } + return this.goSelfOffset(offset); + } + + private void goParentOffset(long offset) { + assert offset >= 0L; + Query parent = this.originQuery; + while (parent != null) { + parent.actualOffset += offset; + parent = parent.originQuery; + } + } + + private long goSelfOffset(long offset) { + assert offset >= 0L; + if (this.originQuery != null) { + this.originQuery.goStoreOffsetBySubQuery(offset); + } + this.actualOffset += offset; + return this.actualOffset; + } + + private long goStoreOffsetBySubQuery(long offset) { + Query parent = this.originQuery; + while (parent != null) { + parent.actualStoreOffset += offset; + parent = parent.originQuery; + } + this.actualStoreOffset += offset; + return this.actualStoreOffset; + } + + public Set skipOffsetIfNeeded(Set elems) { + /* + * Skip index(index query with offset) for performance optimization. + * We assume one result is returned by each index, but if there are + * overridden index it will cause confusing offset and results. + */ + long fromIndex = this.offset() - this.actualOffset(); + if (fromIndex < 0L) { + // Skipping offset is overhead, no need to skip + fromIndex = 0L; + } else if (fromIndex > 0L) { + this.goOffset(fromIndex); + } + E.checkArgument(fromIndex <= Integer.MAX_VALUE, + "Offset must be <= 0x7fffffff, but got '%s'", + fromIndex); + + if (fromIndex >= elems.size()) { + return ImmutableSet.of(); + } + long toIndex = this.total(); + if (this.noLimit() || toIndex > elems.size()) { + toIndex = elems.size(); + } + if (fromIndex == 0L && toIndex == elems.size()) { + return elems; + } + assert fromIndex < elems.size(); + assert toIndex <= elems.size(); + return CollectionUtil.subSet(elems, (int) fromIndex, (int) toIndex); + } + + public long remaining() { + if (this.limit == NO_LIMIT) { + return NO_LIMIT; + } else { + return this.total() - this.actualOffset(); + } + } + + public long total() { + if (this.limit == NO_LIMIT) { + return NO_LIMIT; + } else { + return this.offset + this.limit; + } + } + + public long limit() { + if (this.capacity != NO_CAPACITY) { + E.checkArgument(this.limit == Query.NO_LIMIT || + this.limit <= this.capacity, + "Invalid limit %s, must be <= capacity(%s)", + this.limit, this.capacity); + } + return this.limit; + } + + public void limit(long limit) { + E.checkArgument(limit >= 0L || limit == NO_LIMIT, + "Invalid limit %s", limit); + this.limit = limit; + } + + public boolean noLimit() { + return this.limit() == NO_LIMIT; + } + + public boolean noLimitAndOffset() { + return this.limit() == NO_LIMIT && this.offset() == 0L; + } + + public boolean reachLimit(long count) { + long limit = this.limit(); + if (limit == NO_LIMIT) { + return false; + } + return count >= (limit + this.offset()); + } + + /** + * Set or update the offset and limit by a range [start, end) + * NOTE: it will use the min range one: max start and min end + * + * @param start the range start, include it + * @param end the range end, exclude it + */ + public long range(long start, long end) { + // Update offset + long offset = this.offset(); + start = Math.max(start, offset); + this.offset(start); + + // Update limit + if (end != -1L) { + if (!this.noLimit()) { + end = Math.min(end, offset + this.limit()); + } else { + assert end < Query.NO_LIMIT; + } + E.checkArgument(end >= start, + "Invalid range: [%s, %s)", start, end); + this.limit(end - start); + } else { + // Keep the origin limit + assert this.limit() <= Query.NO_LIMIT; + } + return this.limit; + } + + public String page() { + if (this.page != null) { + E.checkState(this.limit() != 0L, + "Can't set limit=0 when using paging"); + E.checkState(this.offset() == 0L, + "Can't set offset when using paging, but got '%s'", + this.offset()); + } + return this.page; + } + + public String pageWithoutCheck() { + return this.page; + } + + public void page(String page) { + this.page = page; + } + + public boolean paging() { + return this.page != null; + } + + @Deprecated + public void olap(boolean olap) { + this.olap = olap; + } + + @Deprecated + public boolean olap() { + return this.olap; + } + + public void olapPks(Set olapPks) { + for (Id olapPk : olapPks) { + this.olapPk(olapPk); + } + } + + public void olapPk(Id olapPk) { + if (this.olapPks == EMPTY_OLAP_PKS) { + this.olapPks = new IdSet(CollectionType.EC); + } + this.olapPks.add(olapPk); + } + + public Set olapPks() { + return this.olapPks; + } + + public long capacity() { + return this.capacity; + } + + public void capacity(long capacity) { + this.capacity = capacity; + } + + public boolean bigCapacity() { + return this.capacity == NO_CAPACITY || this.capacity > DEFAULT_CAPACITY; + } + + public void checkCapacity(long count) throws LimitExceedException { + // Throw LimitExceedException if reach capacity + if (this.capacity != Query.NO_CAPACITY && count > this.capacity) { + final int MAX_CHARS = 256; + String query = this.toString(); + if (query.length() > MAX_CHARS) { + query = query.substring(0, MAX_CHARS) + "..."; + } + throw new LimitExceedException( + "Too many records(must <= %s) for the query: %s", + this.capacity, query); + } + } + + public Aggregate aggregate() { + return this.aggregate; + } + + public Aggregate aggregateNotNull() { + E.checkArgument(this.aggregate != null, + "The aggregate must be set for number query"); + return this.aggregate; + } + + public void aggregate(AggregateFuncDefine func, String property) { + this.aggregate = new Aggregate(func, property); + } + + public void aggregate(Aggregate aggregate) { + this.aggregate = aggregate; + } + + public boolean showHidden() { + return this.showHidden; + } + + public void showHidden(boolean showHidden) { + this.showHidden = showHidden; + } + + public boolean showDeleting() { + return this.showDeleting; + } + + public void showDeleting(boolean showDeleting) { + this.showDeleting = showDeleting; + } + + public long skipDegree() { + return this.skipDegree; + } + + public void skipDegree(long skipDegree) { + this.skipDegree = skipDegree; + } + + public boolean withProperties() { + return this.withProperties; + } + + public void withProperties(boolean withProperties) { + this.withProperties = withProperties; + } + + public OrderType orderType() { + return this.orderType; + } + + public void orderType(OrderType orderType) { + this.orderType = orderType; + } + + public boolean showExpired() { + return this.showExpired; + } + + public void showExpired(boolean showExpired) { + this.showExpired = showExpired; + } + + public Collection ids() { + return ImmutableList.of(); + } + + public Collection conditions() { + return ImmutableList.of(); + } + + public int idsSize() { + return 0; + } + + public int conditionsSize() { + return 0; + } + + public boolean empty() { + return this.idsSize() == 0 && this.conditionsSize() == 0; + } + + public boolean test(BaseElement element) { + return true; + } + + public Query copy() { + try { + return (Query) this.clone(); + } catch (CloneNotSupportedException e) { + throw new BackendException(e); + } + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof Query)) { + return false; + } + Query other = (Query) object; + return this.resultType.equals(other.resultType) && + this.orders().equals(other.orders()) && + this.offset == other.offset && + this.limit == other.limit && + Objects.equals(this.page, other.page) && + this.ids().equals(other.ids()) && + this.conditions().equals(other.conditions()) && + this.withProperties == other.withProperties; + } + + @Override + public int hashCode() { + int hash = this.orders().hashCode() ^ + Long.hashCode(this.offset) ^ + Long.hashCode(this.limit) ^ + Objects.hashCode(this.page) ^ + this.ids().hashCode() ^ + this.conditions().hashCode() ^ + this.selects().hashCode() ^ + Boolean.hashCode(this.withProperties); + if (this.resultType == null) { + return hash; + } else { + return this.resultType.hashCode() ^ hash; + } + } + + @Override + public String toString() { + Map pairs = InsertionOrderUtil.newMap(); + if (this.page != null) { + pairs.put("page", String.format("'%s'", this.page)); + } + if (this.offset != 0) { + pairs.put("offset", this.offset); + } + if (this.limit != NO_LIMIT) { + pairs.put("limit", this.limit); + } + if (!this.orders().isEmpty()) { + pairs.put("order by", this.orders()); + } + + StringBuilder sb = new StringBuilder(128); + sb.append("`Query "); + if (this.aggregate != null) { + sb.append(this.aggregate); + } else { + sb.append('*'); + } + sb.append(" from ").append(this.resultType); + for (Map.Entry entry : pairs.entrySet()) { + sb.append(' ').append(entry.getKey()) + .append(' ').append(entry.getValue()).append(','); + } + if (!pairs.isEmpty()) { + // Delete last comma + sb.deleteCharAt(sb.length() - 1); + } + + if (!this.empty()) { + sb.append(" where"); + } + + // Append ids + if (!this.ids().isEmpty()) { + sb.append(" id in ").append(this.ids()); + } + + // Append conditions + if (!this.conditions().isEmpty()) { + if (!this.ids().isEmpty()) { + sb.append(" and"); + } + sb.append(" ").append(this.conditions()); + } + + if (!this.groups.isEmpty()) { + sb.append(" group by ").append(Joiner.on(",").join(this.groups)); + } + + sb.append('`'); + return sb.toString(); + } + + public static long defaultCapacity(long capacity) { + Long old = CAPACITY_CONTEXT.get(); + CAPACITY_CONTEXT.set(capacity); + return old != null ? old : DEFAULT_CAPACITY; + } + + public static long defaultCapacity() { + Long capacity = CAPACITY_CONTEXT.get(); + return capacity != null ? capacity : DEFAULT_CAPACITY; + } + + public static void checkForceCapacity(long count) + throws LimitExceedException { + if (count > DEFAULT_CAPACITY) { + throw new LimitExceedException( + "Too many records(must <= %s) for one query", + DEFAULT_CAPACITY); + } + } + + public boolean isTaskQuery() { + if (this.resultType() == HugeType.TASK || + this.resultType == HugeType.VARIABLE) { + return true; + } + + return false; + } + + public static int getIndexStringValueLength() { + return indexStringValueLength; + } + + public static void setIndexStringValueLength(int indexStringValueLengthTmp) { + if (indexStringValueLengthTmp <= 1) { + indexStringValueLengthTmp = 20; + } + indexStringValueLength = indexStringValueLengthTmp; + } + + public void select(Id id) { + if (!this.selects.contains(id)) { + this.selects.add(id); + } else { + LOG.warn("id already in selects: {}", id); + } + } + + public List selects() { + return this.selects; + } + + public void group(Id id) { + if (!this.groups.contains(id)) { + this.groups.add(id); + } else { + LOG.warn("id already in groups: {}", id); + } + } + + public enum OrderType { + // Under batch interface, the requirement for return order + ORDER_NONE, // Allow unordered + ORDER_WITHIN_VERTEX, // Edges within a vertex will not be broken, but there is no order between different vertices. + ORDER_STRICT // Ensure the original input point order + } + + public enum Order { + ASC, + DESC + } + + public enum AggType { + COUNT, + MAX, + MIN, + AVG, + SUM; + } + + +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/AbstractSerializerAdapter.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/AbstractSerializerAdapter.java new file mode 100644 index 000000000..053f4ff14 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/AbstractSerializerAdapter.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.query.serializer; + +import java.lang.reflect.Type; +import java.util.Map; + +import org.apache.hugegraph.exception.BackendException; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +// TODO: optimize by binary protocol +public abstract class AbstractSerializerAdapter implements JsonSerializer, + JsonDeserializer { + + //Note: By overriding the method to get the mapping + public abstract Map validType(); + + @Override + public T deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws + JsonParseException { + JsonObject object = json.getAsJsonObject(); + String type = object.get("cls").getAsString(); + JsonElement element = object.get("el"); + try { + return context.deserialize(element, validType().get(type)); + } catch (Exception e) { + throw new BackendException("Unknown element type: " + type, e); + } + } + + @Override + public JsonElement serialize(T src, Type typeOfSrc, JsonSerializationContext context) { + JsonObject result = new JsonObject(); + Class clazz = src.getClass(); + result.add("cls", new JsonPrimitive(clazz.getSimpleName().substring(0, 1).toUpperCase())); + result.add("el", context.serialize(src, clazz)); + return result; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryAdapter.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryAdapter.java new file mode 100644 index 000000000..e9975f57c --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryAdapter.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.query.serializer; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.apache.hugegraph.query.Condition; +import org.apache.hugegraph.type.define.Directions; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.reflect.TypeToken; + +public class QueryAdapter extends AbstractSerializerAdapter { + + static ImmutableMap cls = + ImmutableMap.builder() + // TODO: uncomment later + .put("N", Condition.Not.class) + .put("A", Condition.And.class) + .put("O", Condition.Or.class) + .put("S", Condition.SyspropRelation.class) + .put("U", Condition.UserpropRelation.class) + .build(); + + static boolean isPrimitive(Class clz) { + try { + return (clz == Date.class) || ((Class) clz.getField("TYPE").get(null)).isPrimitive(); + } catch (Exception e) { + return false; + } + } + + @Override + public Map validType() { + return cls; + } + + @Override + public Condition deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + Condition condition = super.deserialize(json, typeOfT, context); + if (condition instanceof Condition.Relation) { + JsonObject object = json.getAsJsonObject(); + if (object.has("el")) { + JsonElement elElement = object.get("el"); + JsonElement valueElement = elElement.getAsJsonObject().get("value"); + if (valueElement.isJsonObject()) { + String cls = valueElement.getAsJsonObject().get("cls").getAsString(); + try { + Class actualClass = Class.forName(cls); + Object obj = context.deserialize(valueElement, actualClass); + ((Condition.Relation) condition).value(obj); + } catch (ClassNotFoundException e) { + throw new JsonParseException(e.getMessage()); + } + } else if (elElement.getAsJsonObject().has("valuecls")) { + if (valueElement.isJsonArray()) { + String cls = elElement.getAsJsonObject().get("valuecls").getAsString(); + try { + Class actualClass = Class.forName(cls); + Type type = TypeToken.getParameterized(ArrayList.class, actualClass) + .getType(); + Object value = context.deserialize(valueElement, type); + ((Condition.Relation) condition).value(value); + } catch (ClassNotFoundException e) { + throw new JsonParseException(e.getMessage()); + } + } else { + String cls = elElement.getAsJsonObject().get("valuecls").getAsString(); + try { + Class actualClass = Class.forName(cls); + Object obj = context.deserialize(valueElement, actualClass); + ((Condition.Relation) condition).value(obj); + } catch (ClassNotFoundException e) { + throw new JsonParseException(e.getMessage()); + } + } + + } else if (valueElement.isJsonPrimitive() && + valueElement.getAsJsonPrimitive().isString()) { + switch ((String) ((Condition.Relation) condition).value()) { + case "OUT": + ((Condition.Relation) condition).value(Directions.OUT); + break; + case "IN": + ((Condition.Relation) condition).value(Directions.IN); + break; + default: + break; + } + } + } + } + return condition; + } + + @Override + public JsonElement serialize(Condition src, Type typeOfSrc, JsonSerializationContext context) { + JsonElement result = super.serialize(src, typeOfSrc, context); + if (src instanceof Condition.Relation) { + JsonObject object = result.getAsJsonObject(); + JsonElement valueElement = object.get("el").getAsJsonObject().get("value"); + if (valueElement.isJsonObject()) { + valueElement.getAsJsonObject() + .add("cls", + new JsonPrimitive( + ((Condition.Relation) src).value().getClass().getName())); + } else if (isPrimitive(((Condition.Relation) src).value().getClass())) { + object.get("el").getAsJsonObject() + .add("valuecls", + new JsonPrimitive( + ((Condition.Relation) src).value().getClass().getName())); + } else if (valueElement.isJsonArray()) { + if (((Condition.Relation) src).value() instanceof List) { + String valueCls = + ((List) ((Condition.Relation) src).value()).get(0).getClass().getName(); + object.get("el").getAsJsonObject().add("valuecls", new JsonPrimitive(valueCls)); + } + } + } + return result; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryIdAdapter.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryIdAdapter.java new file mode 100644 index 000000000..53f414512 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/query/serializer/QueryIdAdapter.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.query.serializer; + +import java.lang.reflect.Type; +import java.util.Map; + +import org.apache.hugegraph.backend.BinaryId; +import org.apache.hugegraph.id.EdgeId; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; + +import com.google.common.collect.ImmutableMap; + +public class QueryIdAdapter extends AbstractSerializerAdapter { + + static ImmutableMap cls = + ImmutableMap.builder() + .put("E", EdgeId.class) + .put("S", IdGenerator.StringId.class) + .put("L", IdGenerator.LongId.class) + .put("U", IdGenerator.UuidId.class) + .put("O", IdGenerator.ObjectId.class) + .put("B", BinaryId.class) + .build(); + + @Override + public Map validType() { + return cls; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/EdgeLabel.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/EdgeLabel.java new file mode 100644 index 000000000..443b55421 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/EdgeLabel.java @@ -0,0 +1,449 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.schema; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hugegraph.HugeGraphSupplier; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.schema.builder.SchemaBuilder; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.Directions; +import org.apache.hugegraph.type.define.EdgeLabelType; +import org.apache.hugegraph.type.define.Frequency; +import org.apache.hugegraph.type.define.SchemaStatus; +import org.apache.hugegraph.util.E; + +import com.google.common.base.Objects; + +public class EdgeLabel extends SchemaLabel { + + public static final EdgeLabel NONE = new EdgeLabel(null, NONE_ID, UNDEF); + + private Set> links = new HashSet<>(); + private Id sourceLabel = NONE_ID; + private Id targetLabel = NONE_ID; + private Frequency frequency; + private List sortKeys; + + private EdgeLabelType edgeLabelType = EdgeLabelType.NORMAL; + private Id fatherId; + + public EdgeLabel(final HugeGraphSupplier graph, Id id, String name) { + super(graph, id, name); + this.frequency = Frequency.DEFAULT; + this.sortKeys = new ArrayList<>(); + } + + @Override + public HugeType type() { + return HugeType.EDGE_LABEL; + } + + public boolean isFather() { + return this.edgeLabelType.parent(); + } + + public void edgeLabelType(EdgeLabelType type) { + this.edgeLabelType = type; + } + + public EdgeLabelType edgeLabelType() { + return this.edgeLabelType; + } + + public boolean hasFather() { + return this.edgeLabelType.sub(); + } + + public boolean general() { + return this.edgeLabelType.general(); + } + + public Id fatherId() { + return this.fatherId; + } + + public void fatherId(Id fatherId) { + this.fatherId = fatherId; + } + + public Frequency frequency() { + return this.frequency; + } + + public void frequency(Frequency frequency) { + this.frequency = frequency; + } + + public boolean directed() { + // TODO: implement (do we need this method?) + return true; + } + + public String sourceLabelName() { + E.checkState(this.links.size() == 1, + "Only edge label has single vertex label pair can call " + + "sourceLabelName(), but current edge label got %s", + this.links.size()); + return this.graph.vertexLabelOrNone(this.links.iterator().next().getLeft()).name(); + } + + public List linksIds() { + List ids = new ArrayList<>(this.links.size() * 2); + for (Pair link : this.links) { + ids.add(link.getLeft()); + ids.add(link.getRight()); + } + return ids; + } + + public void linksIds(Id[] ids) { + this.links = new HashSet<>(ids.length / 2); + for (int i = 0; i < ids.length; i += 2) { + this.links.add(Pair.of(ids[i], ids[i + 1])); + } + } + + public Id sourceLabel() { + if (links.size() == 1) { + return links.iterator().next().getLeft(); + } + return NONE_ID; + } + + public void sourceLabel(Id id) { + E.checkArgument(this.links.isEmpty(), + "Not allowed add source label to an edge label which " + + "already has links"); + if (this.targetLabel != NONE_ID) { + this.links.add(Pair.of(id, this.targetLabel)); + this.targetLabel = NONE_ID; + } else { + this.sourceLabel = id; + } + } + + public String targetLabelName() { + E.checkState(this.links.size() == 1, + "Only edge label has single vertex label pair can call " + + "sourceLabelName(), but current edge label got %s", + this.links.size()); + return this.graph.vertexLabelOrNone(this.links.iterator().next().getRight()).name(); + } + + public Id targetLabel() { + if (links.size() == 1) { + return links.iterator().next().getRight(); + } + return NONE_ID; + } + + public void targetLabel(Id id) { + E.checkArgument(this.links.isEmpty(), + "Not allowed add source label to an edge label which " + + "already has links"); + if (this.sourceLabel != NONE_ID) { + this.links.add(Pair.of(this.sourceLabel, id)); + this.sourceLabel = NONE_ID; + } else { + this.targetLabel = id; + } + } + + public boolean linkWithLabel(Id id) { + for (Pair link : this.links) { + if (link.getLeft().equals(id) || link.getRight().equals(id)) { + return true; + } + } + return false; + } + + public boolean linkWithVertexLabel(Id label, Directions dir) { + return this.links.stream().anyMatch(pair -> { + Id sourceLabel = pair.getLeft(); + Id targetLabel = pair.getRight(); + if (dir.equals(Directions.IN)) { + return targetLabel.equals(label); + } else if (dir.equals(Directions.OUT)) { + return sourceLabel.equals(label); + } else if (dir.equals(Directions.BOTH)) { + return targetLabel.equals(label) || sourceLabel.equals(label); + } + return false; + }); + } + + public boolean checkLinkEqual(Id sourceLabel, Id targetLabel) { + return this.links.contains(Pair.of(sourceLabel, targetLabel)); + } + + public Set> links() { + return this.links; + } + + public void links(Pair link) { + if (this.links == null) { + this.links = new HashSet<>(); + } + this.links.add(link); + } + + public boolean existSortKeys() { + return !this.sortKeys.isEmpty(); + } + + public List sortKeys() { + return Collections.unmodifiableList(this.sortKeys); + } + + public void sortKey(Id id) { + this.sortKeys.add(id); + } + + public void sortKeys(Id... ids) { + this.sortKeys.addAll(Arrays.asList(ids)); + } + + public boolean hasSameContent(EdgeLabel other) { + return super.hasSameContent(other) && + this.frequency == other.frequency && + Objects.equal(this.sourceLabelName(), other.sourceLabelName()) && + Objects.equal(this.targetLabelName(), other.targetLabelName()) && + Objects.equal(this.graph.mapPkId2Name(this.sortKeys), + other.graph.mapPkId2Name(other.sortKeys)); + } + + public static EdgeLabel undefined(HugeGraphSupplier graph, Id id) { + return new EdgeLabel(graph, id, UNDEF); + } + + public interface Builder extends SchemaBuilder { + + Id rebuildIndex(); + + Builder asBase(); + + Builder withBase(String fatherLabel); + + Builder link(String sourceLabel, String targetLabel); + + @Deprecated + Builder sourceLabel(String label); + + @Deprecated + Builder targetLabel(String label); + + Builder singleTime(); + + Builder multiTimes(); + + Builder sortKeys(String... keys); + + Builder properties(String... properties); + + Builder nullableKeys(String... keys); + + Builder frequency(Frequency frequency); + + Builder ttl(long ttl); + + Builder ttlStartTime(String ttlStartTime); + + Builder enableLabelIndex(boolean enable); + + Builder userdata(String key, Object value); + + Builder userdata(Map userdata); + } + + @Override + public Map asMap() { + Map map = new HashMap<>(); + + if (this.sourceLabel() != null && this.sourceLabel() != NONE_ID) { + map.put(P.SOURCE_LABEL, this.sourceLabel().asString()); + } + + if (this.targetLabel() != null && this.targetLabel() != NONE_ID) { + map.put(P.TARGET_LABEL, this.targetLabel().asString()); + } + + if (this.properties() != null) { + map.put(P.PROPERTIES, this.properties()); + } + + if (this.nullableKeys() != null) { + map.put(P.NULLABLE_KEYS, this.nullableKeys()); + } + + if (this.indexLabels() != null) { + map.put(P.INDEX_LABELS, this.indexLabels()); + } + + if (this.ttlStartTime() != null) { + map.put(P.TT_START_TIME, this.ttlStartTime().asString()); + } + + if (this.sortKeys() != null) { + map.put(P.SORT_KEYS, this.sortKeys); + } + + map.put(P.EDGELABEL_TYPE, this.edgeLabelType); + if (this.fatherId() != null) { + map.put(P.FATHER_ID, this.fatherId().asString()); + } + map.put(P.ENABLE_LABEL_INDEX, this.enableLabelIndex()); + map.put(P.TTL, String.valueOf(this.ttl())); + map.put(P.LINKS, this.links()); + map.put(P.FREQUENCY, this.frequency().toString()); + + return super.asMap(map); + } + + @SuppressWarnings("unchecked") + public static EdgeLabel fromMap(Map map, HugeGraphSupplier graph) { + Id id = IdGenerator.of((int) map.get(EdgeLabel.P.ID)); + String name = (String) map.get(EdgeLabel.P.NAME); + EdgeLabel edgeLabel = new EdgeLabel(graph, id, name); + for (Map.Entry entry : map.entrySet()) { + switch (entry.getKey()) { + case P.ID: + case P.NAME: + break; + case P.STATUS: + edgeLabel.status( + SchemaStatus.valueOf(((String) entry.getValue()).toUpperCase())); + break; + case P.USERDATA: + edgeLabel.userdata(new Userdata((Map) entry.getValue())); + break; + case P.PROPERTIES: + Set ids = ((List) entry.getValue()).stream().map( + IdGenerator::of).collect(Collectors.toSet()); + edgeLabel.properties(ids); + break; + case P.NULLABLE_KEYS: + ids = ((List) entry.getValue()).stream().map( + IdGenerator::of).collect(Collectors.toSet()); + edgeLabel.nullableKeys(ids); + break; + case P.INDEX_LABELS: + ids = ((List) entry.getValue()).stream().map( + IdGenerator::of).collect(Collectors.toSet()); + edgeLabel.addIndexLabels(ids.toArray(new Id[0])); + break; + case P.ENABLE_LABEL_INDEX: + boolean enableLabelIndex = (Boolean) entry.getValue(); + edgeLabel.enableLabelIndex(enableLabelIndex); + break; + case P.TTL: + long ttl = Long.parseLong((String) entry.getValue()); + edgeLabel.ttl(ttl); + break; + case P.TT_START_TIME: + long ttlStartTime = + Long.parseLong((String) entry.getValue()); + edgeLabel.ttlStartTime(IdGenerator.of(ttlStartTime)); + break; + case P.LINKS: + // TODO: serialize and deserialize + List list = (List) entry.getValue(); + for (Map m : list) { + for (Object key : m.keySet()) { + Id sid = IdGenerator.of(Long.parseLong((String) key)); + Id tid = IdGenerator.of(Long.parseLong(String.valueOf(m.get(key)))); + edgeLabel.links(Pair.of(sid, tid)); + } + } + break; + case P.SOURCE_LABEL: + long sourceLabel = + Long.parseLong((String) entry.getValue()); + edgeLabel.sourceLabel(IdGenerator.of(sourceLabel)); + break; + case P.TARGET_LABEL: + long targetLabel = + Long.parseLong((String) entry.getValue()); + edgeLabel.targetLabel(IdGenerator.of(targetLabel)); + break; + case P.FATHER_ID: + long fatherId = + Long.parseLong((String) entry.getValue()); + edgeLabel.fatherId(IdGenerator.of(fatherId)); + break; + case P.EDGELABEL_TYPE: + EdgeLabelType edgeLabelType = + EdgeLabelType.valueOf( + ((String) entry.getValue()).toUpperCase()); + edgeLabel.edgeLabelType(edgeLabelType); + break; + case P.FREQUENCY: + Frequency frequency = + Frequency.valueOf(((String) entry.getValue()).toUpperCase()); + edgeLabel.frequency(frequency); + break; + case P.SORT_KEYS: + ids = ((List) entry.getValue()).stream().map( + IdGenerator::of).collect(Collectors.toSet()); + edgeLabel.sortKeys(ids.toArray(new Id[0])); + break; + default: + throw new AssertionError(String.format( + "Invalid key '%s' for edge label", + entry.getKey())); + } + } + return edgeLabel; + } + + public static final class P { + + public static final String ID = "id"; + public static final String NAME = "name"; + + public static final String STATUS = "status"; + public static final String USERDATA = "userdata"; + + public static final String PROPERTIES = "properties"; + public static final String NULLABLE_KEYS = "nullableKeys"; + public static final String INDEX_LABELS = "indexLabels"; + + public static final String ENABLE_LABEL_INDEX = "enableLabelIndex"; + public static final String TTL = "ttl"; + public static final String TT_START_TIME = "ttlStartTime"; + public static final String LINKS = "links"; + public static final String SOURCE_LABEL = "sourceLabel"; + public static final String TARGET_LABEL = "targetLabel"; + public static final String EDGELABEL_TYPE = "edgeLabelType"; + public static final String FATHER_ID = "fatherId"; + public static final String FREQUENCY = "frequency"; + public static final String SORT_KEYS = "sortKeys"; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/IndexLabel.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/IndexLabel.java new file mode 100644 index 000000000..c3a49467c --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/IndexLabel.java @@ -0,0 +1,498 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.schema; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.hugegraph.HugeGraphSupplier; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.schema.builder.SchemaBuilder; +import org.apache.hugegraph.type.define.IndexType; +import org.apache.hugegraph.type.define.SchemaStatus; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.util.GraphUtils; +import org.apache.hugegraph.util.E; + +import com.google.common.base.Objects; + +public class IndexLabel extends SchemaElement { + + private HugeType baseType; + private Id baseValue; + private IndexType indexType; + private List indexFields; + + public IndexLabel(final HugeGraphSupplier graph, Id id, String name) { + super(graph, id, name); + this.baseType = HugeType.SYS_SCHEMA; + this.baseValue = NONE_ID; + this.indexType = IndexType.SECONDARY; + this.indexFields = new ArrayList<>(); + } + + protected IndexLabel(long id, String name) { + this(null, IdGenerator.of(id), name); + } + + @Override + public HugeType type() { + return HugeType.INDEX_LABEL; + } + + public HugeType baseType() { + return this.baseType; + } + + public void baseType(HugeType baseType) { + this.baseType = baseType; + } + + public Id baseValue() { + return this.baseValue; + } + + public void baseValue(Id id) { + this.baseValue = id; + } + + public IndexType indexType() { + return this.indexType; + } + + public void indexType(IndexType indexType) { + this.indexType = indexType; + } + + public HugeType queryType() { + switch (this.baseType) { + case VERTEX_LABEL: + return HugeType.VERTEX; + case EDGE_LABEL: + return HugeType.EDGE; + case SYS_SCHEMA: + return HugeType.SYS_SCHEMA; + default: + throw new AssertionError(String.format( + "Query type of index label is either '%s' or '%s', " + + "but '%s' is used", + HugeType.VERTEX_LABEL, HugeType.EDGE_LABEL, + this.baseType)); + } + } + + public List indexFields() { + return Collections.unmodifiableList(this.indexFields); + } + + public void indexFields(Id... ids) { + this.indexFields.addAll(Arrays.asList(ids)); + } + + public void indexField(Id id) { + this.indexFields.add(id); + } + + public Id indexField() { + E.checkState(this.indexFields.size() == 1, + "There should be only one field in %s index label, " + + "but got: %s", this.indexType.string(), this.indexFields); + return this.indexFields.get(0); + } + + public SchemaLabel baseLabel() { + return getBaseLabel(this.graph, this.baseType, this.baseValue); + } + + public SchemaLabel baseElement() { + return getElement(this.graph, this.baseType, this.baseValue); + } + + public boolean hasSameContent(IndexLabel other) { + return super.hasSameContent(other) && + this.indexType == other.indexType && + this.baseType == other.baseType && + Objects.equal(this.graph.mapPkId2Name(this.indexFields), + other.graph.mapPkId2Name(other.indexFields)); + } + + public boolean olap() { + return VertexLabel.OLAP_VL.id().equals(this.baseValue); + } + + public Object validValue(Object value) { + if (!(value instanceof Number)) { + return value; + } + + Number number = (Number) value; + switch (this.indexType()) { + case RANGE_INT: + return number.intValue(); + case RANGE_LONG: + return number.longValue(); + case RANGE_FLOAT: + return number.floatValue(); + case RANGE_DOUBLE: + return number.doubleValue(); + default: + return value; + } + } + + // Label index + private static final IndexLabel VL_IL = new IndexLabel(VL_IL_ID, "~vli"); + private static final IndexLabel EL_IL = new IndexLabel(EL_IL_ID, "~eli"); + + // Schema name index + private static final IndexLabel PKN_IL = new IndexLabel(PKN_IL_ID, "~pkni"); + private static final IndexLabel VLN_IL = new IndexLabel(VLN_IL_ID, "~vlni"); + private static final IndexLabel ELN_IL = new IndexLabel(ELN_IL_ID, "~elni"); + private static final IndexLabel ILN_IL = new IndexLabel(ILN_IL_ID, "~ilni"); + + public static IndexLabel label(HugeType type) { + switch (type) { + case TASK: + case SERVER: + case VERTEX: + return VL_IL; + case EDGE: + case EDGE_OUT: + case EDGE_IN: + return EL_IL; + case PROPERTY_KEY: + return PKN_IL; + case VERTEX_LABEL: + return VLN_IL; + case EDGE_LABEL: + return ELN_IL; + case INDEX_LABEL: + return ILN_IL; + default: + throw new AssertionError(String.format( + "No primitive index label for '%s'", type)); + } + } + + public static IndexLabel label(HugeGraphSupplier graph, Id id) { + // Primitive IndexLabel first + if (id.asLong() < 0 && id.asLong() > -NEXT_PRIMITIVE_SYS_ID) { + switch ((int) id.asLong()) { + case VL_IL_ID: + return VL_IL; + case EL_IL_ID: + return EL_IL; + case PKN_IL_ID: + return PKN_IL; + case VLN_IL_ID: + return VLN_IL; + case ELN_IL_ID: + return ELN_IL; + case ILN_IL_ID: + return ILN_IL; + default: + throw new AssertionError(String.format( + "No primitive index label for '%s'", id)); + } + } + return graph.indexLabel(id); + } + + public static SchemaLabel getBaseLabel(HugeGraphSupplier graph, + HugeType baseType, + Object baseValue) { + E.checkNotNull(baseType, "base type", "index label"); + E.checkNotNull(baseValue, "base value", "index label"); + E.checkArgument(baseValue instanceof String || baseValue instanceof Id, + "The base value must be instance of String or Id, " + + "but got %s(%s)", baseValue, + baseValue.getClass().getSimpleName()); + + SchemaLabel label; + switch (baseType) { + case VERTEX_LABEL: + if (baseValue instanceof String) { + label = graph.vertexLabel((String) baseValue); + } else { + assert baseValue instanceof Id; + label = graph.vertexLabel((Id) baseValue); + } + break; + case EDGE_LABEL: + if (baseValue instanceof String) { + label = graph.edgeLabel((String) baseValue); + } else { + assert baseValue instanceof Id; + label = graph.edgeLabel((Id) baseValue); + } + break; + default: + throw new AssertionError(String.format( + "Unsupported base type '%s' of index label", + baseType)); + } + + E.checkArgumentNotNull(label, "Can't find the %s with name '%s'", + baseType.readableName(), baseValue); + return label; + } + + public static SchemaLabel getElement(HugeGraphSupplier graph, + HugeType baseType, Object baseValue) { + E.checkNotNull(baseType, "base type", "index label"); + E.checkNotNull(baseValue, "base value", "index label"); + E.checkArgument(baseValue instanceof String || baseValue instanceof Id, + "The base value must be instance of String or Id, " + + "but got %s(%s)", baseValue, + baseValue.getClass().getSimpleName()); + + SchemaLabel label; + switch (baseType) { + case VERTEX_LABEL: + if (baseValue instanceof String) { + label = graph.vertexLabel((String) baseValue); + } else { + assert baseValue instanceof Id; + label = graph.vertexLabel((Id) baseValue); + } + break; + case EDGE_LABEL: + if (baseValue instanceof String) { + label = graph.edgeLabel((String) baseValue); + } else { + assert baseValue instanceof Id; + label = graph.edgeLabel((Id) baseValue); + } + break; + default: + throw new AssertionError(String.format( + "Unsupported base type '%s' of index label", + baseType)); + } + + E.checkArgumentNotNull(label, "Can't find the %s with name '%s'", + baseType.readableName(), baseValue); + return label; + } + + public String convert2Groovy(boolean attachIdFlag) { + StringBuilder builder = new StringBuilder(SCHEMA_PREFIX); + + // Name + if (!attachIdFlag) { + builder.append("indexLabel").append("('") + .append(this.name()) + .append("')"); + } else { + builder.append("indexLabel").append("(") + .append(longId()).append(", '") + .append(this.name()) + .append("')"); + } + + // On + switch (this.baseType()) { + case VERTEX_LABEL: + VertexLabel vl = this.graph.vertexLabel(this.baseValue); + builder.append(".onV('") + .append(vl.name()) + .append("')"); + break; + case EDGE_LABEL: + EdgeLabel el = this.graph.edgeLabel(this.baseValue); + builder.append(".onE('") + .append(el.name()) + .append("')"); + break; + default: + throw new AssertionError(String.format( + "Invalid base type '%s'", this.baseType())); + } + + // By + builder.append(".by("); + List properties = this.indexFields(); + int size = properties.size(); + for (Id id : properties) { + PropertyKey pk = this.graph.propertyKey(id); + builder.append("'") + .append(pk.name()) + .append("'"); + if (--size > 0) { + builder.append(","); + } + } + builder.append(")"); + + // Index type + builder.append("."); + switch (this.indexType()) { + case SECONDARY: + builder.append("secondary()"); + break; + case RANGE_INT: + case RANGE_LONG: + case RANGE_FLOAT: + case RANGE_DOUBLE: + builder.append("range()"); + break; + case SEARCH: + builder.append("search()"); + break; + case SHARD: + builder.append("shard()"); + break; + case UNIQUE: + builder.append("unique()"); + break; + default: + throw new AssertionError(String.format( + "Invalid index type '%s'", this.indexType())); + } + + // User data + Map userdata = this.userdata(); + if (userdata.isEmpty()) { + return builder.toString(); + } + for (Map.Entry entry : userdata.entrySet()) { + if (GraphUtils.isHidden(entry.getKey())) { + continue; + } + builder.append(".userdata('") + .append(entry.getKey()) + .append("',") + .append(entry.getValue()) + .append(")"); + } + + builder.append(".ifNotExist().create();"); + return builder.toString(); + } + + public interface Builder extends SchemaBuilder { + + TaskWithSchema createWithTask(); + + Id rebuild(); + + Builder onV(String baseValue); + + Builder onE(String baseValue); + + Builder by(String... fields); + + Builder secondary(); + + Builder range(); + + Builder search(); + + Builder shard(); + + Builder unique(); + + Builder on(HugeType baseType, String baseValue); + + Builder indexType(IndexType indexType); + + Builder userdata(String key, Object value); + + Builder userdata(Map userdata); + + Builder rebuild(boolean rebuild); + } + + @Override + public Map asMap() { + HashMap map = new HashMap<>(); + map.put(P.BASE_TYPE, this.baseType().name()); + map.put(P.BASE_VALUE, this.baseValue().asString()); + map.put(P.INDEX_TYPE, this.indexType().name()); + map.put(P.INDEX_FIELDS, this.indexFields()); + return super.asMap(map); + } + + @SuppressWarnings("unchecked") + public static IndexLabel fromMap(Map map, HugeGraphSupplier graph) { + Id id = IdGenerator.of((int) map.get(IndexLabel.P.ID)); + String name = (String) map.get(IndexLabel.P.NAME); + + IndexLabel indexLabel = new IndexLabel(graph, id, name); + for (Map.Entry entry : map.entrySet()) { + switch (entry.getKey()) { + case P.ID: + case P.NAME: + break; + case P.STATUS: + indexLabel.status( + SchemaStatus.valueOf(((String) entry.getValue()).toUpperCase())); + break; + case P.USERDATA: + indexLabel.userdata(new Userdata((Map) entry.getValue())); + break; + case P.BASE_TYPE: + HugeType hugeType = + HugeType.valueOf(((String) entry.getValue()).toUpperCase()); + indexLabel.baseType(hugeType); + break; + case P.BASE_VALUE: + long sourceLabel = + Long.parseLong((String) entry.getValue()); + indexLabel.baseValue(IdGenerator.of(sourceLabel)); + break; + case P.INDEX_TYPE: + IndexType indexType = + IndexType.valueOf(((String) entry.getValue()).toUpperCase()); + indexLabel.indexType(indexType); + break; + case P.INDEX_FIELDS: + List ids = ((List) entry.getValue()).stream().map( + IdGenerator::of).collect(Collectors.toList()); + indexLabel.indexFields(ids.toArray(new Id[0])); + break; + default: + throw new AssertionError(String.format( + "Invalid key '%s' for index label", + entry.getKey())); + } + } + return indexLabel; + } + + public static final class P { + + public static final String ID = "id"; + public static final String NAME = "name"; + + public static final String STATUS = "status"; + public static final String USERDATA = "userdata"; + + public static final String BASE_TYPE = "baseType"; + public static final String BASE_VALUE = "baseValue"; + public static final String INDEX_TYPE = "indexType"; + public static final String INDEX_FIELDS = "indexFields"; + } + +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/PropertyKey.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/PropertyKey.java new file mode 100644 index 000000000..99a46d325 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/PropertyKey.java @@ -0,0 +1,646 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.schema; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +import org.apache.hugegraph.HugeGraphSupplier; + +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.exception.NotSupportException; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.schema.builder.SchemaBuilder; +import org.apache.hugegraph.type.HugeType; + +import org.apache.hugegraph.type.Propfiable; +import org.apache.hugegraph.type.define.AggregateType; +import org.apache.hugegraph.type.define.Cardinality; +import org.apache.hugegraph.type.define.DataType; +import org.apache.hugegraph.type.define.SchemaStatus; +import org.apache.hugegraph.type.define.WriteType; +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.GraphUtils; +import org.apache.hugegraph.util.LongEncoding; + +import static org.apache.hugegraph.type.define.WriteType.OLAP_COMMON; +import static org.apache.hugegraph.type.define.WriteType.OLAP_RANGE; +import static org.apache.hugegraph.type.define.WriteType.OLAP_SECONDARY; + +public class PropertyKey extends SchemaElement implements Propfiable { + + private DataType dataType; + private Cardinality cardinality; + private AggregateType aggregateType; + private WriteType writeType; + + public PropertyKey(final HugeGraphSupplier graph, Id id, String name) { + super(graph, id, name); + this.dataType = DataType.TEXT; + this.cardinality = Cardinality.SINGLE; + this.aggregateType = AggregateType.NONE; + this.writeType = WriteType.OLTP; + } + + @Override + public HugeType type() { + return HugeType.PROPERTY_KEY; + } + + public DataType dataType() { + return this.dataType; + } + + public void dataType(DataType dataType) { + this.dataType = dataType; + } + + public Cardinality cardinality() { + return this.cardinality; + } + + public void cardinality(Cardinality cardinality) { + this.cardinality = cardinality; + } + + public AggregateType aggregateType() { + return this.aggregateType; + } + + public void aggregateType(AggregateType aggregateType) { + this.aggregateType = aggregateType; + } + + public void writeType(WriteType writeType) { + this.writeType = writeType; + } + + public WriteType writeType() { + return this.writeType; + } + + public boolean oltp() { + return this.writeType.oltp(); + } + + public boolean olap() { + return this.writeType.olap(); + } + + @Override + public Set properties() { + return Collections.emptySet(); + } + + public PropertyKey properties(Id... properties) { + if (properties.length > 0) { + throw new NotSupportException("PropertyKey.properties(Id)"); + } + return this; + } + + public void defineDefaultValue(Object value) { + // TODO add a field default_value + this.userdata().put(Userdata.DEFAULT_VALUE, value); + } + + public Object defaultValue() { + // TODO add a field default_value + return this.userdata().get(Userdata.DEFAULT_VALUE); + } + + public boolean hasSameContent(PropertyKey other) { + return super.hasSameContent(other) && + this.dataType == other.dataType() && + this.cardinality == other.cardinality() && + this.aggregateType == other.aggregateType() && + this.writeType == other.writeType(); + } + + public String clazz() { + String dataType = this.dataType().clazz().getSimpleName(); + switch (this.cardinality) { + case SINGLE: + return dataType; + // A set of values: Set + case SET: + return String.format("Set<%s>", dataType); + // A list of values: List + case LIST: + return String.format("List<%s>", dataType); + default: + throw new AssertionError(String.format( + "Unsupported cardinality: '%s'", this.cardinality)); + } + } + + public Class implementClazz() { + Class cls; + switch (this.cardinality) { + case SINGLE: + cls = this.dataType().clazz(); + break; + // A set of values: Set + case SET: + cls = LinkedHashSet.class; + break; + // A list of values: List + case LIST: + cls = ArrayList.class; + break; + default: + throw new AssertionError(String.format( + "Unsupported cardinality: '%s'", this.cardinality)); + } + return cls; + } + + @SuppressWarnings("unchecked") + public T newValue() { + switch (this.cardinality) { + case SET: + return (T) new LinkedHashSet<>(); + case LIST: + return (T) new ArrayList<>(); + default: + // pass + break; + } + + try { + return (T) this.implementClazz().newInstance(); + } catch (Exception e) { + throw new HugeException("Failed to new instance of %s: %s", + this.implementClazz(), e.toString()); + } + } + + /** + * Check property value valid + * + * @param value the property value to be checked data type and cardinality + * @param the property value class + * @return true if data type and cardinality satisfy requirements, + * otherwise false + */ + public boolean checkValueType(V value) { + boolean valid; + + switch (this.cardinality) { + case SINGLE: + valid = this.checkDataType(value); + break; + case SET: + valid = value instanceof Set; + valid = valid && this.checkDataType((Set) value); + break; + case LIST: + valid = value instanceof List; + valid = valid && this.checkDataType((List) value); + break; + default: + throw new AssertionError(String.format( + "Unsupported cardinality: '%s'", this.cardinality)); + } + return valid; + } + + /** + * Check type of the value valid + * + * @param value the property value to be checked data type + * @param the property value original data type + * @return true if the value is or can convert to the data type, + * otherwise false + */ + private boolean checkDataType(V value) { + return this.dataType().clazz().isInstance(value); + } + + /** + * Check type of all the values(maybe some list properties) valid + * + * @param values the property values to be checked data type + * @param the property value class + * @return true if all the values are or can convert to the data type, + * otherwise false + */ + private boolean checkDataType(Collection values) { + boolean valid = true; + for (Object o : values) { + if (!this.checkDataType(o)) { + valid = false; + break; + } + } + return valid; + } + + public Object serialValue(V value, boolean encodeNumber) { + V validValue = this.validValue(value); + E.checkArgument(validValue != null, + "Invalid property value '%s' for key '%s'", + value, this.name()); + E.checkArgument(this.cardinality.single(), + "The cardinality can't be '%s' for navigation key '%s'", + this.cardinality, this.name()); + if (this.dataType.isNumber() || this.dataType.isDate()) { + if (encodeNumber) { + return LongEncoding.encodeNumber(validValue); + } else { + return validValue.toString(); + } + } + return validValue; + } + + public V validValueOrThrow(V value) { + V validValue = this.validValue(value); + if (validValue == null) { + E.checkArgument(false, + "Invalid property value '%s' for key '%s', " + + "expect a value of type %s, actual type %s", + value, this.name(), this.clazz(), + value.getClass().getSimpleName()); + } + return validValue; + } + + public V validValue(V value) { + try { + return this.convValue(value); + } catch (RuntimeException e) { + throw new IllegalArgumentException(String.format( + "Invalid property value '%s' for key '%s': %s", + value, this.name(), e.getMessage())); + } + } + + @SuppressWarnings("unchecked") + private V convValue(V value) { + if (value == null) { + return null; + } + if (this.checkValueType(value)) { + // Same as expected type, no conversion required + return value; + } + + V validValue = null; + Collection validValues; + if (this.cardinality.single()) { + validValue = this.convSingleValue(value); + } else if (value instanceof Collection) { + assert this.cardinality.multiple(); + Collection collection = (Collection) value; + if (value instanceof Set) { + validValues = new LinkedHashSet<>(collection.size()); + } else { + assert value instanceof List; + validValues = new ArrayList<>(collection.size()); + } + for (T element : collection) { + element = this.convSingleValue(element); + if (element == null) { + validValues = null; + break; + } + validValues.add(element); + } + validValue = (V) validValues; + } else { + assert this.cardinality.multiple(); + E.checkArgument(false, + "Property value must be %s, but got '%s'(%s)", + this.cardinality, value, + value.getClass().getSimpleName()); + } + return validValue; + } + + private V convSingleValue(V value) { + if (value == null) { + return null; + } + if (this.dataType().isNumber()) { + @SuppressWarnings("unchecked") + V number = (V) this.dataType().valueToNumber(value); + return number; + } else if (this.dataType().isDate()) { + @SuppressWarnings("unchecked") + V date = (V) this.dataType().valueToDate(value); + return date; + } else if (this.dataType().isUUID()) { + @SuppressWarnings("unchecked") + V uuid = (V) this.dataType().valueToUUID(value); + return uuid; + } else if (this.dataType().isBlob()) { + @SuppressWarnings("unchecked") + V blob = (V) this.dataType().valueToBlob(value); + return blob; + } + + if (this.checkDataType(value)) { + return value; + } + return null; + } + + public String convert2Groovy(boolean attachIdFlag) { + StringBuilder builder = new StringBuilder(SCHEMA_PREFIX); + // Name + if (!attachIdFlag) { + builder.append("propertyKey").append("('") + .append(this.name()) + .append("')"); + } else { + builder.append("propertyKey").append("(") + .append(longId()).append(", '") + .append(this.name()) + .append("')"); + } + + // DataType + switch (this.dataType()) { + case INT: + builder.append(".asInt()"); + break; + case LONG: + builder.append(".asLong()"); + break; + case DOUBLE: + builder.append(".asDouble()"); + break; + case BYTE: + builder.append(".asByte()"); + break; + case DATE: + builder.append(".asDate()"); + break; + case FLOAT: + builder.append(".asFloat()"); + break; + case BLOB: + builder.append(".asBlob()"); + break; + case TEXT: + builder.append(".asText()"); + break; + case UUID: + builder.append(".asUUID()"); + break; + case OBJECT: + builder.append(".asObject()"); + break; + case BOOLEAN: + builder.append(".asBoolean()"); + break; + default: + throw new AssertionError(String.format( + "Invalid data type '%s'", this.dataType())); + } + + // Cardinality + switch (this.cardinality()) { + case SINGLE: + // Single is default, prefer not output + break; + case SET: + builder.append(".valueSet()"); + break; + case LIST: + builder.append(".valueList()"); + break; + default: + throw new AssertionError(String.format( + "Invalid cardinality '%s'", this.cardinality())); + } + + // Aggregate type + switch (this.aggregateType()) { + case NONE: + // NONE is default, prefer not output + break; + case MAX: + builder.append(".calcMax()"); + break; + case MIN: + builder.append(".calcMin()"); + break; + case SUM: + builder.append(".calcSum()"); + break; + case LIST: + builder.append(".calcList()"); + break; + case SET: + builder.append(".calcSet()"); + break; + case OLD: + builder.append(".calcOld()"); + break; + default: + throw new AssertionError(String.format( + "Invalid cardinality '%s'", this.aggregateType())); + } + + // Write type + switch (this.writeType()) { + case OLTP: + // OLTP is default, prefer not output + break; + case OLAP_COMMON: + builder.append(".writeType('") + .append(OLAP_COMMON) + .append("')"); + break; + case OLAP_RANGE: + builder.append(".writeType('") + .append(OLAP_RANGE) + .append("')"); + break; + case OLAP_SECONDARY: + builder.append(".writeType('") + .append(OLAP_SECONDARY) + .append("')"); + break; + default: + throw new AssertionError(String.format( + "Invalid write type '%s'", this.writeType())); + } + + // User data + Map userdata = this.userdata(); + if (userdata.isEmpty()) { + return builder.toString(); + } + for (Map.Entry entry : userdata.entrySet()) { + if (GraphUtils.isHidden(entry.getKey())) { + continue; + } + builder.append(".userdata('") + .append(entry.getKey()) + .append("',") + .append(entry.getValue()) + .append(")"); + } + + builder.append(".ifNotExist().create();"); + return builder.toString(); + } + + public interface Builder extends SchemaBuilder { + + TaskWithSchema createWithTask(); + + Builder asText(); + + Builder asInt(); + + Builder asDate(); + + Builder asUUID(); + + Builder asBoolean(); + + Builder asByte(); + + Builder asBlob(); + + Builder asDouble(); + + Builder asFloat(); + + Builder asLong(); + + Builder valueSingle(); + + Builder valueList(); + + Builder valueSet(); + + Builder calcMax(); + + Builder calcMin(); + + Builder calcSum(); + + Builder calcOld(); + + Builder calcSet(); + + Builder calcList(); + + Builder writeType(WriteType writeType); + + Builder cardinality(Cardinality cardinality); + + Builder dataType(DataType dataType); + + Builder aggregateType(AggregateType aggregateType); + + Builder userdata(String key, Object value); + + Builder userdata(Map userdata); + } + + @Override + public Map asMap() { + Map map = new HashMap<>(); + + if (this.dataType != null) { + map.put(P.DATA_TYPE, this.dataType.string()); + } + + if (this.cardinality != null) { + map.put(P.CARDINALITY, this.cardinality.string()); + } + + if (this.aggregateType != null) { + map.put(P.AGGREGATE_TYPE, this.aggregateType.string()); + } + + if (this.writeType != null) { + map.put(P.WRITE_TYPE, this.writeType.string()); + } + + return super.asMap(map); + } + + // change from HugeGraphSupplier HugeGraphSupplier by 2023/3/30 GraphPlatform-2062 core split merge 3.7.0 + @SuppressWarnings("unchecked") + public static PropertyKey fromMap(Map map, HugeGraphSupplier graph) { + Id id = IdGenerator.of((int) map.get(P.ID)); + String name = (String) map.get(P.NAME); + + PropertyKey propertyKey = new PropertyKey(graph, id, name); + for (Map.Entry entry : map.entrySet()) { + switch (entry.getKey()) { + case P.ID: + case P.NAME: + break; + case P.STATUS: + propertyKey.status(SchemaStatus.valueOf(((String) entry.getValue()).toUpperCase())); + break; + case P.USERDATA: + propertyKey.userdata((Map) entry.getValue()); + break; + case P.AGGREGATE_TYPE: + propertyKey.aggregateType(AggregateType.valueOf(((String) entry.getValue()).toUpperCase())); + break; + case P.WRITE_TYPE: + propertyKey.writeType(WriteType.valueOf(((String) entry.getValue()).toUpperCase())); + break; + case P.DATA_TYPE: + propertyKey.dataType(DataType.valueOf(((String) entry.getValue()).toUpperCase())); + break; + case P.CARDINALITY: + propertyKey.cardinality(Cardinality.valueOf(((String) entry.getValue()).toUpperCase())); + break; + default: + throw new AssertionError(String.format( + "Invalid key '%s' for property key", + entry.getKey())); + } + } + return propertyKey; + } + + public static final class P { + + public static final String ID = "id"; + public static final String NAME = "name"; + + public static final String STATUS = "status"; + public static final String USERDATA = "userdata"; + + public static final String DATA_TYPE = "data_type"; + public static final String CARDINALITY = "cardinality"; + + public static final String AGGREGATE_TYPE = "aggregate_type"; + public static final String WRITE_TYPE = "write_type"; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/SchemaElement.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/SchemaElement.java new file mode 100644 index 000000000..38946d81e --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/SchemaElement.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.schema; + +import java.util.Collections; +import java.util.Map; + +import org.apache.hugegraph.HugeGraphSupplier; + +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.type.Namifiable; +import org.apache.hugegraph.type.Typifiable; +import org.apache.hugegraph.type.define.SchemaStatus; +import org.apache.hugegraph.util.E; + + +import com.google.common.base.Objects; + +import org.apache.hugegraph.util.GraphUtils; + +public abstract class SchemaElement implements Namifiable, Typifiable, + Cloneable { + + public static final int MAX_PRIMITIVE_SYS_ID = 32; + public static final int NEXT_PRIMITIVE_SYS_ID = 8; + + // ABS of system schema id must be below MAX_PRIMITIVE_SYS_ID + protected static final int VL_IL_ID = -1; + protected static final int EL_IL_ID = -2; + protected static final int PKN_IL_ID = -3; + protected static final int VLN_IL_ID = -4; + protected static final int ELN_IL_ID = -5; + protected static final int ILN_IL_ID = -6; + protected static final int OLAP_VL_ID = -7; + + // OLAP_ID means all of vertex label ids + public static final Id OLAP_ID = IdGenerator.of(-7); + // OLAP means all of vertex label names + public static final String OLAP = "~olap"; + + public static final Id NONE_ID = IdGenerator.ZERO; + + public static final String UNDEF = "~undefined"; + + protected static final String SCHEMA_PREFIX = "graph.schema()."; + + protected final HugeGraphSupplier graph; + + private final Id id; + private final String name; + private final Userdata userdata; + private SchemaStatus status; + + public SchemaElement(final HugeGraphSupplier graph, Id id, String name) { + E.checkArgumentNotNull(id, "SchemaElement id can't be null"); + E.checkArgumentNotNull(name, "SchemaElement name can't be null"); + this.graph = graph; + this.id = id; + this.name = name; + this.userdata = new Userdata(); + this.status = SchemaStatus.CREATED; + } + + public HugeGraphSupplier graph() { + return this.graph; + } + + public Id id() { + return this.id; + } + + public long longId() { + return this.id.asLong(); + } + + @Override + public String name() { + return this.name; + } + + public Map userdata() { + return Collections.unmodifiableMap(this.userdata); + } + + public void userdata(String key, Object value) { + E.checkArgumentNotNull(key, "userdata key"); + E.checkArgumentNotNull(value, "userdata value"); + this.userdata.put(key, value); + } + + public void userdata(Userdata userdata) { + this.userdata.putAll(userdata); + } + + public void userdata(Map userdata) { + this.userdata.putAll(userdata); + } + + public void removeUserdata(String key) { + E.checkArgumentNotNull(key, "The userdata key can't be null"); + this.userdata.remove(key); + } + + public void removeUserdata(Userdata userdata) { + for (String key : userdata.keySet()) { + this.userdata.remove(key); + } + } + + public SchemaStatus status() { + return this.status; + } + + public void status(SchemaStatus status) { + this.status = status; + } + + public boolean system() { + return this.longId() < 0L; + } + + public boolean primitive() { + long id = this.longId(); + return -MAX_PRIMITIVE_SYS_ID <= id && id < 0L; + } + + public boolean hidden() { + return GraphUtils.isHidden(this.name()); + } + + public SchemaElement copy() { + try { + return (SchemaElement) super.clone(); + } catch (CloneNotSupportedException e) { + throw new HugeException("Failed to clone schema", e); + } + } + + public boolean hasSameContent(SchemaElement other) { + return Objects.equal(this.name(), other.name()) && + Objects.equal(this.userdata(), other.userdata()); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SchemaElement)) { + return false; + } + + SchemaElement other = (SchemaElement) obj; + return this.type() == other.type() && this.id.equals(other.id()); + } + + @Override + public int hashCode() { + return this.type().hashCode() ^ this.id.hashCode(); + } + + @Override + public String toString() { + return String.format("%s(id=%s)", this.name, this.id); + } + + public static int schemaId(Id id) { + long l = id.asLong(); + // Currently we limit the schema id to within 4 bytes + E.checkArgument(Integer.MIN_VALUE <= l && l <= Integer.MAX_VALUE, + "Schema id is out of bound: %s", l); + return (int) l; + } + + public static class TaskWithSchema { + + private SchemaElement schemaElement; + private Id task; + + public TaskWithSchema(SchemaElement schemaElement, Id task) { + E.checkNotNull(schemaElement, "schema element"); + this.schemaElement = schemaElement; + this.task = task; + } + + public void propertyKey(PropertyKey propertyKey) { + E.checkNotNull(propertyKey, "property key"); + this.schemaElement = propertyKey; + } + + public void indexLabel(IndexLabel indexLabel) { + E.checkNotNull(indexLabel, "index label"); + this.schemaElement = indexLabel; + } + + public PropertyKey propertyKey() { + E.checkState(this.schemaElement instanceof PropertyKey, + "Expect property key, but actual schema type is " + + "'%s'", this.schemaElement.getClass()); + return (PropertyKey) this.schemaElement; + } + + public IndexLabel indexLabel() { + E.checkState(this.schemaElement instanceof IndexLabel, + "Expect index label, but actual schema type is " + + "'%s'", this.schemaElement.getClass()); + return (IndexLabel) this.schemaElement; + } + + public SchemaElement schemaElement() { + return this.schemaElement; + } + + public Id task() { + return this.task; + } + } + + public abstract Map asMap(); + + public Map asMap(Map map) { + E.checkState(this.id != null, + "Property key id can't be null"); + E.checkState(this.name != null, + "Property key name can't be null"); + E.checkState(this.status != null, + "Property status can't be null"); + + map.put(P.ID, this.id); + map.put(P.NAME, this.name); + map.put(P.STATUS, this.status.string()); + map.put(P.USERDATA, this.userdata); + + return map; + } + + public static final class P { + + public static final String ID = "id"; + public static final String NAME = "name"; + + public static final String STATUS = "status"; + public static final String USERDATA = "userdata"; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/SchemaLabel.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/SchemaLabel.java new file mode 100644 index 000000000..74a059c5c --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/SchemaLabel.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.schema; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hugegraph.HugeGraphSupplier; +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.Indexfiable; +import org.apache.hugegraph.type.Propfiable; +import org.apache.hugegraph.util.E; + +import com.google.common.base.Objects; + +public abstract class SchemaLabel extends SchemaElement + implements Indexfiable, Propfiable { + + private final Set properties; + private final Set nullableKeys; + private final Set indexLabels; + private boolean enableLabelIndex; + private long ttl; + private Id ttlStartTime; + + public SchemaLabel(final HugeGraphSupplier graph, Id id, String name) { + super(graph, id, name); + this.properties = new HashSet<>(); + this.nullableKeys = new HashSet<>(); + this.indexLabels = new HashSet<>(); + this.enableLabelIndex = true; + this.ttl = 0L; + this.ttlStartTime = SchemaElement.NONE_ID; + } + + @Override + public Set properties() { + return Collections.unmodifiableSet(this.properties); + } + + public Set extendProperties() { + return this.properties(); + } + + public void properties(Set properties) { + this.properties.addAll(properties); + } + + public SchemaLabel properties(Id... ids) { + this.properties.addAll(Arrays.asList(ids)); + return this; + } + + public void property(Id id) { + this.properties.add(id); + } + + public Set nullableKeys() { + return Collections.unmodifiableSet(this.nullableKeys); + } + + public void nullableKey(Id id) { + this.nullableKeys.add(id); + } + + public void nullableKeys(Id... ids) { + this.nullableKeys.addAll(Arrays.asList(ids)); + } + + public void nullableKeys(Set nullableKeys) { + this.nullableKeys.addAll(nullableKeys); + } + + @Override + public Set indexLabels() { + return Collections.unmodifiableSet(this.indexLabels); + } + + public Set extendIndexLabels() { + return this.indexLabels(); + } + + public void indexLabel(Id id) { + this.indexLabels.add(id); + } + + public void indexLabels(Id... ids) { + this.indexLabels.addAll(Arrays.asList(ids)); + } + + public void addIndexLabel(Id id) { + this.indexLabels.add(id); + } + + public void addIndexLabels(Id... ids) { + this.indexLabels.addAll(Arrays.asList(ids)); + } + + public boolean existsIndexLabel() { + return !this.indexLabels().isEmpty(); + } + + public void removeIndexLabel(Id id) { + this.indexLabels.remove(id); + } + + public boolean enableLabelIndex() { + return this.enableLabelIndex; + } + + public void enableLabelIndex(boolean enable) { + this.enableLabelIndex = enable; + } + + public boolean undefined() { + return this.name() == UNDEF; + } + + public void ttl(long ttl) { + assert ttl >= 0L; + this.ttl = ttl; + } + + public long ttl() { + assert this.ttl >= 0L; + return this.ttl; + } + + public void ttlStartTime(Id id) { + this.ttlStartTime = id; + } + + public Id ttlStartTime() { + return this.ttlStartTime; + } + + public String ttlStartTimeName() { + return NONE_ID.equals(this.ttlStartTime) ? null : + this.graph.propertyKey(this.ttlStartTime).name(); + } + + public boolean hasSameContent(SchemaLabel other) { + return super.hasSameContent(other) && this.ttl == other.ttl && + this.enableLabelIndex == other.enableLabelIndex && + Objects.equal(this.graph.mapPkId2Name(this.properties), + other.graph.mapPkId2Name(other.properties)) && + Objects.equal(this.graph.mapPkId2Name(this.nullableKeys), + other.graph.mapPkId2Name(other.nullableKeys)) && + Objects.equal(this.graph.mapIlId2Name(this.indexLabels), + other.graph.mapIlId2Name(other.indexLabels)) && + Objects.equal(this.ttlStartTimeName(), other.ttlStartTimeName()); + } + + public static Id getLabelId(HugeGraphSupplier graph, HugeType type, Object label) { + E.checkNotNull(graph, "graph"); + E.checkNotNull(type, "type"); + E.checkNotNull(label, "label"); + if (label instanceof Number) { + return IdGenerator.of(((Number) label).longValue()); + } else if (label instanceof String) { + if (type.isVertex()) { + return graph.vertexLabel((String) label).id(); + } else if (type.isEdge()) { + return graph.edgeLabel((String) label).id(); + } else { + throw new HugeException( + "Not support query from '%s' with label '%s'", + type, label); + } + } else { + throw new HugeException( + "The label type must be number or string, but got '%s'", + label.getClass()); + } + } + + public static Id getVertexLabelId(HugeGraphSupplier graph, Object label) { + return SchemaLabel.getLabelId(graph, HugeType.VERTEX, label); + } + + public static Id getEdgeLabelId(HugeGraphSupplier graph, Object label) { + return SchemaLabel.getLabelId(graph, HugeType.EDGE, label); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/Userdata.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/Userdata.java new file mode 100644 index 000000000..d485e558b --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/Userdata.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.schema; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hugegraph.exception.NotAllowException; +import org.apache.hugegraph.type.define.Action; + +public class Userdata extends HashMap { + + private static final long serialVersionUID = -1235451175617197049L; + + public static final String CREATE_TIME = "~create_time"; + public static final String DEFAULT_VALUE = "~default_value"; + + public Userdata() { + } + + public Userdata(Map map) { + this.putAll(map); + } + + public static void check(Userdata userdata, Action action) { + if (userdata == null) { + return; + } + switch (action) { + case INSERT: + case APPEND: + for (Map.Entry e : userdata.entrySet()) { + if (e.getValue() == null) { + throw new NotAllowException( + "Not allowed to pass null userdata value " + + "when create or append schema"); + } + } + break; + case ELIMINATE: + case DELETE: + // pass + break; + default: + throw new AssertionError(String.format( + "Unknown schema action '%s'", action)); + } + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/VertexLabel.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/VertexLabel.java new file mode 100644 index 000000000..d6dbba29e --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/VertexLabel.java @@ -0,0 +1,414 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.schema; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.hugegraph.HugeGraphSupplier; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.schema.builder.SchemaBuilder; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.IdStrategy; +import org.apache.hugegraph.type.define.SchemaStatus; +import org.apache.hugegraph.util.GraphUtils; + +import com.google.common.base.Objects; +import com.google.common.collect.ImmutableSet; + +public class VertexLabel extends SchemaLabel { + + public static final VertexLabel NONE = new VertexLabel(null, NONE_ID, UNDEF); + public static final VertexLabel GENERAL = + new VertexLabel(null, NONE_ID, VertexLabel.GENERAL_VL); + + + // OLAP_VL_ID means all of vertex label ids + private static final Id OLAP_VL_ID = IdGenerator.of(SchemaLabel.OLAP_VL_ID); + // OLAP_VL_NAME means all of vertex label names + private static final String OLAP_VL_NAME = "*olap"; + // OLAP_VL means all of vertex labels + public static final VertexLabel OLAP_VL = new VertexLabel(null, OLAP_VL_ID, + OLAP_VL_NAME); + + public static final String GENERAL_VL = "~general_vl"; + + private IdStrategy idStrategy; + private List primaryKeys; + + public VertexLabel(final HugeGraphSupplier graph, Id id, String name) { + super(graph, id, name); + this.idStrategy = IdStrategy.DEFAULT; + this.primaryKeys = new ArrayList<>(); + } + + @Override + public HugeType type() { + return HugeType.VERTEX_LABEL; + } + + public boolean olap() { + return VertexLabel.OLAP_VL.id().equals(this.id()); + } + + public IdStrategy idStrategy() { + return this.idStrategy; + } + + public void idStrategy(IdStrategy idStrategy) { + this.idStrategy = idStrategy; + } + + public List primaryKeys() { + return Collections.unmodifiableList(this.primaryKeys); + } + + public void primaryKey(Id id) { + this.primaryKeys.add(id); + } + + public void primaryKeys(Id... ids) { + this.primaryKeys.addAll(Arrays.asList(ids)); + } + + + @Override + public Set extendProperties() { + Set properties = new HashSet<>(); + properties.addAll(this.properties()); + properties.addAll(this.primaryKeys); + + this.graph().propertyKeys().stream().forEach(pk -> { + if (pk.olap()) { + properties.add(pk.id()); + } + }); + + return Collections.unmodifiableSet(properties); + } + + @Override + public Set extendIndexLabels() { + Set indexes = new HashSet<>(); + + indexes.addAll(this.indexLabels()); + + for (IndexLabel il : this.graph.indexLabels()) { + if (il.olap()) { + indexes.add(il.id()); + } + } + + return ImmutableSet.copyOf(indexes); + } + + public boolean existsLinkLabel() { + return this.graph().existsLinkLabel(this.id()); + } + + public boolean hasSameContent(VertexLabel other) { + return super.hasSameContent(other) && + this.idStrategy == other.idStrategy && + Objects.equal(this.graph.mapPkId2Name(this.primaryKeys), + other.graph.mapPkId2Name(other.primaryKeys)); + } + + public static VertexLabel undefined(HugeGraphSupplier graph) { + return new VertexLabel(graph, NONE_ID, UNDEF); + } + + public static VertexLabel undefined(HugeGraphSupplier graph, Id id) { + return new VertexLabel(graph, id, UNDEF); + } + + public String convert2Groovy(boolean attachIdFlag) { + StringBuilder builder = new StringBuilder(SCHEMA_PREFIX); + // Name + if (!attachIdFlag) { + builder.append("vertexLabel").append("('") + .append(this.name()) + .append("')"); + } else { + builder.append("vertexLabel").append("(") + .append(longId()).append(", '") + .append(this.name()) + .append("')"); + } + + // Properties + Set properties = this.properties(); + if (!properties.isEmpty()) { + builder.append(".").append("properties("); + + int size = properties.size(); + for (Id id : this.properties()) { + PropertyKey pk = this.graph.propertyKey(id); + builder.append("'") + .append(pk.name()) + .append("'"); + if (--size > 0) { + builder.append(","); + } + } + builder.append(")"); + } + + // Id strategy + switch (this.idStrategy()) { + case PRIMARY_KEY: + builder.append(".primaryKeys("); + List pks = this.primaryKeys(); + int size = pks.size(); + for (Id id : pks) { + PropertyKey pk = this.graph.propertyKey(id); + builder.append("'") + .append(pk.name()) + .append("'"); + if (--size > 0) { + builder.append(","); + } + } + builder.append(")"); + break; + case CUSTOMIZE_STRING: + builder.append(".useCustomizeStringId()"); + break; + case CUSTOMIZE_NUMBER: + builder.append(".useCustomizeNumberId()"); + break; + case CUSTOMIZE_UUID: + builder.append(".useCustomizeUuidId()"); + break; + case AUTOMATIC: + builder.append(".useAutomaticId()"); + break; + default: + throw new AssertionError(String.format( + "Invalid id strategy '%s'", this.idStrategy())); + } + + // Nullable keys + properties = this.nullableKeys(); + if (!properties.isEmpty()) { + builder.append(".").append("nullableKeys("); + int size = properties.size(); + for (Id id : properties) { + PropertyKey pk = this.graph.propertyKey(id); + builder.append("'") + .append(pk.name()) + .append("'"); + if (--size > 0) { + builder.append(","); + } + } + builder.append(")"); + } + + // TTL + if (this.ttl() != 0) { + builder.append(".ttl(") + .append(this.ttl()) + .append(")"); + if (this.ttlStartTime() != null && + !this.ttlStartTime().equals(SchemaLabel.NONE_ID)) { + PropertyKey pk = this.graph.propertyKey(this.ttlStartTime()); + builder.append(".ttlStartTime('") + .append(pk.name()) + .append("')"); + } + } + + // Enable label index + if (this.enableLabelIndex()) { + builder.append(".enableLabelIndex(true)"); + } else { + builder.append(".enableLabelIndex(false)"); + } + + // User data + Map userdata = this.userdata(); + if (userdata.isEmpty()) { + return builder.toString(); + } + for (Map.Entry entry : userdata.entrySet()) { + if (GraphUtils.isHidden(entry.getKey())) { + continue; + } + builder.append(".userdata('") + .append(entry.getKey()) + .append("',") + .append(entry.getValue()) + .append(")"); + } + + builder.append(".ifNotExist().create();"); + return builder.toString(); + } + + public interface Builder extends SchemaBuilder { + + Id rebuildIndex(); + + Builder idStrategy(IdStrategy idStrategy); + + Builder useAutomaticId(); + + Builder usePrimaryKeyId(); + + Builder useCustomizeStringId(); + + Builder useCustomizeNumberId(); + + Builder useCustomizeUuidId(); + + Builder properties(String... properties); + + Builder primaryKeys(String... keys); + + Builder nullableKeys(String... keys); + + Builder ttl(long ttl); + + Builder ttlStartTime(String ttlStartTime); + + Builder enableLabelIndex(boolean enable); + + Builder userdata(String key, Object value); + + Builder userdata(Map userdata); + } + + @Override + public Map asMap() { + HashMap map = new HashMap(); + + map.put(P.PROPERTIES, this.properties()); + + map.put(P.NULLABLE_KEYS, this.nullableKeys()); + + map.put(P.INDEX_LABELS, this.indexLabels()); + + map.put(P.ENABLE_LABEL_INDEX, this.enableLabelIndex()); + + map.put(P.TTL, String.valueOf(this.ttl())); + + map.put(P.TT_START_TIME, this.ttlStartTime().asString()); + + map.put(P.ID_STRATEGY, this.idStrategy().string()); + + map.put(P.PRIMARY_KEYS, this.primaryKeys()); + + return super.asMap(map); + } + + public boolean generalVl(){ + return this.name() == GENERAL_VL; + } + + @SuppressWarnings("unchecked") + public static VertexLabel fromMap(Map map, HugeGraphSupplier graph) { + Id id = IdGenerator.of((int) map.get(VertexLabel.P.ID)); + String name = (String) map.get(VertexLabel.P.NAME); + + VertexLabel vertexLabel = new VertexLabel(graph, id, name); + for (Map.Entry entry : map.entrySet()) { + switch (entry.getKey()) { + case P.ID: + case P.NAME: + break; + case P.STATUS: + vertexLabel.status( + SchemaStatus.valueOf(((String) entry.getValue()).toUpperCase())); + break; + case P.USERDATA: + vertexLabel.userdata(new Userdata((Map) entry.getValue())); + break; + case P.PROPERTIES: + Set ids = ((List) entry.getValue()).stream().map( + IdGenerator::of).collect(Collectors.toSet()); + vertexLabel.properties(ids); + break; + case P.NULLABLE_KEYS: + ids = ((List) entry.getValue()).stream().map( + IdGenerator::of).collect(Collectors.toSet()); + vertexLabel.nullableKeys(ids); + break; + case P.INDEX_LABELS: + ids = ((List) entry.getValue()).stream().map( + IdGenerator::of).collect(Collectors.toSet()); + vertexLabel.addIndexLabels(ids.toArray(new Id[0])); + break; + case P.ENABLE_LABEL_INDEX: + boolean enableLabelIndex = (Boolean) entry.getValue(); + vertexLabel.enableLabelIndex(enableLabelIndex); + break; + case P.TTL: + long ttl = Long.parseLong((String) entry.getValue()); + vertexLabel.ttl(ttl); + break; + case P.TT_START_TIME: + long ttlStartTime = + Long.parseLong((String) entry.getValue()); + vertexLabel.ttlStartTime(IdGenerator.of(ttlStartTime)); + break; + case P.ID_STRATEGY: + IdStrategy idStrategy = + IdStrategy.valueOf(((String) entry.getValue()).toUpperCase()); + vertexLabel.idStrategy(idStrategy); + break; + case P.PRIMARY_KEYS: + ids = ((List) entry.getValue()).stream().map( + IdGenerator::of).collect(Collectors.toSet()); + vertexLabel.primaryKeys(ids.toArray(new Id[0])); + break; + default: + throw new AssertionError(String.format( + "Invalid key '%s' for vertex label", + entry.getKey())); + } + } + return vertexLabel; + } + + public static final class P { + + public static final String ID = "id"; + public static final String NAME = "name"; + + public static final String STATUS = "status"; + public static final String USERDATA = "userdata"; + + public static final String PROPERTIES = "properties"; + public static final String NULLABLE_KEYS = "nullableKeys"; + public static final String INDEX_LABELS = "indexLabels"; + + public static final String ENABLE_LABEL_INDEX = "enableLabelIndex"; + public static final String TTL = "ttl"; + public static final String TT_START_TIME = "ttlStartTime"; + public static final String ID_STRATEGY = "idStrategy"; + public static final String PRIMARY_KEYS = "primaryKeys"; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/builder/SchemaBuilder.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/builder/SchemaBuilder.java new file mode 100644 index 000000000..7b6550981 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/schema/builder/SchemaBuilder.java @@ -0,0 +1,42 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.schema.builder; + +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.schema.SchemaElement; + +public interface SchemaBuilder { + + public SchemaBuilder id(long id); + + public T build(); + + public T create(); + + public T append(); + + public T eliminate(); + + public Id remove(); + + public SchemaBuilder ifNotExist(); + + public SchemaBuilder checkExist(boolean checkExist); +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BinaryElementSerializer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BinaryElementSerializer.java new file mode 100644 index 000000000..fe58accdb --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BinaryElementSerializer.java @@ -0,0 +1,528 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.serializer; + +import com.google.common.primitives.Longs; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.NotImplementedException; +import org.apache.hugegraph.HugeGraphSupplier; +import org.apache.hugegraph.backend.BackendColumn; +import org.apache.hugegraph.backend.BinaryId; +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.id.EdgeId; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.schema.EdgeLabel; +import org.apache.hugegraph.schema.PropertyKey; +import org.apache.hugegraph.schema.SchemaElement; +import org.apache.hugegraph.schema.VertexLabel; +import org.apache.hugegraph.structure.*; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.Cardinality; +import org.apache.hugegraph.type.define.EdgeLabelType; +import org.apache.hugegraph.util.Bytes; +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.Log; +import org.apache.hugegraph.util.StringEncoding; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.Base64; +import java.util.Collection; +import java.util.Map; + +import static org.apache.hugegraph.schema.SchemaElement.UNDEF; + +public class BinaryElementSerializer { + static final BinaryElementSerializer INSTANCE = + new BinaryElementSerializer(); + static Logger log = Log.logger(BinaryElementSerializer.class); + + public static BinaryElementSerializer getInstance() { + return INSTANCE; + } + + /** + * Calculate owner ID of vertex/edge + * + * @param element + * @return + */ + public static Id ownerId(BaseElement element) { + if (element instanceof BaseVertex) { + return element.id(); + } else if (element instanceof BaseEdge) { + return ((EdgeId) element.id()).ownerVertexId(); + } else { + throw new IllegalArgumentException("Only support get ownerid" + + " of BaseVertex or BaseEdge"); + } + } + + /** + * Calculate owner ID of index + * + * @param index + * @return + */ + public static Id ownerId(Index index) { + Id elementId = index.elementId(); + + Id ownerId = null; + if (elementId instanceof EdgeId) { + // Edge ID + ownerId = ((EdgeId) elementId).ownerVertexId(); + } else { + // OLAP index + // Normal vertex index + // Normal secondary index + // Vertex/Edge LabelIndex + ownerId = elementId; + } + + return ownerId; + } + + + protected void parseProperty(HugeGraphSupplier graph, Id pkeyId, + BytesBuffer buffer, + BaseElement owner) { + PropertyKey pkey = graph != null ? + graph.propertyKey(pkeyId) : + new PropertyKey(graph, pkeyId, ""); + // Parse value + Object value = buffer.readProperty(pkey); + // Set properties of vertex/edge + if (pkey.cardinality() == Cardinality.SINGLE) { + owner.addProperty(pkey, value); + } else { + if (!(value instanceof Collection)) { + throw new HugeException( + "Invalid value of non-single property: %s", value); + } + owner.addProperty(pkey, value); + } + } + + public void parseProperties(HugeGraphSupplier graph, BytesBuffer buffer, + BaseElement owner) { + int size = buffer.readVInt(); + assert size >= 0; + for (int i = 0; i < size; i++) { + Id pkeyId = IdGenerator.of(buffer.readVInt()); + this.parseProperty(graph, pkeyId, buffer, owner); + } + } + + /** + * Deserialize vertex KV data into BaseVertex type vertex + * + * @param vertexCol Must be vertex data column + * @param vertex When vertex==null, used for operator sinking, deserialize col data into BaseVertex; + * When vertex!=null, add col information to vertex + */ + public BaseVertex parseVertex(HugeGraphSupplier graph, BackendColumn vertexCol, + BaseVertex vertex) { + if (vertex == null) { + BinaryId binaryId = + BytesBuffer.wrap(vertexCol.name).parseId(HugeType.VERTEX); + vertex = new BaseVertex(binaryId.origin(), VertexLabel.NONE); + } + + if (ArrayUtils.isEmpty(vertexCol.value)) { + // No need to parse vertex properties + return vertex; + } + BytesBuffer buffer = BytesBuffer.wrap(vertexCol.value); + Id labelId = buffer.readId(); + // Parse vertex label + if (graph != null) { + VertexLabel label = graph.vertexLabelOrNone(labelId); + vertex.correctVertexLabel(label); + } else { + VertexLabel label = new VertexLabel(null, labelId, UNDEF); + vertex.correctVertexLabel(label); + } + // Parse properties + this.parseProperties(graph, buffer, vertex); + + // Parse vertex expired time if needed + if (buffer.remaining() > 0 /*edge.hasTtl()*/) { + this.parseExpiredTime(buffer, vertex); + } + return vertex; + } + + /** + * Reverse sequence the vertex kv data into vertices of type BaseVertex + * + * @param olapVertexCol It must be a column of vertex data + * @param vertex When vertex==null, it is used for operator sinking to reverse sequence the col data into olapBaseVertex. + * vertex! When =null, add the col information to olapBaseVertex + */ + public BaseVertex parseVertexOlap(HugeGraphSupplier graph, + BackendColumn olapVertexCol, BaseVertex vertex) { + if (vertex == null) { + BytesBuffer idBuffer = BytesBuffer.wrap(olapVertexCol.name); + // read olap property id + idBuffer.readId(); + // read vertex id which olap property belongs to + Id vertexId = idBuffer.readId(); + vertex = new BaseVertex(vertexId, VertexLabel.NONE); + } + + BytesBuffer buffer = BytesBuffer.wrap(olapVertexCol.value); + Id pkeyId = IdGenerator.of(buffer.readVInt()); + this.parseProperty(graph, pkeyId, buffer, vertex); + return vertex; + } + + /** + * @param cols Deserializing a complete vertex may require multiple cols + * The first col represents the common vertex information in the g+v table, and each subsequent col represents the olap vertices stored in the olap table + */ + public BaseVertex parseVertexFromCols(HugeGraphSupplier graph, + BackendColumn... cols) { + assert cols.length > 0; + BaseVertex vertex = null; + for (int index = 0; index < cols.length; index++) { + BackendColumn col = cols[index]; + if (index == 0) { + vertex = this.parseVertex(graph, col, vertex); + } else { + this.parseVertexOlap(graph, col, vertex); + } + } + return vertex; + } + + public BaseEdge parseEdge(HugeGraphSupplier graph, BackendColumn edgeCol, + BaseVertex ownerVertex, + boolean withEdgeProperties) { + // owner-vertex + dir + edge-label.id() + subLabel.id() + + // + sort-values + other-vertex + + BytesBuffer buffer = BytesBuffer.wrap(edgeCol.name); + // Consume owner-vertex id + Id id = buffer.readId(); + if (ownerVertex == null) { + ownerVertex = new BaseVertex(id, VertexLabel.NONE); + } + + E.checkState(buffer.remaining() > 0, "Missing column type"); + + byte type = buffer.read(); + if (type == HugeType.EDGE_IN.code() || + type == HugeType.EDGE_OUT.code()) { + E.checkState(true, + "Invalid column(%s) with unknown type(%s): 0x%s", + id, type & 0xff, Bytes.toHex(edgeCol.name)); + } + + Id labelId = buffer.readId(); + Id subLabelId = buffer.readId(); + String sortValues = buffer.readStringWithEnding(); + Id otherVertexId = buffer.readId(); + boolean direction = EdgeId.isOutDirectionFromCode(type); + BaseEdge edge; + EdgeLabel edgeLabel; + if (graph == null) { /* when calculation sinking */ + edgeLabel = new EdgeLabel(null, subLabelId, UNDEF); + // If not equal here, need to add fatherId for correct operator sinking + if (subLabelId != labelId) { + edgeLabel.edgeLabelType(EdgeLabelType.SUB); + edgeLabel.fatherId(labelId); + } + + } else { + edgeLabel = graph.edgeLabelOrNone(subLabelId); + } + edge = BaseEdge.constructEdge(graph, ownerVertex, direction, + edgeLabel, sortValues, otherVertexId); + + if (!withEdgeProperties /*&& !edge.hasTtl()*/) { + // only skip properties for edge without ttl + // todo: save expiredTime before properties + return edge; + } + + if (ArrayUtils.isEmpty(edgeCol.value)) { + // There is no edge-properties here. + return edge; + } + + // Parse edge-id + edge-properties + buffer = BytesBuffer.wrap(edgeCol.value); + + // Parse edge properties + this.parseProperties(graph, buffer, edge); + + /* Skip TTL parsing process first + * Can't determine if edge has TTL through edge, need to judge by bytebuffer length */ +// // Parse edge expired time if needed + if (buffer.remaining() > 0 /*edge.hasTtl()*/) { + this.parseExpiredTime(buffer, edge); + } + return edge; + } + + /** + * @param graph When parsing index, graph cannot be null + * @param index When null, used for operator sinking, store can restore index based on one col data + */ + public Index parseIndex(HugeGraphSupplier graph, BackendColumn indexCol, + Index index) { + HugeType indexType = parseIndexType(indexCol); + + BytesBuffer buffer = BytesBuffer.wrap(indexCol.name); + BinaryId indexId = buffer.readIndexId(indexType); + Id elemId = buffer.readId(); + + if (index == null) { + index = Index.parseIndexId(graph, indexType, indexId.asBytes()); + } + + long expiredTime = 0L; + + if (indexCol.value.length > 0) { + + // Get delimiter address + int delimiterIndex = + Bytes.indexOf(indexCol.value, BytesBuffer.STRING_ENDING_BYTE); + + if (delimiterIndex >= 0) { + // Delimiter is in the data, need to parse from data + // 1. field value real content + byte[] fieldValueBytes = + Arrays.copyOfRange(indexCol.value, 0, delimiterIndex); + if (fieldValueBytes.length > 0) { + index.fieldValues(StringEncoding.decode(fieldValueBytes)); + } + + // 2. Expiration time + byte[] expiredTimeBytes = + Arrays.copyOfRange(indexCol.value, delimiterIndex + 1, + indexCol.value.length); + + if (expiredTimeBytes.length > 0) { + byte[] rawBytes = + Base64.getDecoder().decode(expiredTimeBytes); + if (rawBytes.length >= Longs.BYTES) { + expiredTime = Longs.fromByteArray(rawBytes); + } + } + } else { + // Only field value data + index.fieldValues(StringEncoding.decode(indexCol.value)); + } + } + + index.elementIds(elemId, expiredTime); + return index; + } + + public BackendColumn parseIndex(BackendColumn indexCol) { + // Self-parsing index + throw new NotImplementedException( + "BinaryElementSerializer.parseIndex"); + } + + public BackendColumn writeVertex(BaseVertex vertex) { + if (vertex.olap()) { + return this.writeOlapVertex(vertex); + } + + BytesBuffer bufferName = BytesBuffer.allocate(vertex.id().length()); + bufferName.writeId(vertex.id()); + + int propsCount = vertex.getProperties().size(); + BytesBuffer buffer = BytesBuffer.allocate(8 + 16 * propsCount); + + // Write vertex label + buffer.writeId(vertex.schemaLabel().id()); + + // Write all properties of the vertex + this.formatProperties(vertex.getProperties().values(), buffer); + + // Write vertex expired time if needed + if (vertex.hasTtl()) { + this.formatExpiredTime(vertex.expiredTime(), buffer); + } + + return BackendColumn.of(bufferName.bytes(), buffer.bytes()); + } + + public BackendColumn writeOlapVertex(BaseVertex vertex) { + BytesBuffer buffer = BytesBuffer.allocate(8 + 16); + + BaseProperty baseProperty = vertex.getProperties().values() + .iterator().next(); + PropertyKey propertyKey = baseProperty.propertyKey(); + buffer.writeVInt(SchemaElement.schemaId(propertyKey.id())); + buffer.writeProperty(propertyKey.cardinality(), propertyKey.dataType(), + baseProperty.value()); + + // OLAP table merge, key is {property_key_id}{vertex_id} + BytesBuffer bufferName = + BytesBuffer.allocate(1 + propertyKey.id().length() + 1 + + vertex.id().length()); + bufferName.writeId(propertyKey.id()); + bufferName.writeId(vertex.id()).bytes(); + + return BackendColumn.of(bufferName.bytes(), buffer.bytes()); + } + + public BackendColumn writeEdge(BaseEdge edge) { + byte[] name = this.formatEdgeName(edge); + byte[] value = this.formatEdgeValue(edge); + return BackendColumn.of(name, value); + } + + /** + * Convert an index data to a BackendColumn + */ + public BackendColumn writeIndex(Index index) { + return BackendColumn.of(formatIndexName(index), + formatIndexValue(index)); + } + + private byte[] formatIndexName(Index index) { + BytesBuffer buffer; + Id elemId = index.elementId(); + Id indexId = index.id(); + HugeType type = index.type(); + int idLen = 1 + elemId.length() + 1 + indexId.length(); + buffer = BytesBuffer.allocate(idLen); + // Write index-id + buffer.writeIndexId(indexId, type); + // Write element-id + buffer.writeId(elemId); + + return buffer.bytes(); + } + + /** + * @param index value + * @return format + * | empty(field-value) | 0x00 | base64(expiredtime) | + */ + private byte[] formatIndexValue(Index index) { + if (index.hasTtl()) { + BytesBuffer valueBuffer = BytesBuffer.allocate(14); + + valueBuffer.write(BytesBuffer.STRING_ENDING_BYTE); + byte[] ttlBytes = + Base64.getEncoder().encode(Longs.toByteArray(index.expiredTime())); + valueBuffer.write(ttlBytes); + + return valueBuffer.bytes(); + } + + return null; + } + + public BackendColumn mergeCols(BackendColumn vertexCol, BackendColumn... olapVertexCols) { + if (olapVertexCols.length == 0) { + return vertexCol; + } + BytesBuffer mergedBuffer = BytesBuffer.allocate( + vertexCol.value.length + olapVertexCols.length * 16); + + BytesBuffer buffer = BytesBuffer.wrap(vertexCol.value); + Id vl = buffer.readId(); + int size = buffer.readVInt(); + + mergedBuffer.writeId(vl); + mergedBuffer.writeVInt(size + olapVertexCols.length); + // Prioritize writing vertexCol properties, because vertexCol may contain TTL + for (BackendColumn olapVertexCol : olapVertexCols) { + mergedBuffer.write(olapVertexCol.value); + } + mergedBuffer.write(buffer.remainingBytes()); + + return BackendColumn.of(vertexCol.name, mergedBuffer.bytes()); + } + + public BaseElement index2Element(HugeGraphSupplier graph, + BackendColumn indexCol) { + throw new NotImplementedException( + "BinaryElementSerializer.index2Element"); + } + + public byte[] formatEdgeName(BaseEdge edge) { + // owner-vertex + dir + edge-label + sort-values + other-vertex + return BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID) + .writeEdgeId(edge.id()).bytes(); + } + + protected byte[] formatEdgeValue(BaseEdge edge) { + Map> properties = edge.getProperties(); + int propsCount = properties.size(); + BytesBuffer buffer = BytesBuffer.allocate(4 + 16 * propsCount); + + // Write edge properties + this.formatProperties(properties.values(), buffer); + + // Write edge expired time if needed + if (edge.hasTtl()) { + this.formatExpiredTime(edge.expiredTime(), buffer); + } + + return buffer.bytes(); + } + + public void formatProperties(Collection> props, + BytesBuffer buffer) { + // Write properties size + buffer.writeVInt(props.size()); + + // Write properties data + for (BaseProperty property : props) { + PropertyKey pkey = property.propertyKey(); + buffer.writeVInt(SchemaElement.schemaId(pkey.id())); + buffer.writeProperty(pkey.cardinality(), pkey.dataType(), + property.value()); + } + } + + public void formatExpiredTime(long expiredTime, BytesBuffer buffer) { + buffer.writeVLong(expiredTime); + } + + protected void parseExpiredTime(BytesBuffer buffer, BaseElement element) { + element.expiredTime(buffer.readVLong()); + } + + private HugeType parseIndexType(BackendColumn col) { + /** + * Reference formatIndexName method + * For range type index, col.name first byte writes type.code (1 byte) + * Other type indexes will write type.name in first two bytes (2 byte) + */ + byte first = col.name[0]; + byte second = col.name[1]; + if (first < 0) { + return HugeType.fromCode(first); + } + assert second >= 0; + String type = new String(new byte[]{first, second}); + return HugeType.fromString(type); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BytesBuffer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BytesBuffer.java new file mode 100644 index 000000000..30e07a70a --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/BytesBuffer.java @@ -0,0 +1,1012 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.serializer; + +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.UUID; + +import org.apache.hugegraph.backend.BinaryId; +import org.apache.hugegraph.id.EdgeId; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.Id.IdType; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.schema.PropertyKey; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.Cardinality; +import org.apache.hugegraph.type.define.DataType; +import org.apache.hugegraph.type.define.SerialEnum; +import org.apache.hugegraph.util.Blob; +import org.apache.hugegraph.util.Bytes; +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.StringEncoding; + +/** + * class BytesBuffer is a util for read/write binary + */ +public class BytesBuffer extends OutputStream { + + public static final int BYTE_LEN = Byte.BYTES; + public static final int SHORT_LEN = Short.BYTES; + public static final int INT_LEN = Integer.BYTES; + public static final int LONG_LEN = Long.BYTES; + public static final int CHAR_LEN = Character.BYTES; + public static final int FLOAT_LEN = Float.BYTES; + public static final int DOUBLE_LEN = Double.BYTES; + public static final int BLOB_LEN = 4; + + public static final int UINT8_MAX = ((byte) -1) & 0xff; + public static final int UINT16_MAX = ((short) -1) & 0xffff; + public static final long UINT32_MAX = (-1) & 0xffffffffL; + public static final long WRITE_BYTES_MAX_LENGTH = 10 * Bytes.MB; + + // NOTE: +1 to let code 0 represent length 1 + public static final int ID_LEN_MAX = 0x7fff + 1; + public static final int BIG_ID_LEN_MAX = 0xfffff + 1; + + public static final byte STRING_ENDING_BYTE = (byte) 0x00; + public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff; + public static final int STRING_LEN_MAX = UINT16_MAX; + public static final long BLOB_LEN_MAX = 1 * Bytes.GB; + + // The value must be in range [8, ID_LEN_MAX] + public static final int INDEX_HASH_ID_THRESHOLD = 32; + + public static final int DEFAULT_CAPACITY = 64; + public static final int MAX_BUFFER_CAPACITY = 128 * 1024 * 1024; // 128M + + public static final int BUF_EDGE_ID = 128; + public static final int BUF_PROPERTY = 64; + + private ByteBuffer buffer; + private final boolean resize; + + public BytesBuffer() { + this(DEFAULT_CAPACITY); + } + + public BytesBuffer(int capacity) { + E.checkArgument(capacity <= MAX_BUFFER_CAPACITY, + "Capacity exceeds max buffer capacity: %s", + MAX_BUFFER_CAPACITY); + this.buffer = ByteBuffer.allocate(capacity); + this.resize = true; + } + + public BytesBuffer(ByteBuffer buffer) { + E.checkNotNull(buffer, "buffer"); + this.buffer = buffer; + this.resize = false; + } + + public static BytesBuffer allocate(int capacity) { + return new BytesBuffer(capacity); + } + + public static BytesBuffer wrap(ByteBuffer buffer) { + return new BytesBuffer(buffer); + } + + public static BytesBuffer wrap(byte[] array) { + return new BytesBuffer(ByteBuffer.wrap(array)); + } + + public static BytesBuffer wrap(byte[] array, int offset, int length) { + return new BytesBuffer(ByteBuffer.wrap(array, offset, length)); + } + + public ByteBuffer asByteBuffer() { + return this.buffer; + } + + public BytesBuffer forReadWritten() { + this.buffer.flip(); + return this; + } + + public BytesBuffer forReadAll() { + this.buffer.position(this.buffer.limit()); + return this; + } + + public byte[] array() { + return this.buffer.array(); + } + + public byte[] bytes() { + byte[] bytes = this.buffer.array(); + int position = this.buffer.position(); + if (position == bytes.length) { + return bytes; + } else { + return Arrays.copyOf(bytes, position); + } + } + + public int position() { + return this.buffer.position(); + } + + public BytesBuffer copyFrom(BytesBuffer other) { + this.write(other.bytes()); + return this; + } + + public int remaining() { + return this.buffer.remaining(); + } + + private void require(int size) { + // Does need to resize? + if (this.buffer.limit() - this.buffer.position() >= size) { + return; + } + // Can't resize for wrapped buffer since will change the origin ref + E.checkState(this.resize, "Can't resize for wrapped buffer"); + + // Extra capacity as buffer + int newcapacity = size + this.buffer.limit() + DEFAULT_CAPACITY; + E.checkArgument(newcapacity <= MAX_BUFFER_CAPACITY, + "Capacity exceeds max buffer capacity: %s", + MAX_BUFFER_CAPACITY); + ByteBuffer newBuffer = ByteBuffer.allocate(newcapacity); + this.buffer.flip(); + newBuffer.put(this.buffer); + this.buffer = newBuffer; + } + + public BytesBuffer write(byte val) { + require(BYTE_LEN); + this.buffer.put(val); + return this; + } + + @Override + public void write(int val) { + assert val <= UINT8_MAX; + require(BYTE_LEN); + this.buffer.put((byte) val); + } + + @Override + public void write(byte[] val) { + require(BYTE_LEN * val.length); + this.buffer.put(val); + } + + @Override + public void write(byte[] val, int offset, int length) { + require(BYTE_LEN * length); + this.buffer.put(val, offset, length); + } + + public BytesBuffer writeBoolean(boolean val) { + this.write(val ? 1 : 0); + return this; + } + + public BytesBuffer writeChar(char val) { + require(CHAR_LEN); + this.buffer.putChar(val); + return this; + } + + public BytesBuffer writeShort(short val) { + require(SHORT_LEN); + this.buffer.putShort(val); + return this; + } + + public BytesBuffer writeInt(int val) { + require(INT_LEN); + this.buffer.putInt(val); + return this; + } + + public BytesBuffer writeLong(long val) { + require(LONG_LEN); + this.buffer.putLong(val); + return this; + } + + public BytesBuffer writeFloat(float val) { + require(FLOAT_LEN); + this.buffer.putFloat(val); + return this; + } + + public BytesBuffer writeDouble(double val) { + require(DOUBLE_LEN); + this.buffer.putDouble(val); + return this; + } + + public byte peek() { + return this.buffer.get(this.buffer.position()); + } + + public byte peekLast() { + return this.buffer.get(this.buffer.capacity() - 1); + } + + public byte read() { + return this.buffer.get(); + } + + public byte[] read(int length) { + byte[] bytes = new byte[length]; + this.buffer.get(bytes); + return bytes; + } + + public byte[] readToEnd() { + byte[] bytes = new byte[this.remaining()]; + this.buffer.get(bytes); + return bytes; + } + + public boolean readBoolean() { + return this.buffer.get() == 0 ? false : true; + } + + public char readChar() { + return this.buffer.getChar(); + } + + public short readShort() { + return this.buffer.getShort(); + } + + public int readInt() { + return this.buffer.getInt(); + } + + public long readLong() { + return this.buffer.getLong(); + } + + public float readFloat() { + return this.buffer.getFloat(); + } + + public double readDouble() { + return this.buffer.getDouble(); + } + + public BytesBuffer writeBytes(byte[] bytes) { + // Original limit as above, consider this limit may be due to performance considerations when multiple storage backends are used. + // The above limit will cause errors when writing value to property exceeds the limit. So adjust size to 5M + E.checkArgument(bytes.length <= WRITE_BYTES_MAX_LENGTH, + "The max length of bytes is %s, but got %s", + WRITE_BYTES_MAX_LENGTH, bytes.length); + require(SHORT_LEN + bytes.length); + this.writeVInt(bytes.length); + this.write(bytes); + return this; + } + + public byte[] readBytes() { + int length = this.readVInt(); + assert length >= 0; + byte[] bytes = this.read(length); + return bytes; + } + + public BytesBuffer writeBigBytes(byte[] bytes) { + E.checkArgument(bytes.length <= BLOB_LEN_MAX, + "The max length of bytes is %s, but got %s", + BLOB_LEN_MAX, bytes.length); + require(BLOB_LEN + bytes.length); + this.writeVInt(bytes.length); + this.write(bytes); + return this; + } + + public byte[] readBigBytes() { + int length = this.readVInt(); + assert length >= 0; + byte[] bytes = this.read(length); + return bytes; + } + + public BytesBuffer writeStringRaw(String val) { + this.write(StringEncoding.encode(val)); + return this; + } + + public BytesBuffer writeString(String val) { + byte[] bytes = StringEncoding.encode(val); + this.writeBytes(bytes); + return this; + } + + public String readString() { + return StringEncoding.decode(this.readBytes()); + } + + public BytesBuffer writeStringWithEnding(String value) { + if (!value.isEmpty()) { + byte[] bytes = StringEncoding.encode(value); + /* + * assert '0x00'/'0xFF' not exist in string index id + * NOTE: + * 0x00 is NULL in UTF8(or ASCII) bytes + * 0xFF is not a valid byte in UTF8 bytes + */ + assert !Bytes.contains(bytes, STRING_ENDING_BYTE_FF) : + "Invalid UTF8 bytes: " + value; + if (Bytes.contains(bytes, STRING_ENDING_BYTE)) { + E.checkArgument(false, + "Can't contains byte '0x00' in string: '%s'", + value); + } + this.write(bytes); + } + /* + * Choose 0x00 as ending symbol (see #1057) + * The following is out of date: + * A reasonable ending symbol should be 0x00(to ensure order), but + * considering that some backends like PG do not support 0x00 string, + * so choose 0xFF currently. + */ + this.write(STRING_ENDING_BYTE); + return this; + } + + public String readStringWithEnding() { + return StringEncoding.decode(this.readBytesWithEnding()); + } + public String skipBytesWithEnding(){ + boolean foundEnding = false; + while (this.remaining() > 0) { + byte current = this.read(); + if (current == STRING_ENDING_BYTE) { + foundEnding = true; + break; + } + } + return ""; + } + + public BytesBuffer writeStringToRemaining(String value) { + byte[] bytes = StringEncoding.encode(value); + this.write(bytes); + return this; + } + + public String readStringFromRemaining() { + byte[] bytes = new byte[this.buffer.remaining()]; + this.buffer.get(bytes); + return StringEncoding.decode(bytes); + } + + public BytesBuffer writeUInt8(int val) { + assert val <= UINT8_MAX; + this.write(val); + return this; + } + + public int readUInt8() { + return this.read() & 0x000000ff; + } + + public BytesBuffer writeUInt16(int val) { + assert val <= UINT16_MAX; + this.writeShort((short) val); + return this; + } + + public int readUInt16() { + return this.readShort() & 0x0000ffff; + } + + public BytesBuffer writeUInt32(long val) { + assert val <= UINT32_MAX; + this.writeInt((int) val); + return this; + } + + public long readUInt32() { + return this.readInt() & 0xffffffffL; + } + + public BytesBuffer writeVInt(int value) { + // NOTE: negative numbers are not compressed + if (value > 0x0fffffff || value < 0) { + this.write(0x80 | ((value >>> 28) & 0x7f)); + } + if (value > 0x1fffff || value < 0) { + this.write(0x80 | ((value >>> 21) & 0x7f)); + } + if (value > 0x3fff || value < 0) { + this.write(0x80 | ((value >>> 14) & 0x7f)); + } + if (value > 0x7f || value < 0) { + this.write(0x80 | ((value >>> 7) & 0x7f)); + } + this.write(value & 0x7f); + + return this; + } + + public int readVInt() { + byte leading = this.read(); + int value = leading & 0x7f; + if (leading >= 0) { + assert (leading & 0x80) == 0; + return value; + } + + int i = 1; + for (; i < 5; i++) { + byte b = this.read(); + if (b >= 0) { + value = b | (value << 7); + break; + } else { + value = (b & 0x7f) | (value << 7); + } + } + + return value; + } + + public BytesBuffer writeVLong(long value) { + if (value < 0) { + this.write((byte) 0x81); + } + if (value > 0xffffffffffffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 56) & 0x7f)); + } + if (value > 0x1ffffffffffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 49) & 0x7f)); + } + if (value > 0x3ffffffffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 42) & 0x7f)); + } + if (value > 0x7ffffffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 35) & 0x7f)); + } + if (value > 0xfffffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 28) & 0x7f)); + } + if (value > 0x1fffffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 21) & 0x7f)); + } + if (value > 0x3fffL || value < 0L) { + this.write(0x80 | ((int) (value >>> 14) & 0x7f)); + } + if (value > 0x7fL || value < 0L) { + this.write(0x80 | ((int) (value >>> 7) & 0x7f)); + } + this.write((int) value & 0x7f); + + return this; + } + + public long readVLong() { + byte leading = this.read(); + E.checkArgument(leading != 0x80, + "Unexpected varlong with leading byte '0x%s'", + Bytes.toHex(leading)); + long value = leading & 0x7fL; + if (leading >= 0) { + assert (leading & 0x80) == 0; + return value; + } + + int i = 1; + for (; i < 10; i++) { + byte b = this.read(); + if (b >= 0) { + value = b | (value << 7); + break; + } else { + value = (b & 0x7f) | (value << 7); + } + } + + E.checkArgument(i < 10, + "Unexpected varlong %s with too many bytes(%s)", + value, i + 1); + E.checkArgument(i < 9 || (leading & 0x7e) == 0, + "Unexpected varlong %s with leading byte '0x%s'", + value, Bytes.toHex(leading)); + return value; + } + + public T newValue(Cardinality cardinality) { + switch (cardinality) { + case SET: + return (T) new LinkedHashSet<>(); + case LIST: + return (T) new ArrayList<>(); + default: + // pass + break; + } + return null; + } + + private byte getCardinalityAndType(int cardinality, int type){ + return (byte) ((cardinality << 6) | type); + } + + public static byte getCardinality(int value){ + return (byte) ((value & 0xc0) >> 6); + } + + public static byte getType(int value){ + return (byte) (value & 0x3f); + } + + public BytesBuffer writeProperty(PropertyKey pkey, Object value) { + return writeProperty(pkey.cardinality(), pkey.dataType(), value); + } + + public BytesBuffer writeProperty(Cardinality cardinality, DataType dataType, Object value) { + this.write(getCardinalityAndType(cardinality.code(),dataType.code())); + if (cardinality == Cardinality.SINGLE) { + this.writeProperty(dataType, value); + return this; + } + assert cardinality == Cardinality.LIST || + cardinality == Cardinality.SET; + Collection values = (Collection) value; + this.writeVInt(values.size()); + for (Object o : values) { + this.writeProperty(dataType, o); + } + return this; + } + + public Object readProperty(PropertyKey propertyKey) { + byte cardinalityAndType = this.read(); + Cardinality cardinality; + DataType type; + cardinality = SerialEnum.fromCode(Cardinality.class, + getCardinality(cardinalityAndType)); + + type = SerialEnum.fromCode(DataType.class, getType(cardinalityAndType)); + propertyKey.cardinality(cardinality); + propertyKey.dataType(type); + if (cardinality == Cardinality.SINGLE) { + Object value = this.readProperty(type); + return value; + } + Collection values = this.newValue(cardinality); + assert cardinality == Cardinality.LIST || + cardinality == Cardinality.SET; + int size = this.readVInt(); + for (int i = 0; i < size; i++) { + values.add(this.readProperty(type)); + } + return values; + } + + public void writeProperty(DataType dataType, Object value) { + switch (dataType) { + case BOOLEAN: + this.writeVInt(((Boolean) value) ? 1 : 0); + break; + case BYTE: + this.writeVInt((Byte) value); + break; + case INT: + this.writeVInt((Integer) value); + break; + case FLOAT: + this.writeFloat((Float) value); + break; + case LONG: + this.writeVLong((Long) value); + break; + case DATE: + this.writeVLong(((Date) value).getTime()); + break; + case DOUBLE: + this.writeDouble((Double) value); + break; + case TEXT: + this.writeString((String) value); + break; + case BLOB: + byte[] bytes = value instanceof byte[] ? + (byte[]) value : ((Blob) value).bytes(); + this.writeBigBytes(bytes); + break; + case UUID: + UUID uuid = (UUID) value; + // Generally writeVLong(uuid) can't save space + this.writeLong(uuid.getMostSignificantBits()); + this.writeLong(uuid.getLeastSignificantBits()); + break; + default: + throw new IllegalArgumentException("Unsupported data type " + dataType); + } + } + + public Object readProperty(DataType dataType) { + switch (dataType) { + case BOOLEAN: + return this.readVInt() == 1; + case BYTE: + return (byte) this.readVInt(); + case INT: + return this.readVInt(); + case FLOAT: + return this.readFloat(); + case LONG: + return this.readVLong(); + case DATE: + return new Date(this.readVLong()); + case DOUBLE: + return this.readDouble(); + case TEXT: + return this.readString(); + case BLOB: + return Blob.wrap(this.readBigBytes()); + case UUID: + return new UUID(this.readLong(), this.readLong()); + default: + throw new IllegalArgumentException("Unsupported data type " + dataType); + } + } + + public BytesBuffer writeId(Id id) { + return this.writeId(id, false); + } + + public BytesBuffer writeId(Id id, boolean big) { + switch (id.type()) { + case LONG: + // Number Id + long value = id.asLong(); + this.writeNumber(value); + break; + case UUID: + // UUID Id + byte[] bytes = id.asBytes(); + assert bytes.length == Id.UUID_LENGTH; + this.writeUInt8(0x7f); // 0b01111111 means UUID + this.write(bytes); + break; + case EDGE: + // Edge Id + this.writeUInt8(0x7e); // 0b01111110 means EdgeId + this.writeEdgeId(id); + break; + default: + // String Id + bytes = id.asBytes(); + int len = bytes.length; + E.checkArgument(len > 0, "Can't write empty id"); + E.checkArgument(len <= 16384, + "Big id max length is %s, but got %s {%s}", + 16384, len, id); + len -= 1; + if (len <= 63) { + this.writeUInt8(len | 0x80); + } else { + int high = len >> 8; + int low = len & 0xff; + this.writeUInt8(high | 0xc0); + this.writeUInt8(low); + } + + this.write(bytes); + break; + } + return this; + } + + public Id readId() { + return this.readId(false); + } + + public Id readId(boolean big) { + byte b = this.read(); + boolean number = (b & 0x80) == 0; + if (number) { + if (b == 0x7f) { + // UUID Id + return IdGenerator.of(this.read(Id.UUID_LENGTH), IdType.UUID); + } else if (b == 0x7e) { + // Edge Id + return this.readEdgeId(); + } else { + // Number Id + return IdGenerator.of(this.readNumber(b)); + } + } else { + // String Id + int len = b & 0x3f; + if ((b & 0x40) != 0) { + int high = len << 8; + int low = this.readUInt8(); + len = high + low; + } + len += 1; + byte[] id = this.read(len); + return IdGenerator.of(id, IdType.STRING); + } + } + + public BytesBuffer writeEdgeId(Id id) { + EdgeId edge = (EdgeId) id; + this.writeId(edge.ownerVertexId()); + this.write(edge.directionCode()); + this.writeId(edge.edgeLabelId()); + this.writeId(edge.subLabelId()); + this.writeStringWithEnding(edge.sortValues()); + this.writeId(edge.otherVertexId()); + return this; + } + + public Id readEdgeId() { + return new EdgeId(this.readId(), EdgeId.directionFromCode(this.read()), + this.readId(), this.readId(), + this.readStringWithEnding(), this.readId()); + } + + public Id readEdgeIdSkipSortValues() { + return new EdgeId(this.readId(), EdgeId.directionFromCode(this.read()), + this.readId(), this.readId(), + this.skipBytesWithEnding(), + this.readId()); + } + + + public BytesBuffer writeIndexId(Id id, HugeType type) { + return this.writeIndexId(id, type, true); + } + + public BytesBuffer writeIndexId(Id id, HugeType type, boolean withEnding) { + byte[] bytes = id.asBytes(); + int len = bytes.length; + E.checkArgument(len > 0, "Can't write empty id"); + + this.write(bytes); + if (type.isStringIndex()) { + if (Bytes.contains(bytes, STRING_ENDING_BYTE)) { + // Not allow STRING_ENDING_BYTE exist in string index id + E.checkArgument(false, + "The %s type index id can't contains " + + "byte '0x%s', but got: 0x%s", type, + Bytes.toHex(STRING_ENDING_BYTE), + Bytes.toHex(bytes)); + } + if (withEnding) { + this.writeStringWithEnding(""); + } + } + return this; + } + + public BinaryId readIndexId(HugeType type) { + byte[] id; + if (type.isRange4Index()) { + // HugeCodeType 1 bytes + IndexLabel 4 bytes + fieldValue 4 bytes + id = this.read(9); + } else if (type.isRange8Index()) { + // HugeCodeType 1 bytes + IndexLabel 4 bytes + fieldValue 8 bytes + id = this.read(13); + } else { + assert type.isStringIndex(); + id = this.readBytesWithEnding(); + } + return new BinaryId(id, IdGenerator.of(id, IdType.STRING)); + } + + public BinaryId asId() { + return new BinaryId(this.bytes(), null); + } + + public BinaryId parseId(HugeType type) { + if (type.isIndex()) { + return this.readIndexId(type); + } + // Parse id from bytes + int start = this.buffer.position(); + /* + * Since edge id in edges table doesn't prefix with leading 0x7e, + * so readId() will return the source vertex id instead of edge id, + * can't call: type.isEdge() ? this.readEdgeId() : this.readId(); + */ + Id id = this.readId(); + int end = this.buffer.position(); + int len = end - start; + byte[] bytes = new byte[len]; + System.arraycopy(this.array(), start, bytes, 0, len); + return new BinaryId(bytes, id); + } + + /** + * Parse OLAP id + * @param type + * @param isOlap + * @return + */ + public BinaryId parseOlapId(HugeType type, boolean isOlap) { + if (type.isIndex()) { + return this.readIndexId(type); + } + // Parse id from bytes + int start = this.buffer.position(); + /** + * OLAP + * {PropertyKey}{VertexId} + */ + if (isOlap) { + // First read OLAP property id + Id pkId = this.readId(); + } + Id id = this.readId(); + int end = this.buffer.position(); + int len = end - start; + byte[] bytes = new byte[len]; + System.arraycopy(this.array(), start, bytes, 0, len); + return new BinaryId(bytes, id); + } + + private void writeNumber(long val) { + /* + * 8 kinds of number, 2 ~ 9 bytes number: + * 0b 0kkksxxx X... + * 0(1 bit) + kind(3 bits) + signed(1 bit) + number(n bits) + * + * 2 byte : 0b 0000 1xxx X(8 bits) [0, 2047] + * 0b 0000 0xxx X(8 bits) [-2048, -1] + * 3 bytes: 0b 0001 1xxx X X [0, 524287] + * 0b 0001 0xxx X X [-524288, -1] + * 4 bytes: 0b 0010 1xxx X X X [0, 134217727] + * 0b 0010 0xxx X X X [-134217728, -1] + * 5 bytes: 0b 0011 1xxx X X X X [0, 2^35 - 1] + * 0b 0011 0xxx X X X X [-2^35, -1] + * 6 bytes: 0b 0100 1xxx X X X X X [0, 2^43 - 1] + * 0b 0100 0xxx X X X X X [-2^43, -1] + * 7 bytes: 0b 0101 1xxx X X X X X X [0, 2^51 - 1] + * 0b 0101 0xxx X X X X X X [-2^51, -1] + * 8 bytes: 0b 0110 1xxx X X X X X X X [0, 2^59 - 1] + * 0b 0110 0xxx X X X X X X X [-2^59, -1] + * 9 bytes: 0b 0111 1000 X X X X X X X X [0, 2^64 - 1] + * 0b 0111 0000 X X X X X X X X [-2^64, -1] + * + * NOTE: 0b 0111 1111 is used by 128 bits UUID + * 0b 0111 1110 is used by EdgeId + */ + int positive = val >= 0 ? 0x08 : 0x00; + if (~0x7ffL <= val && val <= 0x7ffL) { + int high3bits = (int) (val >> 8) & 0x07; + this.writeUInt8(0x00 | positive | high3bits); + this.writeUInt8((byte) val); + } else if (~0x7ffffL <= val && val <= 0x7ffffL) { + int high3bits = (int) (val >> 16) & 0x07; + this.writeUInt8(0x10 | positive | high3bits); + this.writeShort((short) val); + } else if (~0x7ffffffL <= val && val <= 0x7ffffffL) { + int high3bits = (int) (val >> 24 & 0x07); + this.writeUInt8(0x20 | positive | high3bits); + this.write((byte) (val >> 16)); + this.writeShort((short) val); + } else if (~0x7ffffffffL <= val && val <= 0x7ffffffffL) { + int high3bits = (int) (val >> 32) & 0x07; + this.writeUInt8(0x30 | positive | high3bits); + this.writeInt((int) val); + } else if (~0x7ffffffffffL <= val && val <= 0x7ffffffffffL) { + int high3bits = (int) (val >> 40) & 0x07; + this.writeUInt8(0x40 | positive | high3bits); + this.write((byte) (val >> 32)); + this.writeInt((int) val); + } else if (~0x7ffffffffffffL <= val && val <= 0x7ffffffffffffL) { + int high3bits = (int) (val >> 48) & 0x07; + this.writeUInt8(0x50 | positive | high3bits); + this.writeShort((short) (val >> 32)); + this.writeInt((int) val); + } else if (~0x7ffffffffffffffL <= val && val <= 0x7ffffffffffffffL) { + int high3bits = (int) (val >> 56) & 0x07; + this.writeUInt8(0x60 | positive | high3bits); + this.write((byte) (val >> 48)); + this.writeShort((short) (val >> 32)); + this.writeInt((int) val); + } else { + // high3bits is always 0b000 for 9 bytes number + this.writeUInt8(0x70 | positive); + this.writeLong(val); + } + } + + private long readNumber(byte b) { + // Parse the kind from byte 0kkksxxx + int kind = b >>> 4; + boolean positive = (b & 0x08) > 0; + long high3bits = b & 0x07; + long value = high3bits << ((kind + 1) * 8); + switch (kind) { + case 0: + value |= this.readUInt8(); + break; + case 1: + value |= this.readUInt16(); + break; + case 2: + value |= this.readUInt8() << 16 | this.readUInt16(); + break; + case 3: + value |= this.readUInt32(); + break; + case 4: + value |= (long) this.readUInt8() << 32 | this.readUInt32(); + break; + case 5: + value |= (long) this.readUInt16() << 32 | this.readUInt32(); + break; + case 6: + value |= (long) this.readUInt8() << 48 | + (long) this.readUInt16() << 32 | + this.readUInt32(); + break; + case 7: + assert high3bits == 0L; + value |= this.readLong(); + break; + default: + throw new AssertionError("Invalid length of number: " + kind); + } + if (!positive && kind < 7) { + // Restore the bits of the original negative number + long mask = Long.MIN_VALUE >> (52 - kind * 8); + value |= mask; + } + return value; + } + + private byte[] readBytesWithEnding() { + int start = this.buffer.position(); + boolean foundEnding = false; + while (this.remaining() > 0) { + byte current = this.read(); + if (current == STRING_ENDING_BYTE) { + foundEnding = true; + break; + } + } + E.checkArgument(foundEnding, "Not found ending '0x%s'", + Bytes.toHex(STRING_ENDING_BYTE)); + int end = this.buffer.position() - 1; + int len = end - start; + byte[] bytes = new byte[len]; + System.arraycopy(this.array(), start, bytes, 0, len); + return bytes; + } + + public byte[] remainingBytes(){ + int length = this.remaining(); + int start = this.position(); + byte[] bytes = new byte[length]; + System.arraycopy(this.array(), start, bytes, 0, length); + return bytes; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/DirectBinarySerializer.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/DirectBinarySerializer.java new file mode 100644 index 000000000..e758194b8 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/serializer/DirectBinarySerializer.java @@ -0,0 +1,128 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.serializer; + +import java.util.Arrays; +import java.util.Base64; + +import org.apache.hugegraph.util.Bytes; +import org.apache.hugegraph.util.Log; +import org.slf4j.Logger; + +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.schema.PropertyKey; +import com.google.common.primitives.Longs; + +public class DirectBinarySerializer { + + protected static final Logger LOG = Log.logger(DirectBinarySerializer.class); + + public static class DirectHugeElement { + private Id id; + private long expiredTime; + + public DirectHugeElement(Id id, long expiredTime) { + this.id = id; + this.expiredTime = expiredTime; + } + + public Id id() { + return id; + } + + public long expiredTime() { + return expiredTime; + } + } + + public DirectHugeElement parseIndex(byte[] key, byte[] value) { + long expiredTime = 0L; + + if (value.length > 0) { + // Get delimiter address + int delimiterIndex = + Bytes.indexOf(value, BytesBuffer.STRING_ENDING_BYTE); + + if (delimiterIndex >= 0) { + // Delimiter is in the data, need to parse from data + // Parse expiration time + byte[] expiredTimeBytes = + Arrays.copyOfRange(value, delimiterIndex + 1, + value.length); + + if (expiredTimeBytes.length > 0) { + byte[] rawBytes = + Base64.getDecoder().decode(expiredTimeBytes); + if (rawBytes.length >= Longs.BYTES) { + expiredTime = Longs.fromByteArray(rawBytes); + } + } + } + } + + return new DirectHugeElement(IdGenerator.of(key), expiredTime); + } + + public DirectHugeElement parseVertex(byte[] key, byte[] value) { + long expiredTime = 0L; + + BytesBuffer buffer = BytesBuffer.wrap(value); + // read schema label id + buffer.readId(); + // Skip edge properties + this.skipProperties(buffer); + // Parse edge expired time if needed + if (buffer.remaining() > 0) { + expiredTime = buffer.readVLong(); + } + + return new DirectHugeElement(IdGenerator.of(key), expiredTime); + } + + public DirectHugeElement parseEdge(byte[] key, byte[] value) { + long expiredTime = 0L; + + BytesBuffer buffer = BytesBuffer.wrap(value); + // Skip edge properties + this.skipProperties(buffer); + // Parse edge expired time if needed + if (buffer.remaining() > 0) { + expiredTime = buffer.readVLong(); + } + + return new DirectHugeElement(IdGenerator.of(key), expiredTime); + } + + private void skipProperties(BytesBuffer buffer) { + int size = buffer.readVInt(); + assert size >= 0; + for (int i = 0; i < size; i++) { + Id pkeyId = IdGenerator.of(buffer.readVInt()); + this.skipProperty(pkeyId, buffer); + } + } + + protected void skipProperty(Id pkeyId, BytesBuffer buffer) { + // Parse value + PropertyKey pkey = new PropertyKey(null, pkeyId, ""); + buffer.readProperty(pkey); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseEdge.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseEdge.java new file mode 100644 index 000000000..6362203d3 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseEdge.java @@ -0,0 +1,288 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.structure; + +import org.apache.hugegraph.HugeGraphSupplier; +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.id.EdgeId; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.SplicingIdGenerator; +import org.apache.hugegraph.schema.EdgeLabel; +import org.apache.hugegraph.schema.SchemaLabel; +import org.apache.hugegraph.schema.VertexLabel; +import org.apache.hugegraph.serializer.BytesBuffer; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.Directions; +import org.apache.hugegraph.type.define.HugeKeys; +import com.google.common.collect.ImmutableList; + +import org.apache.hugegraph.util.E; + +import java.util.ArrayList; +import java.util.List; + +/* Only as basic data container, id generation logic relies on upper layer encapsulation*/ +public class BaseEdge extends BaseElement implements Cloneable { + + private BaseVertex sourceVertex; + private BaseVertex targetVertex; + boolean isOutEdge; + + private String name; + + public BaseEdge(Id id, EdgeLabel label) { + this.id(id); + this.schemaLabel(label); + } + + public BaseEdge(SchemaLabel label, boolean isOutEdge) { + this.schemaLabel(label); + this.isOutEdge = isOutEdge; + } + + + public boolean isOutEdge() { + return isOutEdge; + } + + public void isOutEdge(boolean isOutEdge) { + this.isOutEdge = isOutEdge; + } + + public EdgeId idWithDirection() { + return ((EdgeId) this.id()).directed(true); + } + + @Override + public String name() { + if (this.name == null) { + this.name = SplicingIdGenerator.concatValues(sortValues()); + } + return this.name; + } + + public void name(String name) { + this.name = name; + } + + @Override + public HugeType type() { + // NOTE: we optimize the edge type that let it include direction + return this.isOutEdge() ? HugeType.EDGE_OUT : HugeType.EDGE_IN; + } + + public List sortValues() { + List sortKeys = this.schemaLabel().sortKeys(); + if (sortKeys.isEmpty()) { + return ImmutableList.of(); + } + List propValues = new ArrayList<>(sortKeys.size()); + for (Id sk : sortKeys) { + BaseProperty property = this.getProperty(sk); + E.checkState(property != null, + "The value of sort key '%s' can't be null", sk); + propValues.add(property.propertyKey().serialValue(property.value(), true)); + } + return propValues; + } + + public Directions direction() { + return this.isOutEdge ? Directions.OUT : Directions.IN; + } + + public Id sourceVertexId() { + return this.sourceVertex.id(); + } + + public Id targetVertexId() { + return this.targetVertex.id(); + } + + public void sourceVertex(BaseVertex sourceVertex) { + this.sourceVertex = sourceVertex; + } + + public BaseVertex sourceVertex() { + return this.sourceVertex; + } + + public void targetVertex(BaseVertex targetVertex) { + this.targetVertex = targetVertex; + } + + public BaseVertex targetVertex() { + return this.targetVertex; + } + + public Id ownerVertexId() { + return this.isOutEdge() ? this.sourceVertexId() : this.targetVertexId(); + } + + public Id otherVertexId() { + return this.isOutEdge() ? this.targetVertexId() : this.sourceVertexId() ; + } + + public void vertices(boolean outEdge, BaseVertex owner, BaseVertex other) { + this.isOutEdge = outEdge ; + if (outEdge) { + this.sourceVertex(owner); + this.targetVertex(other); + } else { + this.sourceVertex(other); + this.targetVertex(owner); + } + } + + + + public EdgeLabel schemaLabel() { + return (EdgeLabel) super.schemaLabel(); + } + + public BaseVertex ownerVertex() { + return this.isOutEdge() ? this.sourceVertex() : this.targetVertex(); + } + + public BaseVertex otherVertex() { + return this.isOutEdge() ? this.targetVertex() : this.sourceVertex(); + } + + public void assignId() { + // Generate an id and assign + if (this.schemaLabel().hasFather()) { + this.id(new EdgeId(this.ownerVertex().id(), this.direction(), + this.schemaLabel().fatherId(), + this.schemaLabel().id(), + this.name(), + this.otherVertex().id())); + } else { + this.id(new EdgeId(this.ownerVertex().id(), this.direction(), + this.schemaLabel().id(), + this.schemaLabel().id(), + this.name(), this.otherVertex().id())); + } + + + if (this.fresh()) { + int len = this.id().length(); + E.checkArgument(len <= BytesBuffer.BIG_ID_LEN_MAX, + "The max length of edge id is %s, but got %s {%s}", + BytesBuffer.BIG_ID_LEN_MAX, len, this.id()); + } + } + @Override + public Object sysprop(HugeKeys key) { + switch (key) { + case ID: + return this.id(); + case OWNER_VERTEX: + return this.ownerVertexId(); + case LABEL: + if (this.schemaLabel().fatherId() != null) { + return this.schemaLabel().fatherId(); + } else { + return this.schemaLabel().id(); + } + case DIRECTION: + return this.direction(); + + case SUB_LABEL: + return this.schemaLabel().id(); + + case OTHER_VERTEX: + return this.otherVertexId(); + case SORT_VALUES: + return this.name(); + case PROPERTIES: + return this.getPropertiesMap(); + default: + E.checkArgument(false, + "Invalid system property '%s' of Edge", key); + return null; + } + + } + + @Override + public BaseEdge clone() { + try { + return (BaseEdge) super.clone(); + } catch (CloneNotSupportedException e) { + throw new HugeException("Failed to clone HugeEdge", e); + } + } + + public BaseEdge switchOwner() { + BaseEdge edge = this.clone(); + edge.isOutEdge(!edge.isOutEdge()); + if (edge.id() != null) { + edge.id(((EdgeId) edge.id()).switchDirection()); + } + return edge; + } + + public static BaseEdge constructEdge(HugeGraphSupplier graph, + BaseVertex ownerVertex, + boolean isOutEdge, + EdgeLabel edgeLabel, + String sortValues, + Id otherVertexId) { + Id ownerLabelId = edgeLabel.sourceLabel(); + Id otherLabelId = edgeLabel.targetLabel(); + VertexLabel srcLabel; + VertexLabel tgtLabel; + if (graph == null) { + srcLabel = new VertexLabel(null, ownerLabelId, "UNDEF"); + tgtLabel = new VertexLabel(null, otherLabelId, "UNDEF"); + } else { + if (edgeLabel.general()) { + srcLabel = VertexLabel.GENERAL; + tgtLabel = VertexLabel.GENERAL; + } else { + srcLabel = graph.vertexLabelOrNone(ownerLabelId); + tgtLabel = graph.vertexLabelOrNone(otherLabelId); + } + } + + VertexLabel otherVertexLabel; + if (isOutEdge) { + ownerVertex.correctVertexLabel(srcLabel); + otherVertexLabel = tgtLabel; + } else { + ownerVertex.correctVertexLabel(tgtLabel); + otherVertexLabel = srcLabel; + } + BaseVertex otherVertex = new BaseVertex(otherVertexId, otherVertexLabel); + + ownerVertex.propLoaded(false); + otherVertex.propLoaded(false); + + BaseEdge edge = new BaseEdge(edgeLabel, isOutEdge); + edge.name(sortValues); + edge.vertices(isOutEdge, ownerVertex, otherVertex); + edge.assignId(); + + ownerVertex.addEdge(edge); + otherVertex.addEdge(edge.switchOwner()); + + return edge; + } + +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseElement.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseElement.java new file mode 100644 index 000000000..57fffe602 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseElement.java @@ -0,0 +1,355 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.structure; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import org.apache.hugegraph.util.CollectionUtil; +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.Log; +import org.eclipse.collections.api.map.primitive.MutableIntObjectMap; +import org.eclipse.collections.api.tuple.primitive.IntObjectPair; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; +import org.slf4j.Logger; + +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.schema.PropertyKey; +import org.apache.hugegraph.schema.SchemaLabel; +import org.apache.hugegraph.serializer.BytesBuffer; +import org.apache.hugegraph.type.GraphType; +import org.apache.hugegraph.type.Idfiable; +import org.apache.hugegraph.type.define.Cardinality; +import org.apache.hugegraph.type.define.HugeKeys; +import org.apache.hugegraph.util.collection.CollectionFactory; + + +public abstract class BaseElement implements GraphType, Idfiable, Serializable { + + private static final Logger LOG = Log.logger(BaseElement.class); + + public static final MutableIntObjectMap> EMPTY_MAP = + new IntObjectHashMap<>(); + + private static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX; + + MutableIntObjectMap> properties; + + Id id; + private SchemaLabel schemaLabel; + long expiredTime; // TODO: move into properties to keep small object + + private boolean removed; + private boolean fresh; + private boolean propLoaded; + private boolean defaultValueUpdated; + + public BaseElement() { + this.properties = EMPTY_MAP; + this.removed = false; + this.fresh = false; + this.propLoaded = true; + this.defaultValueUpdated = false; + } + + public void setProperties(MutableIntObjectMap> properties) { + this.properties = properties; + } + + public Id id(){ + return id; + } + + public void id(Id id) { + this.id = id; + } + + public boolean removed() { + return removed; + } + + public void removed(boolean removed) { + this.removed = removed; + } + + public boolean fresh() { + return fresh; + } + + public void fresh(boolean fresh) { + this.fresh = fresh; + } + + public boolean propLoaded() { + return propLoaded; + } + + public void propLoaded(boolean propLoaded) { + this.propLoaded = propLoaded; + } + + public boolean defaultValueUpdated() { + return defaultValueUpdated; + } + + public void defaultValueUpdated(boolean defaultValueUpdated) { + this.defaultValueUpdated = defaultValueUpdated; + } + public SchemaLabel schemaLabel() { + return schemaLabel; + } + + public void schemaLabel(SchemaLabel label) { + this.schemaLabel = label; + } + public long expiredTime() { + return expiredTime; + } + + public void expiredTime(long expiredTime) { + this.expiredTime = expiredTime; + } + + public boolean hasTtl() { + return this.schemaLabel.ttl() > 0L; + } + public boolean expired(long now) { + boolean expired; + SchemaLabel label = this.schemaLabel(); + if (label.ttl() == 0L) { + // No ttl, not expired + return false; + } + if (this.expiredTime() > 0L) { + // Has ttl and set expiredTime properly + expired = this.expiredTime() < now; + LOG.debug("The element {} {} with expired time {} and now {}", + this, expired ? "expired" : "not expired", + this.expiredTime(), now); + return expired; + } + // Has ttl, but failed to set expiredTime when insert + LOG.error("The element {} should have positive expired time, " + + "but got {}! ttl is {} ttl start time is {}", + this, this.expiredTime(), label.ttl(), label.ttlStartTimeName()); + if (SchemaLabel.NONE_ID.equals(label.ttlStartTime())) { + // No ttlStartTime, can't decide whether timeout, treat not expired + return false; + } + Date date = this.getPropertyValue(label.ttlStartTime()); + if (date == null) { + // No ttlStartTime, can't decide whether timeout, treat not expired + return false; + } + // Has ttlStartTime, re-calc expiredTime to decide whether timeout, + long expiredTime = date.getTime() + label.ttl(); + expired = expiredTime < now; + LOG.debug("The element {} {} with expired time {} and now {}", + this, expired ? "expired" : "not expired", + expiredTime, now); + return expired; + } + + public long ttl(long now) { + if (this.expiredTime() == 0L || this.expiredTime() < now) { + return 0L; + } + return this.expiredTime() - now; + } + protected BaseProperty newProperty(PropertyKey pkey, V val) { + return new BaseProperty<>(pkey, val); + } + + public boolean hasProperty(Id key) { + return this.properties.containsKey(intFromId(key)); + } + + public boolean hasProperties() { + return this.properties.size() > 0; + } + + + public void setExpiredTimeIfNeeded(long now) { + SchemaLabel label = this.schemaLabel(); + if (label.ttl() == 0L) { + return; + } + + if (SchemaLabel.NONE_ID.equals(label.ttlStartTime())) { + this.expiredTime(now + label.ttl()); + return; + } + Date date = this.getPropertyValue(label.ttlStartTime()); + if (date == null) { + this.expiredTime(now + label.ttl()); + return; + } + long expired = date.getTime() + label.ttl(); + E.checkArgument(expired > now, + "The expired time '%s' of '%s' is prior to now: %s", + new Date(expired), this, now); + this.expiredTime(expired); + } + + public void resetProperties() { + this.properties = CollectionFactory.newIntObjectMap(); + this.propLoaded(true); + } + + public V getPropertyValue(Id key) { + BaseProperty prop = this.properties.get(intFromId(key)); + if (prop == null) { + return null; + } + return (V) prop.value(); + } + public MutableIntObjectMap> properties() { + return this.properties; + } + + public void properties(MutableIntObjectMap> properties) { + this.properties = properties; + } + public BaseProperty getProperty(Id key) { + return (BaseProperty) this.properties.get(intFromId(key)); + } + + private BaseProperty addProperty(PropertyKey pkey, V value, + Supplier> supplier) { + assert pkey.cardinality().multiple(); + BaseProperty> property; + if (this.hasProperty(pkey.id())) { + property = this.getProperty(pkey.id()); + } else { + property = this.newProperty(pkey, supplier.get()); + this.addProperty(property); + } + + Collection values; + if (pkey.cardinality() == Cardinality.SET) { + if (value instanceof Set) { + values = (Set) value; + } else { + values = CollectionUtil.toSet(value); + } + } else { + assert pkey.cardinality() == Cardinality.LIST; + if (value instanceof List) { + values = (List) value; + } else { + values = CollectionUtil.toList(value); + } + } + property.value().addAll(values); + + // Any better ways? + return (BaseProperty) property; + } + + public BaseProperty addProperty(PropertyKey pkey, V value) { + BaseProperty prop = null; + switch (pkey.cardinality()) { + case SINGLE: + prop = this.newProperty(pkey, value); + this.addProperty(prop); + break; + case SET: + prop = this.addProperty(pkey, value, HashSet::new); + break; + case LIST: + prop = this.addProperty(pkey, value, ArrayList::new); + break; + default: + assert false; + break; + } + return prop; + } + + public BaseProperty addProperty(BaseProperty prop) { + if (this.properties == EMPTY_MAP) { + this.properties = new IntObjectHashMap<>(); // change to CollectionFactory.newIntObjectMap(); + } + PropertyKey pkey = prop.propertyKey(); + + E.checkArgument(this.properties.containsKey(intFromId(pkey.id())) || + this.properties.size() < MAX_PROPERTIES, + "Exceeded the maximum number of properties"); + return this.properties.put(intFromId(pkey.id()), prop); + } + public Map> getProperties() { + Map> props = new HashMap<>(); + for (IntObjectPair> e : this.properties.keyValuesView()) { + props.put(IdGenerator.of(e.getOne()), e.getTwo()); + } + return props; + } + + public BaseProperty removeProperty(Id key) { + return this.properties.remove(intFromId(key)); + } + + /* a util may be should be moved to other place */ + public static int intFromId(Id id) { + E.checkArgument(id instanceof IdGenerator.LongId, + "Can't get number from %s(%s)", id, id.getClass()); + return ((IdGenerator.LongId) id).intValue(); + } + + public abstract Object sysprop(HugeKeys key); + + public Map getPropertiesMap() { + Map props = new HashMap<>(); + for (IntObjectPair> e : this.properties.keyValuesView()) { + props.put(IdGenerator.of(e.getOne()), e.getTwo().value()); + } + return props; + } + + public int sizeOfProperties() { + return this.properties.size(); + } + + public int sizeOfSubProperties() { + int size = 0; + for (BaseProperty p : this.properties.values()) { + size++; + if (p.propertyKey().cardinality() != Cardinality.SINGLE && + p.value() instanceof Collection) { + size += ((Collection) p.value()).size(); + } + } + return size; + } + + @Override + public BaseElement clone() throws CloneNotSupportedException{ + return (BaseElement) super.clone(); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseProperty.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseProperty.java new file mode 100644 index 000000000..6cc8279c9 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseProperty.java @@ -0,0 +1,68 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.structure; + +import org.apache.hugegraph.schema.PropertyKey; +import org.apache.hugegraph.type.define.Cardinality; +import org.apache.hugegraph.type.define.DataType; + +public class BaseProperty { + private PropertyKey propertyKey; + + protected V value; + + public BaseProperty(PropertyKey propertyKey, V value) { + this.propertyKey = propertyKey; + this.value = value; + } + + public DataType getDataType() { + return propertyKey.dataType(); + } + + public void setDataType(DataType dataType) { + this.propertyKey.dataType(dataType); + } + + public Cardinality getCardinality() { + return propertyKey.cardinality(); + } + + public void setCardinality(Cardinality cardinality) { + this.propertyKey.cardinality(cardinality); + } + + public V value() { + return value; + } + + public void value(V value) { + this.value = value; + } + + public PropertyKey propertyKey() { + return propertyKey; + } + + public Object serialValue(boolean encodeNumber) { + return this.propertyKey.serialValue(this.value, encodeNumber); + } + +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseRawElement.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseRawElement.java new file mode 100644 index 000000000..c86887fd1 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseRawElement.java @@ -0,0 +1,57 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.structure; + +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.HugeKeys; + +public class BaseRawElement extends BaseElement implements Cloneable { + + private byte[] key; + private byte[] value; + + public BaseRawElement(byte[] key, byte[] value) { + this.key = key; + this.value = value; + } + + public byte[] key() { + return this.key; + } + + public byte[] value() { + return this.value; + } + + @Override + public Object sysprop(HugeKeys key) { + return null; + } + + @Override + public String name() { + return null; + } + + @Override + public HugeType type() { + return HugeType.KV_RAW; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseVertex.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseVertex.java new file mode 100644 index 000000000..d5d6028d7 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/BaseVertex.java @@ -0,0 +1,168 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.structure; + + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.hugegraph.perf.PerfUtil; +import org.apache.hugegraph.util.E; + +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.SplicingIdGenerator; +import org.apache.hugegraph.schema.SchemaLabel; +import org.apache.hugegraph.schema.VertexLabel; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.CollectionType; +import org.apache.hugegraph.type.define.HugeKeys; +import org.apache.hugegraph.type.define.IdStrategy; +import org.apache.hugegraph.util.collection.CollectionFactory; +import com.google.common.collect.ImmutableList; + +public class BaseVertex extends BaseElement implements Cloneable { + private static final List EMPTY_LIST = ImmutableList.of(); + + + protected Collection edges; + + public BaseVertex(Id id) { + this.edges = EMPTY_LIST; + id(id); + } + + public BaseVertex(Id id, SchemaLabel label) { + // Note: + // If vertex is OLAP Vertex, id is the id of the vertex that the olap property belongs to, not including the olap property id. + this(id); + this.schemaLabel(label); + } + + @Override + public String name() { + E.checkState(this.schemaLabel().idStrategy() == IdStrategy.PRIMARY_KEY, + "Only primary key vertex has name, " + + "but got '%s' with id strategy '%s'", + this, this.schemaLabel().idStrategy()); + String name; + if (this.id() != null) { + String[] parts = SplicingIdGenerator.parse(this.id()); + E.checkState(parts.length == 2, + "Invalid primary key vertex id '%s'", this.id()); + name = parts[1]; + } else { + assert this.id() == null; + List propValues = this.primaryValues(); + E.checkState(!propValues.isEmpty(), + "Primary values must not be empty " + + "(has properties %s)", hasProperties()); + name = SplicingIdGenerator.concatValues(propValues); + E.checkArgument(!name.isEmpty(), + "The value of primary key can't be empty"); + } + return name; + } + + @PerfUtil.Watched(prefix = "vertex") + public List primaryValues() { + E.checkArgument(this.schemaLabel().idStrategy() == IdStrategy.PRIMARY_KEY, + "The id strategy '%s' don't have primary keys", + this.schemaLabel().idStrategy()); + List primaryKeys = this.schemaLabel().primaryKeys(); + E.checkArgument(!primaryKeys.isEmpty(), + "Primary key can't be empty for id strategy '%s'", + IdStrategy.PRIMARY_KEY); + + List propValues = new ArrayList<>(primaryKeys.size()); + for (Id pk : primaryKeys) { + BaseProperty property = this.getProperty(pk); + E.checkState(property != null, + "The value of primary key '%s' can't be null" + /*this.graph().propertyKey(pk).name() complete log*/); + propValues.add(property.serialValue(true)); + } + return propValues; + } + + public void addEdge(BaseEdge edge) { + if (this.edges == EMPTY_LIST) { + this.edges = CollectionFactory.newList(CollectionType.EC); + } + this.edges.add(edge); + } + + public void correctVertexLabel(VertexLabel correctLabel) { + E.checkArgumentNotNull(correctLabel, "Vertex label can't be null"); + if (this.schemaLabel() != null && !this.schemaLabel().undefined() && + !correctLabel.undefined() && !this.schemaLabel().generalVl() && !correctLabel.generalVl()) { + E.checkArgument(this.schemaLabel().equals(correctLabel), + "[%s]'s Vertex label can't be changed from '%s' " + + "to '%s'", this.id(), this.schemaLabel(), + correctLabel); + } + this.schemaLabel(correctLabel); + } + public Collection edges() { + return this.edges; + } + + public void edges(Collection edges) { + this.edges = edges; + } + + @Override + public Object sysprop(HugeKeys key) { + switch (key) { + case ID: + return this.id(); + case LABEL: + return this.schemaLabel().id(); + case PRIMARY_VALUES: + return this.name(); + case PROPERTIES: + return this.getPropertiesMap(); + default: + E.checkArgument(false, + "Invalid system property '%s' of Vertex", key); + return null; + } + } + + public VertexLabel schemaLabel() { + return (VertexLabel)super.schemaLabel(); + } + + public boolean olap() { + return VertexLabel.OLAP_VL.equals(this.schemaLabel()); + } + + public HugeType type() { + // For Vertex type, when label is task, return TASK type, convenient for getting storage table information based on type + /* Magic: ~task ~taskresult ~variables*/ + if (schemaLabel() != null && + (schemaLabel().name().equals("~task") || + schemaLabel().name().equals("~taskresult") || + schemaLabel().name().equals("~variables"))) { + return HugeType.TASK; + } + return HugeType.VERTEX; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/Index.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/Index.java new file mode 100644 index 000000000..df3b34e46 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/Index.java @@ -0,0 +1,334 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.structure; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.InsertionOrderUtil; +import org.apache.hugegraph.util.NumericUtil; + +import org.apache.hugegraph.HugeGraphSupplier; +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.Id.IdType; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.id.SplicingIdGenerator; +import org.apache.hugegraph.schema.IndexLabel; +import org.apache.hugegraph.schema.SchemaElement; +import org.apache.hugegraph.serializer.BytesBuffer; +import org.apache.hugegraph.type.GraphType; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.DataType; +import com.google.common.collect.ImmutableSet; + +public class Index implements GraphType, Cloneable { + + private final HugeGraphSupplier graph; + private Object fieldValues; + private IndexLabel indexLabel; + /* + * Index read use elementIds, Index write always one element, use + * elementId + */ + private Set elementIds; + private IdWithExpiredTime elementId; + + public Index(HugeGraphSupplier graph, IndexLabel indexLabel) { + E.checkNotNull(graph, "graph"); + E.checkNotNull(indexLabel, "label"); + E.checkNotNull(indexLabel.id(), "label id"); + this.graph = graph; + this.indexLabel = indexLabel; + this.elementIds = new LinkedHashSet<>(); + this.fieldValues = null; + } + + public Index(HugeGraphSupplier graph, IndexLabel indexLabel, boolean write) { + E.checkNotNull(graph, "graph"); + E.checkNotNull(indexLabel, "label"); + E.checkNotNull(indexLabel.id(), "label id"); + this.graph = graph; + this.indexLabel = indexLabel; + if (!write) { + this.elementIds = new LinkedHashSet<>(); + } + this.elementId = null; + this.fieldValues = null; + } + + @Override + public String name() { + return this.indexLabel.name(); + } + + @Override + public HugeType type() { + if (this.indexLabel == IndexLabel.label(HugeType.VERTEX)) { + return HugeType.VERTEX_LABEL_INDEX; + } else if (this.indexLabel == IndexLabel.label(HugeType.EDGE)) { + return HugeType.EDGE_LABEL_INDEX; + } + return this.indexLabel.indexType().type(); + } + + public HugeGraphSupplier graph() { + return this.graph; + } + + public Id id() { + return formatIndexId(type(), this.indexLabelId(), this.fieldValues()); + } + + public Object fieldValues() { + return this.fieldValues; + } + + public void fieldValues(Object fieldValues) { + this.fieldValues = fieldValues; + } + + public Id indexLabelId() { + return this.indexLabel.id(); + } + + public IndexLabel indexLabel() { + return this.indexLabel; + } + + public IdWithExpiredTime elementIdWithExpiredTime() { + if (this.elementIds == null) { + return this.elementId; + } + E.checkState(this.elementIds.size() == 1, + "Expect one element id, actual %s", + this.elementIds.size()); + return this.elementIds.iterator().next(); + } + + public Id elementId() { + return this.elementIdWithExpiredTime().id(); + } + + public Set elementIds() { + if (this.elementIds == null) { + return ImmutableSet.of(); + } + Set ids = InsertionOrderUtil.newSet(this.elementIds.size()); + for (IdWithExpiredTime idWithExpiredTime : this.elementIds) { + ids.add(idWithExpiredTime.id()); + } + return Collections.unmodifiableSet(ids); + } + + public Set expiredElementIds() { + long now = this.graph.now(); + Set expired = InsertionOrderUtil.newSet(); + for (IdWithExpiredTime id : this.elementIds) { + if (0L < id.expiredTime && id.expiredTime < now) { + expired.add(id); + } + } + this.elementIds.removeAll(expired); + return expired; + } + + public void elementIds(Id elementId) { + this.elementIds(elementId, 0L); + } + + public void elementIds(Id elementId, long expiredTime) { + if (this.elementIds == null) { + this.elementId = new IdWithExpiredTime(elementId, expiredTime); + } else { + this.elementIds.add(new IdWithExpiredTime(elementId, expiredTime)); + } + } + + public void resetElementIds() { + this.elementIds = null; + } + + public long expiredTime() { + return this.elementIdWithExpiredTime().expiredTime(); + } + + public boolean hasTtl() { + if ((this.indexLabel() == IndexLabel.label(HugeType.VERTEX) || + this.indexLabel() == IndexLabel.label(HugeType.EDGE)) && + this.expiredTime() > 0) { + // LabelIndex index, if element has expiration time, then index also has TTL + return true; + } + + if (this.indexLabel.system()) { + return false; + } + return this.indexLabel.baseElement().ttl() > 0L; + } + + public long ttl() { + return this.expiredTime() - this.graph.now(); + } + + @Override + public Index clone() { + try { + return (Index) super.clone(); + } catch (CloneNotSupportedException e) { + throw new HugeException("Failed to clone Index", e); + } + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Index)) { + return false; + } + + Index other = (Index) obj; + return this.id().equals(other.id()); + } + + @Override + public int hashCode() { + return this.id().hashCode(); + } + + @Override + public String toString() { + return String.format("{label=%s<%s>, fieldValues=%s, elementIds=%s}", + this.indexLabel.name(), + this.indexLabel.indexType().string(), + this.fieldValues, this.elementIds); + } + + + public static Id formatIndexId(HugeType type, Id indexLabelId, + Object fieldValues) { + if (type.isStringIndex()) { + String value = ""; + if (fieldValues instanceof Id) { + value = IdGenerator.asStoredString((Id) fieldValues); + } else if (fieldValues != null) { + value = fieldValues.toString(); + } + /* + * Modify order between index label and field-values to put the + * index label in front(hugegraph-1317) + */ + String strIndexLabelId = IdGenerator.asStoredString(indexLabelId); + // Add id prefix according to type + return SplicingIdGenerator.splicing(type.string(), strIndexLabelId, value); + } else { + assert type.isRangeIndex(); + int length = type.isRange4Index() ? 4 : 8; + // 1 is table type, 4 is labelId, length is value + BytesBuffer buffer = BytesBuffer.allocate(1 + 4 + length); + // Add table type id + buffer.write(type.code()); + + buffer.writeInt(SchemaElement.schemaId(indexLabelId)); + if (fieldValues != null) { + E.checkState(fieldValues instanceof Number, + "Field value of range index must be number:" + + " %s", fieldValues.getClass().getSimpleName()); + byte[] bytes = number2bytes((Number) fieldValues); + buffer.write(bytes); + } + return buffer.asId(); + } + } + + public static Index parseIndexId(HugeGraphSupplier graph, HugeType type, + byte[] id) { + Object values; + IndexLabel indexLabel; + if (type.isStringIndex()) { + Id idObject = IdGenerator.of(id, IdType.STRING); + String[] parts = SplicingIdGenerator.parse(idObject); + E.checkState(parts.length == 3, "Invalid secondary index id"); + Id label = IdGenerator.ofStoredString(parts[1], IdType.LONG); + indexLabel = IndexLabel.label(graph, label); + values = parts[2]; + } else { + assert type.isRange4Index() || type.isRange8Index(); + final int labelLength = 4; + E.checkState(id.length > labelLength, "Invalid range index id"); + BytesBuffer buffer = BytesBuffer.wrap(id); + // Read the first byte representing the table type + final int hugeTypeCodeLength = 1; + byte[] read = buffer.read(hugeTypeCodeLength); + + Id label = IdGenerator.of(buffer.readInt()); + indexLabel = IndexLabel.label(graph, label); + List fields = indexLabel.indexFields(); + E.checkState(fields.size() == 1, "Invalid range index fields"); + DataType dataType = graph.propertyKey(fields.get(0)).dataType(); + E.checkState(dataType.isNumber() || dataType.isDate(), + "Invalid range index field type"); + Class clazz = dataType.isNumber() ? + dataType.clazz() : DataType.LONG.clazz(); + values = bytes2number(buffer.read(id.length - labelLength - hugeTypeCodeLength), clazz); + } + Index index = new Index(graph, indexLabel); + index.fieldValues(values); + return index; + } + + public static byte[] number2bytes(Number number) { + if (number instanceof Byte) { + // Handle byte as integer to store as 4 bytes in RANGE4_INDEX + number = number.intValue(); + } + return NumericUtil.numberToSortableBytes(number); + } + + public static Number bytes2number(byte[] bytes, Class clazz) { + return NumericUtil.sortableBytesToNumber(bytes, clazz); + } + + public static class IdWithExpiredTime { + + private Id id; + private long expiredTime; + + public IdWithExpiredTime(Id id, long expiredTime) { + this.id = id; + this.expiredTime = expiredTime; + } + + public Id id() { + return this.id; + } + + public long expiredTime() { + return this.expiredTime; + } + + @Override + public String toString() { + return String.format("%s(%s)", this.id, this.expiredTime); + } + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/KvElement.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/KvElement.java new file mode 100644 index 000000000..ac8618d73 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/KvElement.java @@ -0,0 +1,101 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.structure; + +import java.util.List; + +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.HugeKeys; + +/** + * for aggregation calculation + */ +public class KvElement extends BaseElement implements Comparable{ + + private List keys; + + private List values; + + private KvElement(List keys, List values) { + this.keys = keys; + this.values = values; + } + + public static KvElement of (List keys, List values) { + return new KvElement(keys, values); + } + + public List getKeys() { + return keys; + } + + public List getValues() { + return values; + } + + @Override + public Object sysprop(HugeKeys key) { + return null; + } + + @Override + public String name() { + return null; + } + + @Override + public HugeType type() { + return HugeType.KV_TYPE; + } + + /** + * compare by keys + * @param other the object to be compared. + * @return -1 = this > other, 0 = this == other, 1 = this < other. + */ + @Override + public int compareTo(KvElement other) { + if (this == other) { + return 0; + } + + if (other == null || other.keys == null) { + return keys == null ? 0 : 1; + } + + int len = Math.min(keys.size(), other.keys.size()); + for (int i = 0; i < len; i++) { + var o1 = keys.get(i); + var o2 = other.keys.get(i); + if (o1 != o2) { + if (o1 == null || o2 == null) { + return o1 == null ? -1 : 1; + } + + int v = o1.compareTo(o2); + if (v != 0) { + return v; + } + } + } + + return keys.size() - other.keys.size(); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/builder/IndexBuilder.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/builder/IndexBuilder.java new file mode 100644 index 000000000..ef68e3132 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/structure/builder/IndexBuilder.java @@ -0,0 +1,327 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.structure.builder; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hugegraph.HugeGraphSupplier; +import org.apache.hugegraph.analyzer.Analyzer; +import org.apache.hugegraph.analyzer.AnalyzerFactory; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.query.ConditionQuery; +import org.apache.hugegraph.schema.EdgeLabel; +import org.apache.hugegraph.schema.IndexLabel; +import org.apache.hugegraph.schema.SchemaLabel; +import org.apache.hugegraph.schema.VertexLabel; +import org.apache.hugegraph.structure.BaseEdge; +import org.apache.hugegraph.structure.BaseElement; +import org.apache.hugegraph.structure.BaseProperty; +import org.apache.hugegraph.structure.BaseVertex; +import org.apache.hugegraph.structure.Index; +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.Log; +import org.apache.hugegraph.util.NumericUtil; +import org.slf4j.Logger; + +public class IndexBuilder { + private static final Logger LOG = Log.logger(IndexBuilder.class); + + private final HugeGraphSupplier graph; + private final Analyzer textAnalyzer; + + + public static final String INDEX_SYM_NULL = "\u0001"; + public static final String INDEX_SYM_EMPTY = "\u0002"; + public static final char INDEX_SYM_MAX = '\u0003'; + + private static final String TEXT_ANALYZER = "search.text_analyzer"; + private static final String TEXT_ANALYZER_MODE = + "search.text_analyzer_mode"; + + private static final String DEFAULT_TEXT_ANALYZER = "ikanalyzer"; + private static final String DEFAULT_TEXT_ANALYZER_MODE = "smart"; + + public IndexBuilder(HugeGraphSupplier graph) { + this.graph = graph; + + String name = graph.configuration().get(String.class, TEXT_ANALYZER); + String mode = graph.configuration().get(String.class, + TEXT_ANALYZER_MODE); + + name = name == null ? DEFAULT_TEXT_ANALYZER : name; + mode = mode == null ? DEFAULT_TEXT_ANALYZER_MODE : mode; + + LOG.debug("Loading text analyzer '{}' with mode '{}' for graph '{}'", + name, mode, graph.name()); + this.textAnalyzer = AnalyzerFactory.analyzer(name, mode); + } + + public List buildLabelIndex(BaseElement element) { + + List indexList = new ArrayList(); + // Don't Build label index if it's not enabled + SchemaLabel label = element.schemaLabel(); + + // Build label index if backend store not supports label-query + Index index = new Index(graph, + IndexLabel.label(element.type()), + true); + index.fieldValues(element.schemaLabel().id()); + index.elementIds(element.id(), element.expiredTime()); + + indexList.add(index); + + /**When adding a sub-type edge, put its edgeID into the parent type's edgeLabelIndex at the same time + * to support: g.E().hasLabel("parent type") + * */ + if (element instanceof BaseEdge && ((EdgeLabel) label).hasFather()) { + Index fatherIndex = new Index(graph, + IndexLabel.label(element.type())); + fatherIndex.fieldValues(((EdgeLabel) label).fatherId()); + fatherIndex.elementIds(element.id(), element.expiredTime()); + + indexList.add(fatherIndex); + } + + return indexList; + } + + public List buildVertexOlapIndex(BaseVertex vertex) { + + List indexs = new ArrayList<>(); + + Id pkId = vertex.getProperties().keySet().iterator().next(); + Collection indexLabels = graph.indexLabels(); + for (IndexLabel il : indexLabels) { + if (il.indexFields().contains(pkId)) { + indexs.addAll(this.buildIndex(vertex, il)); + } + } + + return indexs; + } + + public List buildVertexIndex(BaseVertex vertex) { + List indexs = new ArrayList<>(); + + VertexLabel label = vertex.schemaLabel(); + + if (label.enableLabelIndex()) { + indexs.addAll(this.buildLabelIndex(vertex)); + } + + for (Id il : label.indexLabels()) { + indexs.addAll(this.buildIndex(vertex, graph.indexLabel(il))); + } + + return indexs; + } + + public List buildEdgeIndex(BaseEdge edge) { + List indexs = new ArrayList<>(); + + EdgeLabel label = edge.schemaLabel(); + + if (label.enableLabelIndex()) { + indexs.addAll(this.buildLabelIndex(edge)); + } + + + for (Id il : label.indexLabels()) { + indexs.addAll(this.buildIndex(edge, graph.indexLabel(il))); + } + + return indexs; + } + + /** + * Build index(user properties) of vertex or edge + * Notice: This method does not use unique index validation to check if the current element already exists + * + * @param indexLabel the index label + * @param element the properties owner + */ + public List buildIndex(BaseElement element, IndexLabel indexLabel) { + E.checkArgument(indexLabel != null, + "Not exist index label with id '%s'", indexLabel.id()); + + List indexs = new ArrayList<>(); + + // Collect property values of index fields + List allPropValues = new ArrayList<>(); + int fieldsNum = indexLabel.indexFields().size(); + int firstNullField = fieldsNum; + for (Id fieldId : indexLabel.indexFields()) { + BaseProperty property = element.getProperty(fieldId); + if (property == null) { + E.checkState(hasNullableProp(element, fieldId), + "Non-null property '%s' is null for '%s'", + graph.propertyKey(fieldId), element); + if (firstNullField == fieldsNum) { + firstNullField = allPropValues.size(); + } + allPropValues.add(INDEX_SYM_NULL); + } else { + E.checkArgument(!INDEX_SYM_NULL.equals(property.value()), + "Illegal value of index property: '%s'", + INDEX_SYM_NULL); + allPropValues.add(property.value()); + } + } + + if (firstNullField == 0 && !indexLabel.indexType().isUnique()) { + // The property value of first index field is null + return indexs; + } + // Not build index for record with nullable field (except unique index) + List propValues = allPropValues.subList(0, firstNullField); + + // Expired time + long expiredTime = element.expiredTime(); + + // Build index for each index type + switch (indexLabel.indexType()) { + case RANGE_INT: + case RANGE_FLOAT: + case RANGE_LONG: + case RANGE_DOUBLE: + E.checkState(propValues.size() == 1, + "Expect only one property in range index"); + Object value = NumericUtil.convertToNumber(propValues.get(0)); + indexs.add(this.buildIndex(indexLabel, value, element.id(), + expiredTime)); + break; + case SEARCH: + E.checkState(propValues.size() == 1, + "Expect only one property in search index"); + value = propValues.get(0); + Set words = + this.segmentWords(propertyValueToString(value)); + for (String word : words) { + indexs.add(this.buildIndex(indexLabel, word, element.id(), + expiredTime)); + } + break; + case SECONDARY: + // Secondary index maybe include multi prefix index + if (isCollectionIndex(propValues)) { + /* + * Property value is a collection + * we should create index for each item + */ + for (Object propValue : + (Collection) propValues.get(0)) { + value = ConditionQuery.concatValuesLimitLength( + propValue); + value = escapeIndexValueIfNeeded((String) value); + indexs.add(this.buildIndex(indexLabel, value, + element.id(), + expiredTime)); + } + } else { + for (int i = 0, n = propValues.size(); i < n; i++) { + List prefixValues = + propValues.subList(0, i + 1); + value = ConditionQuery.concatValuesLimitLength( + prefixValues); + value = escapeIndexValueIfNeeded((String) value); + indexs.add(this.buildIndex(indexLabel, value, + element.id(), + expiredTime)); + } + } + break; + case SHARD: + value = ConditionQuery.concatValuesLimitLength(propValues); + value = escapeIndexValueIfNeeded((String) value); + indexs.add(this.buildIndex(indexLabel, value, element.id(), + expiredTime)); + break; + case UNIQUE: + value = ConditionQuery.concatValuesLimitLength(allPropValues); + assert !"".equals(value); + indexs.add(this.buildIndex(indexLabel, value, element.id(), + expiredTime)); + break; + default: + throw new AssertionError(String.format( + "Unknown index type '%s'", indexLabel.indexType())); + } + + return indexs; + } + + private Index buildIndex(IndexLabel indexLabel, Object propValue, + Id elementId, long expiredTime) { + Index index = new Index(graph, indexLabel, true); + index.fieldValues(propValue); + index.elementIds(elementId, expiredTime); + + return index; + } + + + private static String escapeIndexValueIfNeeded(String value) { + for (int i = 0; i < value.length(); i++) { + char ch = value.charAt(i); + if (ch <= INDEX_SYM_MAX) { + /* + * Escape symbols can't be used due to impossible to parse, + * and treat it as illegal value for the origin text property + */ + E.checkArgument(false, "Illegal char '\\u000%s' " + + "in index property: '%s'", (int) ch, + value); + } + } + if (value.isEmpty()) { + // Escape empty String to INDEX_SYM_EMPTY (char `\u0002`) + value = INDEX_SYM_EMPTY; + } + return value; + } + + private static boolean hasNullableProp(BaseElement element, Id key) { + return element.schemaLabel().nullableKeys().contains(key); + } + + private static boolean isCollectionIndex(List propValues) { + return propValues.size() == 1 && + propValues.get(0) instanceof Collection; + } + + private Set segmentWords(String text) { + return this.textAnalyzer.segment(text); + } + + private static String propertyValueToString(Object value) { + /* + * Join collection items with white space if the value is Collection, + * or else keep the origin value. + */ + return value instanceof Collection ? + StringUtils.join(((Collection) value).toArray(), " ") : + value.toString(); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/GraphType.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/GraphType.java new file mode 100644 index 000000000..8e6825a94 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/GraphType.java @@ -0,0 +1,23 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type; + +public interface GraphType extends Namifiable, Typifiable { +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/HugeType.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/HugeType.java new file mode 100644 index 000000000..6dde30c56 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/HugeType.java @@ -0,0 +1,213 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hugegraph.type.define.SerialEnum; + +public enum HugeType implements SerialEnum { + + UNKNOWN(0, "UNKNOWN"), + + /* Schema types */ + VERTEX_LABEL(1, "VL"), + EDGE_LABEL(2, "EL"), + PROPERTY_KEY(3, "PK"), + INDEX_LABEL(4, "IL"), + + COUNTER(50, "C"), + + /* Data types */ + VERTEX(101, "V"), + // System meta + SYS_PROPERTY(102, "S"), + // Property + PROPERTY(103, "U"), + // Vertex aggregate property + AGGR_PROPERTY_V(104, "VP"), + // Edge aggregate property + AGGR_PROPERTY_E(105, "EP"), + // Olap property + OLAP(106, "AP"), + // Edge + EDGE(120, "E"), + // Edge's direction is OUT for the specified vertex + EDGE_OUT(130, "O"), + // Edge's direction is IN for the specified vertex + EDGE_IN(140, "I"), + + SECONDARY_INDEX(150, "SI"), + VERTEX_LABEL_INDEX(151, "VI"), + EDGE_LABEL_INDEX(152, "EI"), + RANGE_INT_INDEX(160, "II"), + RANGE_FLOAT_INDEX(161, "FI"), + RANGE_LONG_INDEX(162, "LI"), + RANGE_DOUBLE_INDEX(163, "DI"), + SEARCH_INDEX(170, "AI"), + SHARD_INDEX(175, "HI"), + UNIQUE_INDEX(178, "UI"), + + TASK(180, "T"), + SERVER(181, "SERVER"), + + VARIABLE(185,"VA"), + + KV_TYPE(200, "KV"), + KV_RAW(201, "KVR"), + + // System schema + SYS_SCHEMA(250, "SS"), + + MAX_TYPE(255, "~"); + + private byte type = 0; + private String name; + + private static final Map ALL_NAME = new HashMap<>(); + + static { + SerialEnum.register(HugeType.class); + for (HugeType type : values()) { + ALL_NAME.put(type.name, type); + } + } + + HugeType(int type, String name) { + assert type < 256; + this.type = (byte) type; + this.name = name; + } + + @Override + public byte code() { + return this.type; + } + + public String string() { + return this.name; + } + + public String readableName() { + return this.name().replace('_', ' ').toLowerCase(); + } + + public boolean isSchema() { + return this == HugeType.VERTEX_LABEL || + this == HugeType.EDGE_LABEL || + this == HugeType.PROPERTY_KEY || + this == HugeType.INDEX_LABEL; + } + + public boolean isGraph() { + return this.isVertex() || this.isEdge() ; + } + + public boolean isVertex() { + // Consider task vertex variable as the same, all used to store HugeVertex structure + return this == HugeType.VERTEX || this == HugeType.TASK || + this == HugeType.VARIABLE; + } + + public boolean isEdge() { + return this == EDGE || this == EDGE_OUT || this == EDGE_IN; + } + + public boolean isEdgeLabel() { + return this == EDGE_LABEL; + } + + + public boolean isIndex() { + return this == VERTEX_LABEL_INDEX || this == EDGE_LABEL_INDEX || + this == SECONDARY_INDEX || this == SEARCH_INDEX || + this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX || + this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX || + this == SHARD_INDEX || this == UNIQUE_INDEX; + } + + public boolean isLabelIndex() { + return this == VERTEX_LABEL_INDEX || this == EDGE_LABEL_INDEX; + } + + public boolean isStringIndex() { + return this == VERTEX_LABEL_INDEX || this == EDGE_LABEL_INDEX || + this == SECONDARY_INDEX || this == SEARCH_INDEX || + this == SHARD_INDEX || this == UNIQUE_INDEX; + } + + public boolean isNumericIndex() { + return this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX || + this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX || + this == SHARD_INDEX; + } + + public boolean isSecondaryIndex() { + return this == VERTEX_LABEL_INDEX || this == EDGE_LABEL_INDEX || + this == SECONDARY_INDEX; + } + + public boolean isSearchIndex() { + return this == SEARCH_INDEX; + } + + public boolean isRangeIndex() { + return this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX || + this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX; + } + + public boolean isRange4Index() { + return this == RANGE_INT_INDEX || this == RANGE_FLOAT_INDEX; + } + + public boolean isRange8Index() { + return this == RANGE_LONG_INDEX || this == RANGE_DOUBLE_INDEX; + } + + public boolean isShardIndex() { + return this == SHARD_INDEX; + } + + public boolean isUniqueIndex() { + return this == UNIQUE_INDEX; + } + + public boolean isVertexAggregateProperty() { + return this == AGGR_PROPERTY_V; + } + + public boolean isEdgeAggregateProperty() { + return this == AGGR_PROPERTY_E; + } + + public boolean isAggregateProperty() { + return this.isVertexAggregateProperty() || + this.isEdgeAggregateProperty(); + } + + public static HugeType fromString(String type) { + return ALL_NAME.get(type); + } + + public static HugeType fromCode(byte code) { + return SerialEnum.fromCode(HugeType.class, code); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Idfiable.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Idfiable.java new file mode 100644 index 000000000..c5a58c0eb --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Idfiable.java @@ -0,0 +1,27 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type; + +import org.apache.hugegraph.id.Id; + +public interface Idfiable { + + public Id id(); +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Indexfiable.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Indexfiable.java new file mode 100644 index 000000000..a809a49a7 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Indexfiable.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type; + +import org.apache.hugegraph.id.Id; + +import java.util.Set; + +public interface Indexfiable { + + public Set indexLabels(); +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Namifiable.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Namifiable.java new file mode 100644 index 000000000..a2448acdf --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Namifiable.java @@ -0,0 +1,31 @@ +// Copyright 2017 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.hugegraph.type; + +/** + * Represents an entity that can be uniquely identified by a String name. + * + * @author Matthias Broecheler (me@matthiasb.com) + */ +public interface Namifiable { + + /** + * Returns the unique name of this entity. + * + * @return Name of this entity. + */ + String name(); + +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Propfiable.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Propfiable.java new file mode 100644 index 000000000..021d0c00f --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Propfiable.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type; + +import java.util.Set; + +import org.apache.hugegraph.id.Id; + +public interface Propfiable { + + public Set properties(); +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Typifiable.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Typifiable.java new file mode 100644 index 000000000..9a510722b --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/Typifiable.java @@ -0,0 +1,26 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type; + +public interface Typifiable { + + // Return schema/data type + public HugeType type(); +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Action.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Action.java new file mode 100644 index 000000000..042594c22 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Action.java @@ -0,0 +1,76 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +public enum Action implements SerialEnum { + + INSERT(1, "insert"), + + APPEND(2, "append"), + + ELIMINATE(3, "eliminate"), + + DELETE(4, "delete"), + + UPDATE_IF_PRESENT(5, "update_if_present"), + + UPDATE_IF_ABSENT(6, "update_if_absent"); + + private final byte code; + private final String name; + + static { + SerialEnum.register(Action.class); + } + + Action(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public static Action fromCode(byte code) { + switch (code) { + case 1: + return INSERT; + case 2: + return APPEND; + case 3: + return ELIMINATE; + case 4: + return DELETE; + case 5: + return UPDATE_IF_PRESENT; + case 6: + return UPDATE_IF_ABSENT; + default: + throw new AssertionError("Unsupported action code: " + code); + } + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/AggregateType.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/AggregateType.java new file mode 100644 index 000000000..e949d4af1 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/AggregateType.java @@ -0,0 +1,93 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +public enum AggregateType implements SerialEnum { + + NONE(0, "none"), + MAX(1, "max"), + MIN(2, "min"), + SUM(3, "sum"), + OLD(4, "old"), + SET(5, "set"), + LIST(6, "list"); + + private final byte code; + private final String name; + + static { + SerialEnum.register(AggregateType.class); + } + + AggregateType(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public boolean isNone() { + return this == NONE; + } + + public boolean isMax() { + return this == MAX; + } + + public boolean isMin() { + return this == MIN; + } + + public boolean isSum() { + return this == SUM; + } + + public boolean isNumber() { + return this.isMax() || this.isMin() || this.isSum(); + } + + public boolean isOld() { + return this == OLD; + } + + public boolean isSet() { + return this == SET; + } + + public boolean isList() { + return this == LIST; + } + + public boolean isUnion() { + return this == SET || this == LIST; + } + + public boolean isIndexable() { + return this == NONE || this == MAX || this == MIN || this == OLD; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Cardinality.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Cardinality.java new file mode 100644 index 000000000..cc935ef43 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Cardinality.java @@ -0,0 +1,69 @@ +// Copyright 2017 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.hugegraph.type.define; + +/** + * The cardinality of the values associated with given key for a particular element. + * + * @author Matthias Broecheler (me@matthiasb.com) + */ +public enum Cardinality implements SerialEnum { + + /** + * Only a single value may be associated with the given key. + */ + SINGLE(1, "single"), + + /** + * Multiple values and duplicate values may be associated with the given + * key. + */ + LIST(2, "list"), + + /** + * Multiple but distinct values may be associated with the given key. + */ + SET(3, "set"); + + private byte code = 0; + private String name = null; + + static { + SerialEnum.register(Cardinality.class); + } + + Cardinality(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public boolean single() { + return this == SINGLE; + } + + public boolean multiple() { + return this == LIST || this == SET; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/CollectionType.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/CollectionType.java new file mode 100644 index 000000000..e8ff98ec9 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/CollectionType.java @@ -0,0 +1,68 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +public enum CollectionType implements SerialEnum { + + // Java Collection Framework + JCF(1, "jcf"), + + // Eclipse Collection + EC(2, "ec"), + + // FastUtil + FU(3, "fu"); + + private final byte code; + private final String name; + + static { + SerialEnum.register(CollectionType.class); + } + + CollectionType(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public static CollectionType fromCode(byte code) { + switch (code) { + case 1: + return JCF; + case 2: + return EC; + case 3: + return FU; + default: + throw new AssertionError( + "Unsupported collection code: " + code); + } + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/DataType.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/DataType.java new file mode 100644 index 000000000..6a04a8303 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/DataType.java @@ -0,0 +1,224 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +import java.nio.ByteBuffer; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import org.apache.hugegraph.util.Bytes; +import org.apache.hugegraph.util.DateUtil; + +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.serializer.BytesBuffer; +import org.apache.hugegraph.util.Blob; +import org.apache.hugegraph.util.StringEncoding; +import com.google.common.collect.ImmutableSet; + +public enum DataType implements SerialEnum { + + UNKNOWN(0, "unknown", Object.class), + OBJECT(1, "object", Object.class), + BOOLEAN(2, "boolean", Boolean.class), + BYTE(3, "byte", Byte.class), + INT(4, "int", Integer.class), + LONG(5, "long", Long.class), + FLOAT(6, "float", Float.class), + DOUBLE(7, "double", Double.class), + TEXT(8, "text", String.class), + BLOB(9, "blob", Blob.class), + DATE(10, "date", Date.class), + UUID(11, "uuid", UUID.class); + + private final byte code; + private final String name; + private final Class clazz; + + private static final ImmutableSet SPECIAL_FLOATS = ImmutableSet.of("-Infinity", "Infinity", "NaN"); + + + static { + SerialEnum.register(DataType.class); + } + + DataType(int code, String name, Class clazz) { + assert code < 256; + this.code = (byte) code; + this.name = name; + this.clazz = clazz; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public Class clazz() { + return this.clazz; + } + + public boolean isText() { + return this == DataType.TEXT; + } + + public boolean isNumber() { + return this == BYTE || this == INT || this == LONG || + this == FLOAT || this == DOUBLE; + } + + public boolean isNumber4() { + // Store index value of Byte using 4 bytes + return this == BYTE || this == INT || this == FLOAT; + } + + public boolean isNumber8() { + return this == LONG || this == DOUBLE; + } + + public boolean isBlob() { + return this == DataType.BLOB; + } + + public boolean isDate() { + return this == DataType.DATE; + } + + public boolean isUUID() { + return this == DataType.UUID; + } + + public Number valueToNumber(V value) { + if (!(this.isNumber() && value instanceof Number) && + !(value instanceof String && SPECIAL_FLOATS.contains(value))) { + return null; + } + if (this.clazz.isInstance(value)) { + return (Number) value; + } + + Number number; + try { + switch (this) { + case BYTE: + number = Byte.valueOf(value.toString()); + break; + case INT: + number = Integer.valueOf(value.toString()); + break; + case LONG: + number = Long.valueOf(value.toString()); + break; + case FLOAT: + number = Float.valueOf(value.toString()); + break; + case DOUBLE: + number = Double.valueOf(value.toString()); + break; + default: + throw new AssertionError(String.format( + "Number type only contains Byte, Integer, " + + "Long, Float, Double, but got %s", this.clazz())); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "Can't read '%s' as %s: %s", + value, this.name, e.getMessage())); + } + return number; + } + + public Date valueToDate(V value) { + if (!this.isDate()) { + return null; + } + if (value instanceof Date) { + return (Date) value; + } else if (value instanceof Integer) { + return new Date(((Number) value).intValue()); + } else if (value instanceof Long) { + return new Date(((Number) value).longValue()); + } else if (value instanceof String) { + return DateUtil.parse((String) value); + } + return null; + } + + public UUID valueToUUID(V value) { + if (!this.isUUID()) { + return null; + } + if (value instanceof UUID) { + return (UUID) value; + } else if (value instanceof String) { + return StringEncoding.uuid((String) value); + } + return null; + } + + public Blob valueToBlob(V value) { + if (!this.isBlob()) { + return null; + } + if (value instanceof Blob) { + return (Blob) value; + } else if (value instanceof byte[]) { + return Blob.wrap((byte[]) value); + } else if (value instanceof ByteBuffer) { + return Blob.wrap(((ByteBuffer) value).array()); + } else if (value instanceof BytesBuffer) { + return Blob.wrap(((BytesBuffer) value).bytes()); + } else if (value instanceof String) { + // Only base64 string or hex string accepted + String str = ((String) value); + if (str.startsWith("0x")) { + return Blob.wrap(Bytes.fromHex(str.substring(2))); + } + return Blob.wrap(StringEncoding.decodeBase64(str)); + } else if (value instanceof List) { + List values = (List) value; + byte[] bytes = new byte[values.size()]; + for (int i = 0; i < bytes.length; i++) { + Object v = values.get(i); + if (v instanceof Byte || v instanceof Integer) { + bytes[i] = ((Number) v).byteValue(); + } else { + throw new IllegalArgumentException(String.format( + "expect byte or int value, but got '%s'", v)); + } + } + return Blob.wrap(bytes); + } + return null; + } + + public static DataType fromClass(Class clazz) { + for (DataType type : DataType.values()) { + if (type.clazz() == clazz) { + return type; + } + } + throw new HugeException("Unknown clazz '%s' for DataType", clazz); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Directions.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Directions.java new file mode 100644 index 000000000..4c45990ab --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Directions.java @@ -0,0 +1,89 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +import org.apache.hugegraph.type.HugeType; + +public enum Directions implements SerialEnum { + + // TODO: add NONE enum for non-directional edges + + BOTH(0, "both"), + + OUT(1, "out"), + + IN(2, "in"); + + private byte code = 0; + private String name = null; + + static { + SerialEnum.register(Directions.class); + } + + Directions(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public HugeType type() { + switch (this) { + case OUT: + return HugeType.EDGE_OUT; + case IN: + return HugeType.EDGE_IN; + default: + throw new IllegalArgumentException(String.format( + "Can't convert direction '%s' to HugeType", this)); + } + } + + public Directions opposite() { + if (this.equals(OUT)) { + return IN; + } else { + return this.equals(IN) ? OUT : BOTH; + } + } + + + + public static Directions convert(HugeType edgeType) { + switch (edgeType) { + case EDGE_OUT: + return OUT; + case EDGE_IN: + return IN; + default: + throw new IllegalArgumentException(String.format( + "Can't convert type '%s' to Direction", edgeType)); + } + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/EdgeLabelType.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/EdgeLabelType.java new file mode 100644 index 000000000..7e90e7a24 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/EdgeLabelType.java @@ -0,0 +1,72 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +public enum EdgeLabelType implements SerialEnum { + + + NORMAL(1, "NORMAL"), + + PARENT(2, "PARENT"), + + SUB(3, "SUB"), + + GENERAL(4, "GENERAL"), + ; + + private final byte code; + private final String name; + + static { + SerialEnum.register(EdgeLabelType.class); + } + + EdgeLabelType(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public boolean normal() { + return this == NORMAL; + } + + public boolean parent() { + return this == PARENT; + } + + public boolean sub() { + return this == SUB; + } + + public boolean general() { + return this == GENERAL; + } + +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Frequency.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Frequency.java new file mode 100644 index 000000000..4ebe24867 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Frequency.java @@ -0,0 +1,51 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +public enum Frequency implements SerialEnum { + + DEFAULT(0, "default"), + + SINGLE(1, "single"), + + MULTIPLE(2, "multiple"); + + private byte code = 0; + private String name = null; + + static { + SerialEnum.register(Frequency.class); + } + + Frequency(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/HugeKeys.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/HugeKeys.java new file mode 100644 index 000000000..dc00972cb --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/HugeKeys.java @@ -0,0 +1,108 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +public enum HugeKeys { + + UNKNOWN(0, "undefined"), + + /* Column names of schema type (common) */ + ID(1, "id"), + NAME(2, "name"), + TIMESTAMP(3, "timestamp"), + SCHEMA_TYPE(4, "schema_type"), + + USER_DATA(10, "user_data"), + STATUS(11, "status"), + + /* Column names of schema type (VertexLabel) */ + ID_STRATEGY(50, "id_strategy"), + PROPERTIES(51, "properties"), + PRIMARY_KEYS(52, "primary_keys"), + INDEX_LABELS(53, "index_labels"), + NULLABLE_KEYS(54, "nullable_keys"), + ENABLE_LABEL_INDEX(55, "enable_label_index"), + + /* Column names of schema type (EdgeLabel) */ + LINKS(80, "links"), + FREQUENCY(81, "frequency"), + SOURCE_LABEL(82, "source_label"), + TARGET_LABEL(83, "target_label"), + SORT_KEYS(84, "sort_keys"), + TTL(85, "ttl"), + TTL_START_TIME(86, "ttl_start_time"), + EDGELABEL_TYPE(87, "edgelabel_type"), + PARENT_LABEL(89, "parent_label"), + + + /* Column names of schema type (PropertyKey) */ + DATA_TYPE(120, "data_type"), + CARDINALITY(121, "cardinality"), + AGGREGATE_TYPE(122, "aggregate_type"), + WRITE_TYPE(123, "write_type"), + + /* Column names of schema type (IndexLabel) */ + BASE_TYPE(150, "base_type"), + BASE_VALUE(151, "base_value"), + INDEX_TYPE(152, "index_type"), + FIELDS(153, "fields"), + + /* Column names of index data */ + INDEX_NAME(180, "index_name"), + FIELD_VALUES(181, "field_values"), + INDEX_LABEL_ID(182, "index_label_id"), + ELEMENT_IDS(183, "element_ids"), + + /* Column names of data type (Vertex/Edge) */ + LABEL(200, "label"), + OWNER_VERTEX(201, "owner_vertex"), + OTHER_VERTEX(202, "other_vertex"), + PROPERTY_KEY(203, "property_key"), + PROPERTY_VALUE(204, "property_value"), + DIRECTION(205, "direction"), + SORT_VALUES(206, "sort_values"), + PRIMARY_VALUES(207, "primary_values"), + EXPIRED_TIME(208, "expired_time"), + SUB_LABEL(211,"sub_label"), + + PROPERTY_TYPE(249, "property_type"), + AGGREGATE_PROPERTIES(250, "aggregate_properties"), + ; + + public static final long NORMAL_PROPERTY_ID = 0L; + + /* HugeKeys define */ + private byte code = 0; + private String name = null; + + HugeKeys(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/IdStrategy.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/IdStrategy.java new file mode 100644 index 000000000..4149c8db9 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/IdStrategy.java @@ -0,0 +1,71 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +public enum IdStrategy implements SerialEnum { + + DEFAULT(0, "default"), + + AUTOMATIC(1, "automatic"), + + PRIMARY_KEY(2, "primary_key"), + + CUSTOMIZE_STRING(3, "customize_string"), + + CUSTOMIZE_NUMBER(4, "customize_number"), + + CUSTOMIZE_UUID(5, "customize_uuid"); + + private byte code = 0; + private String name = null; + + static { + SerialEnum.register(IdStrategy.class); + } + + IdStrategy(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public boolean isAutomatic() { + return this == AUTOMATIC; + } + + public boolean isPrimaryKey() { + return this == PRIMARY_KEY; + } + + public boolean isCustomized() { + return this == CUSTOMIZE_STRING || + this == CUSTOMIZE_NUMBER || + this == CUSTOMIZE_UUID; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/IndexType.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/IndexType.java new file mode 100644 index 000000000..77e59932e --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/IndexType.java @@ -0,0 +1,122 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +import org.apache.hugegraph.type.HugeType; + +public enum IndexType implements SerialEnum { + + // For secondary query + SECONDARY(1, "secondary"), + + // For range query + RANGE(2, "range"), + RANGE_INT(21, "range_int"), + RANGE_FLOAT(22, "range_float"), + RANGE_LONG(23, "range_long"), + RANGE_DOUBLE(24, "range_double"), + + // For full-text query (not supported now) + SEARCH(3, "search"), + + // For prefix + range query + SHARD(4, "shard"), + + // For unique index + UNIQUE(5, "unique"); + + private byte code = 0; + private String name = null; + + static { + SerialEnum.register(IndexType.class); + } + + IndexType(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public HugeType type() { + switch (this) { + case SECONDARY: + return HugeType.SECONDARY_INDEX; + case RANGE_INT: + return HugeType.RANGE_INT_INDEX; + case RANGE_FLOAT: + return HugeType.RANGE_FLOAT_INDEX; + case RANGE_LONG: + return HugeType.RANGE_LONG_INDEX; + case RANGE_DOUBLE: + return HugeType.RANGE_DOUBLE_INDEX; + case SEARCH: + return HugeType.SEARCH_INDEX; + case SHARD: + return HugeType.SHARD_INDEX; + case UNIQUE: + return HugeType.UNIQUE_INDEX; + default: + throw new AssertionError(String.format( + "Unknown index type '%s'", this)); + } + } + + public boolean isString() { + return this == SECONDARY || this == SEARCH || + this == SHARD || this == UNIQUE; + } + + public boolean isNumeric() { + return this == RANGE_INT || this == RANGE_FLOAT || + this == RANGE_LONG || this == RANGE_DOUBLE || + this == SHARD; + } + + public boolean isSecondary() { + return this == SECONDARY; + } + + public boolean isRange() { + return this == RANGE_INT || this == RANGE_FLOAT || + this == RANGE_LONG || this == RANGE_DOUBLE; + } + + public boolean isSearch() { + return this == SEARCH; + } + + public boolean isShard() { + return this == SHARD; + } + + public boolean isUnique() { + return this == UNIQUE; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/SchemaStatus.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/SchemaStatus.java new file mode 100644 index 000000000..9222aa8ec --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/SchemaStatus.java @@ -0,0 +1,67 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +public enum SchemaStatus implements SerialEnum { + + CREATED(1, "created"), + + CREATING(2, "creating"), + + REBUILDING(3, "rebuilding"), + + DELETING(4, "deleting"), + + UNDELETED(5, "undeleted"), + + INVALID(6, "invalid"), + + CLEARING(7, "clearing"); + + private byte code = 0; + private String name = null; + + static { + SerialEnum.register(SchemaStatus.class); + } + + SchemaStatus(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + public boolean ok() { + return this == CREATED; + } + + public boolean deleting() { + return this == DELETING || this == UNDELETED; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/SerialEnum.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/SerialEnum.java new file mode 100644 index 000000000..337c981a7 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/SerialEnum.java @@ -0,0 +1,83 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.type.HugeType; + +import org.apache.hugegraph.util.CollectionUtil; +import org.apache.hugegraph.util.E; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public interface SerialEnum { + + public byte code(); + +// static Table, Byte, SerialEnum> table = HashBasedTable.create(); + + static Map>table =new ConcurrentHashMap<>(); + + public static void register(Class clazz) { + Object enums; + try { + enums = clazz.getMethod("values").invoke(null); + } catch (Exception e) { + throw new HugeException("Exception in backend", e); + } + ConcurrentHashMap map=new ConcurrentHashMap(); + for (SerialEnum e : CollectionUtil.toList(enums)) { + map.put(e.code(), e); + } + table.put(clazz,map); + } + + + public static T fromCode(Class clazz, byte code) { + Map clazzMap=table.get(clazz); + if (clazzMap == null) { + SerialEnum.register(clazz); + clazzMap=table.get(clazz); + } + E.checkArgument(clazzMap != null, "Can't get class registery for %s", + clazz.getSimpleName()); + T value = (T) clazzMap.get(code); + if (value == null) { + E.checkArgument(false, "Can't construct %s from code %s", + clazz.getSimpleName(), code); + } + return value; + } + + public static void registerInternalEnums() { + SerialEnum.register(Action.class); + SerialEnum.register(AggregateType.class); + SerialEnum.register(Cardinality.class); + SerialEnum.register(DataType.class); + SerialEnum.register(Directions.class); + SerialEnum.register(Frequency.class); + SerialEnum.register(HugeType.class); + SerialEnum.register(IdStrategy.class); + SerialEnum.register(IndexType.class); + SerialEnum.register(SchemaStatus.class); +// SerialEnum.register(HugePermission.class); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/WriteType.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/WriteType.java new file mode 100644 index 000000000..538b5bc40 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/WriteType.java @@ -0,0 +1,67 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.type.define; + +public enum WriteType implements SerialEnum { + + // OLTP property key + OLTP(1, "oltp"), + + // OLAP property key without index + OLAP_COMMON(2, "olap_common"), + + // OLAP property key with secondary index + OLAP_SECONDARY(3, "olap_secondary"), + + // OLAP property key with range index + OLAP_RANGE(4, "olap_range"); + + private final byte code; + private final String name; + + static { + SerialEnum.register(WriteType.class); + } + + WriteType(int code, String name) { + assert code < 256; + this.code = (byte) code; + this.name = name; + } + + @Override + public byte code() { + return this.code; + } + + public String string() { + return this.name; + } + + public boolean oltp() { + return this == OLTP; + } + + public boolean olap() { + return this == OLAP_COMMON || + this == OLAP_RANGE || + this == OLAP_SECONDARY; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/util/Blob.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/Blob.java new file mode 100644 index 000000000..03d82e916 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/Blob.java @@ -0,0 +1,73 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.util; + +import java.util.Arrays; + +import org.apache.hugegraph.util.Bytes; +import org.apache.hugegraph.util.E; + +public class Blob implements Comparable { + + public static final Blob EMPTY = new Blob(new byte[0]); + + private final byte[] bytes; + + private Blob(byte[] bytes) { + E.checkNotNull(bytes, "bytes"); + this.bytes = bytes; + } + + public byte[] bytes() { + return this.bytes; + } + + public static Blob wrap(byte[] bytes) { + return new Blob(bytes); + } + + @Override + public int hashCode() { + return Arrays.hashCode(this.bytes); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Blob)) { + return false; + } + Blob other = (Blob) obj; + return Arrays.equals(this.bytes, other.bytes); + } + + @Override + public String toString() { + String hex = Bytes.toHex(this.bytes); + StringBuilder sb = new StringBuilder(6 + hex.length()); + sb.append("Blob{").append(hex).append("}"); + return sb.toString(); + } + + @Override + public int compareTo(Blob other) { + E.checkNotNull(other, "other blob"); + return Bytes.compare(this.bytes, other.bytes); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/util/GraphUtils.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/GraphUtils.java new file mode 100644 index 000000000..b4f2d274a --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/GraphUtils.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.util; + +public class GraphUtils { + + private static final String HIDDEN_PREFIX = "~"; + + /** + * Determine if it is a system variable + * @param key + * @return + */ + public static boolean isHidden(final String key) { + return key.startsWith(HIDDEN_PREFIX); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/util/LZ4Util.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/LZ4Util.java new file mode 100644 index 000000000..98f23b9b2 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/LZ4Util.java @@ -0,0 +1,95 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.util; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import org.apache.hugegraph.exception.BackendException; +import org.apache.hugegraph.serializer.BytesBuffer; + +import net.jpountz.lz4.LZ4BlockInputStream; +import net.jpountz.lz4.LZ4BlockOutputStream; +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; + +public class LZ4Util { + + protected static final float DEFAULT_BUFFER_RATIO = 1.5f; + + public static BytesBuffer compress(byte[] bytes, int blockSize) { + return compress(bytes, blockSize, DEFAULT_BUFFER_RATIO); + } + + public static BytesBuffer compress(byte[] bytes, int blockSize, + float bufferRatio) { + float ratio = bufferRatio <= 0.0F ? DEFAULT_BUFFER_RATIO : bufferRatio; + LZ4Factory factory = LZ4Factory.fastestInstance(); + LZ4Compressor compressor = factory.fastCompressor(); + int initBufferSize = Math.round(bytes.length / ratio); + BytesBuffer buf = new BytesBuffer(initBufferSize); + LZ4BlockOutputStream lz4Output = new LZ4BlockOutputStream( + buf, blockSize, compressor); + try { + lz4Output.write(bytes); + lz4Output.close(); + } catch (IOException e) { + throw new BackendException("Failed to compress", e); + } + /* + * If need to perform reading outside the method, + * remember to call forReadWritten() + */ + return buf; + } + + public static BytesBuffer decompress(byte[] bytes, int blockSize) { + return decompress(bytes, blockSize, DEFAULT_BUFFER_RATIO); + } + + public static BytesBuffer decompress(byte[] bytes, int blockSize, + float bufferRatio) { + float ratio = bufferRatio <= 0.0F ? DEFAULT_BUFFER_RATIO : bufferRatio; + LZ4Factory factory = LZ4Factory.fastestInstance(); + LZ4FastDecompressor decompressor = factory.fastDecompressor(); + ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + int initBufferSize = Math.min(Math.round(bytes.length * ratio), + BytesBuffer.MAX_BUFFER_CAPACITY); + BytesBuffer buf = new BytesBuffer(initBufferSize); + LZ4BlockInputStream lzInput = new LZ4BlockInputStream(bais, + decompressor); + int count; + byte[] buffer = new byte[blockSize]; + try { + while ((count = lzInput.read(buffer)) != -1) { + buf.write(buffer, 0, count); + } + lzInput.close(); + } catch (IOException e) { + throw new BackendException("Failed to decompress", e); + } + /* + * If need to perform reading outside the method, + * remember to call forReadWritten() + */ + return buf; + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/util/StringEncoding.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/StringEncoding.java new file mode 100644 index 000000000..7e9ab6d8f --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/StringEncoding.java @@ -0,0 +1,203 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +// Copyright 2017 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.hugegraph.util; + +import java.io.UnsupportedEncodingException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.UUID; + +import org.apache.hugegraph.util.Bytes; +import org.apache.hugegraph.util.E; +import org.mindrot.jbcrypt.BCrypt; + +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.serializer.BytesBuffer; +import com.google.common.base.CharMatcher; + +/** + * @author Matthias Broecheler (me@matthiasb.com) + * @author HugeGraph Authors + */ +public final class StringEncoding { + + private static final MessageDigest DIGEST; + private static final byte[] BYTES_EMPTY = new byte[0]; + private static final int BLOCK_SIZE = 4096; + + static { + final String ALG = "SHA-256"; + try { + DIGEST = MessageDigest.getInstance(ALG); + } catch (NoSuchAlgorithmException e) { + throw new HugeException("Failed to load algorithm %s", e, ALG); + } + } + + private static final Base64.Encoder BASE64_ENCODER = Base64.getEncoder(); + private static final Base64.Decoder BASE64_DECODER = Base64.getDecoder(); + + // Similar to {@link StringSerializer} + public static int writeAsciiString(byte[] array, int offset, String value) { + E.checkArgument(CharMatcher.ascii().matchesAllOf(value), + "'%s' must be ASCII string", value); + int len = value.length(); + if (len == 0) { + array[offset++] = (byte) 0x80; + return offset; + } + + int i = 0; + do { + int c = value.charAt(i); + assert c <= 127; + byte b = (byte) c; + if (++i == len) { + b |= 0x80; // End marker + } + array[offset++] = b; + } while (i < len); + + return offset; + } + + public static String readAsciiString(byte[] array, int offset) { + StringBuilder sb = new StringBuilder(); + int c = 0; + do { + c = 0xFF & array[offset++]; + if (c != 0x80) { + sb.append((char) (c & 0x7F)); + } + } while ((c & 0x80) <= 0); + return sb.toString(); + } + + public static int getAsciiByteLength(String value) { + E.checkArgument(CharMatcher.ascii().matchesAllOf(value), + "'%s' must be ASCII string", value); + return value.isEmpty() ? 1 : value.length(); + } + + public static byte[] encode(String value) { + try { + return value.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new HugeException("Failed to encode string", e); + } + } + + public static String decode(byte[] bytes) { + try { + return new String(bytes, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new HugeException("Failed to decode string", e); + } + } + + public static String decode(byte[] bytes, int offset, int length) { + try { + return new String(bytes, offset, length, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new HugeException("Failed to decode string", e); + } + } + + public static String encodeBase64(byte[] bytes) { + return BASE64_ENCODER.encodeToString(bytes); + } + + public static byte[] decodeBase64(String value) { + if (value.isEmpty()) { + return BYTES_EMPTY; + } + return BASE64_DECODER.decode(value); + } + + public static byte[] compress(String value) { + return compress(value, LZ4Util.DEFAULT_BUFFER_RATIO); + } + + public static byte[] compress(String value, float bufferRatio) { + BytesBuffer buf = LZ4Util.compress(encode(value), BLOCK_SIZE, + bufferRatio); + return buf.bytes(); + } + + public static String decompress(byte[] value) { + return decompress(value, LZ4Util.DEFAULT_BUFFER_RATIO); + } + + public static String decompress(byte[] value, float bufferRatio) { + BytesBuffer buf = LZ4Util.decompress(value, BLOCK_SIZE, bufferRatio); + return decode(buf.array(), 0, buf.position()); + } + + public static String hashPassword(String password) { + return BCrypt.hashpw(password, BCrypt.gensalt(4)); + } + + public static boolean checkPassword(String candidatePassword, + String dbPassword) { + return BCrypt.checkpw(candidatePassword, dbPassword); + } + + public static String sha256(String string) { + byte[] stringBytes = encode(string); + DIGEST.reset(); + return StringEncoding.encodeBase64(DIGEST.digest(stringBytes)); + } + + public static String format(byte[] bytes) { + return String.format("%s[0x%s]", decode(bytes), Bytes.toHex(bytes)); + } + + public static UUID uuid(String value) { + E.checkArgument(value != null, "The UUID can't be null"); + try { + if (value.contains("-") && value.length() == 36) { + return UUID.fromString(value); + } + // UUID represented by hex string + E.checkArgument(value.length() == 32, + "Invalid UUID string: %s", value); + String high = value.substring(0, 16); + String low = value.substring(16); + return new UUID(Long.parseUnsignedLong(high, 16), + Long.parseUnsignedLong(low, 16)); + } catch (NumberFormatException ignored) { + throw new IllegalArgumentException("Invalid UUID string: " + value); + } + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/util/collection/CollectionFactory.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/collection/CollectionFactory.java new file mode 100644 index 000000000..fb42b8416 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/collection/CollectionFactory.java @@ -0,0 +1,264 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.util.collection; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hugegraph.util.E; +import org.eclipse.collections.api.map.primitive.IntObjectMap; +import org.eclipse.collections.api.map.primitive.MutableIntObjectMap; +import org.eclipse.collections.impl.list.mutable.FastList; +import org.eclipse.collections.impl.map.mutable.UnifiedMap; +import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; +import org.eclipse.collections.impl.set.mutable.UnifiedSet; + +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.type.define.CollectionType; + +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.ObjectArrayList; +import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; + +public class CollectionFactory { + + private final CollectionType type; + + public CollectionFactory() { + this.type = CollectionType.EC; + } + + public CollectionFactory(CollectionType type) { + this.type = type; + } + + public List newList() { + return newList(this.type); + } + + public List newList(int initialCapacity) { + return newList(this.type, initialCapacity); + } + + public List newList(Collection collection) { + return newList(this.type, collection); + } + + public static List newList(CollectionType type) { + switch (type) { + case EC: + return new FastList<>(); + case JCF: + return new ArrayList<>(); + case FU: + return new ObjectArrayList<>(); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static List newList(CollectionType type, + int initialCapacity) { + switch (type) { + case EC: + return new FastList<>(initialCapacity); + case JCF: + return new ArrayList<>(initialCapacity); + case FU: + return new ObjectArrayList<>(initialCapacity); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static List newList(CollectionType type, + Collection collection) { + switch (type) { + case EC: + return new FastList<>(collection); + case JCF: + return new ArrayList<>(collection); + case FU: + return new ObjectArrayList<>(collection); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public Set newSet() { + return newSet(this.type); + } + + public Set newSet(int initialCapacity) { + return newSet(this.type, initialCapacity); + } + + public Set newSet(Collection collection) { + return newSet(this.type, collection); + } + + public static Set newSet(CollectionType type) { + switch (type) { + case EC: + return new UnifiedSet<>(); + case JCF: + return new HashSet<>(); + case FU: + return new ObjectOpenHashSet<>(); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static Set newSet(CollectionType type, + int initialCapacity) { + switch (type) { + case EC: + return new UnifiedSet<>(initialCapacity); + case JCF: + return new HashSet<>(initialCapacity); + case FU: + return new ObjectOpenHashSet<>(initialCapacity); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static Set newSet(CollectionType type, + Collection collection) { + switch (type) { + case EC: + return new UnifiedSet<>(collection); + case JCF: + return new HashSet<>(collection); + case FU: + return new ObjectOpenHashSet<>(collection); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public Map newMap() { + return newMap(this.type); + } + + public Map newMap(int initialCapacity) { + return newMap(this.type, initialCapacity); + } + + public Map newMap(Map map) { + return newMap(this.type, map); + } + + public static Map newMap(CollectionType type) { + /* + * EC is faster 10%-20% than JCF, and it's more stable & less + * memory cost(size is bigger, EC is better). + */ + switch (type) { + case EC: + return new UnifiedMap<>(); + case JCF: + return new HashMap<>(); + case FU: + return new Object2ObjectOpenHashMap<>(); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static Map newMap(CollectionType type, + int initialCapacity) { + switch (type) { + case EC: + return new UnifiedMap<>(initialCapacity); + case JCF: + return new HashMap<>(initialCapacity); + case FU: + return new Object2ObjectOpenHashMap<>(initialCapacity); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static Map newMap(CollectionType type, + Map map) { + switch (type) { + case EC: + return new UnifiedMap<>(map); + case JCF: + return new HashMap<>(map); + case FU: + return new Object2ObjectOpenHashMap<>(map); + default: + throw new AssertionError( + "Unsupported collection type: " + type); + } + } + + public static MutableIntObjectMap newIntObjectMap() { + return new IntObjectHashMap<>(); + } + + public static MutableIntObjectMap newIntObjectMap(int initialCapacity) { + return new IntObjectHashMap<>(initialCapacity); + } + + public static MutableIntObjectMap newIntObjectMap( + IntObjectMap map) { + return new IntObjectHashMap<>(map); + } + + @SuppressWarnings("unchecked") + public static MutableIntObjectMap newIntObjectMap( + Object... objects) { + IntObjectHashMap map = IntObjectHashMap.newMap(); + E.checkArgument(objects.length % 2 == 0, + "Must provide even arguments for " + + "CollectionFactory.newIntObjectMap"); + for (int i = 0; i < objects.length; i += 2) { + int key = objects[i] instanceof Id ? + (int) ((Id) objects[i]).asLong() : (int) objects[i]; + map.put(key, (V) objects[i + 1]); + } + return map; + } + + public IdSet newIdSet() { + return newIdSet(this.type); + } + + public static IdSet newIdSet(CollectionType type) { + return new IdSet(type); + } +} diff --git a/hugegraph-struct/src/main/java/org/apache/hugegraph/util/collection/IdSet.java b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/collection/IdSet.java new file mode 100644 index 000000000..d77ddfb04 --- /dev/null +++ b/hugegraph-struct/src/main/java/org/apache/hugegraph/util/collection/IdSet.java @@ -0,0 +1,120 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.util.collection; + +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.type.define.CollectionType; + +import org.apache.hugegraph.iterator.ExtendableIterator; +import org.eclipse.collections.api.iterator.MutableLongIterator; +import org.eclipse.collections.impl.set.mutable.primitive.LongHashSet; + +import java.util.AbstractSet; +import java.util.Iterator; +import java.util.Set; + +public class IdSet extends AbstractSet { + + private final LongHashSet numberIds; + private final Set nonNumberIds; + + public IdSet(CollectionType type) { + this.numberIds = new LongHashSet(); + this.nonNumberIds = CollectionFactory.newSet(type); + } + + @Override + public int size() { + return this.numberIds.size() + this.nonNumberIds.size(); + } + + @Override + public boolean isEmpty() { + return this.numberIds.isEmpty() && this.nonNumberIds.isEmpty(); + } + + @Override + public boolean contains(Object object) { + if (!(object instanceof Id)) { + return false; + } + Id id = (Id) object; + if (id.type() == Id.IdType.LONG) { + return this.numberIds.contains(id.asLong()); + } else { + return this.nonNumberIds.contains(id); + } + } + + @Override + public Iterator iterator() { + return new ExtendableIterator<>( + this.nonNumberIds.iterator(), + new EcIdIterator(this.numberIds.longIterator())); + } + + @Override + public boolean add(Id id) { + if (id.type() == Id.IdType.LONG) { + return this.numberIds.add(id.asLong()); + } else { + return this.nonNumberIds.add(id); + } + } + + public boolean remove(Id id) { + if (id.type() == Id.IdType.LONG) { + return this.numberIds.remove(id.asLong()); + } else { + return this.nonNumberIds.remove(id); + } + } + + @Override + public void clear() { + this.numberIds.clear(); + this.nonNumberIds.clear(); + } + + private static class EcIdIterator implements Iterator { + + private final MutableLongIterator iterator; + + public EcIdIterator(MutableLongIterator iter) { + this.iterator = iter; + } + + @Override + public boolean hasNext() { + return this.iterator.hasNext(); + } + + @Override + public Id next() { + return IdGenerator.of(this.iterator.next()); + } + + @Override + public void remove() { + this.iterator.remove(); + } + } +} diff --git a/pom.xml b/pom.xml index 3b9d71ba0..ba53c0a36 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - 4.0.0 org.apache.hugegraph @@ -104,6 +104,7 @@ hugegraph-commons install-dist hugegraph-cluster-test + hugegraph-struct @@ -133,20 +134,28 @@ true - /org/codehaus/mojo/license/third-party-file-groupByMultiLicense.ftl + /org/codehaus/mojo/license/third-party-file-groupByMultiLicense.ftl + - The Apache Software License, Version 2.0|The Apache License, Version 2.0 - The Apache Software License, Version 2.0|Apache License, Version 2.0 - The Apache Software License, Version 2.0|Apache Public License 2.0 + The Apache Software License, Version 2.0|The Apache License, Version + 2.0 + + The Apache Software License, Version 2.0|Apache License, Version 2.0 + + The Apache Software License, Version 2.0|Apache Public License 2.0 + The Apache Software License, Version 2.0|Apache 2 The Apache Software License, Version 2.0|Apache 2.0 The Apache Software License, Version 2.0|Apache-2.0 - The Apache Software License, Version 2.0|Apache License 2.0 - The Apache Software License, Version 2.0|Apache License, version 2.0 + The Apache Software License, Version 2.0|Apache License 2.0 + + The Apache Software License, Version 2.0|Apache License, version 2.0 + 3-Clause BSD License|BSD 3-clause 3-Clause BSD License|BSD 3-Clause Eclipse Public License v1.0|Eclipse Public License 1.0 - Eclipse Public License v1.0|Eclipse Public License - v 1.0 + Eclipse Public License v1.0|Eclipse Public License - v 1.0 + The MIT License|MIT License