fix bug: remove secondary index of cassandra backend exceeding 65535 limit (#387)

fixed: #384

Change-Id: I3b0db265b621c149404a97ab688af8bd2e4ca4bd

README.md

@@ -1,8 +1,8 @@
 # HugeGraph
 
 [![License](https://img.shields.io/badge/license-Apache%202-0E78BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
-[![Build Status](https://travis-ci.org/hugegraph/hugegraph.svg?branch=master)](https://travis-ci.org/hugegraph/hugegraph)
-[![codecov](https://codecov.io/gh/hugegraph/hugegraph/branch/master/graph/badge.svg)](https://codecov.io/gh/hugegraph/hugegraph)
+[![Build Status](https://travis-ci.org/hugegraph/hugegraph.svg?branch=release-0.8)](https://travis-ci.org/hugegraph/hugegraph)
+[![codecov](https://codecov.io/gh/hugegraph/hugegraph/branch/release-0.8/graph/badge.svg)](https://codecov.io/gh/hugegraph/hugegraph)
 
 HugeGraph is a fast-speed and highly-scalable [graph database](https://en.wikipedia.org/wiki/Graph_database). Billions of vertices and edges can be easily stored into and queried from HugeGraph due to its excellent OLTP ability. As compliance to [Apache TinkerPop 3](https://tinkerpop.apache.org/) framework, various complicated graph queries can be accomplished through [Gremlin](https://tinkerpop.apache.org/gremlin.html)(a powerful graph traversal language).
 

hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTables.java

@@ -558,12 +558,18 @@ public class CassandraTables {
             }
 
             final String FIELD_VALUES = formatKey(HugeKeys.FIELD_VALUES);
+            int count = 0;
             for (Iterator<Row> it = rs.iterator(); it.hasNext();) {
                 fieldValues = it.next().get(FIELD_VALUES, String.class);
                 Delete delete = QueryBuilder.delete().from(this.table());
                 delete.where(formatEQ(HugeKeys.INDEX_LABEL_ID, indexLabel));
                 delete.where(formatEQ(HugeKeys.FIELD_VALUES, fieldValues));
                 session.add(delete);
+
+                if (++count >= COMMIT_DELETE_BATCH) {
+                    session.commit();
+                    count = 0;
+                }
             }
         }
 

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/id/IdGenerator.java

@@ -184,10 +184,10 @@ public abstract class IdGenerator {
 
         @Override
         public boolean equals(Object other) {
-            if (!(other instanceof LongId)) {
+            if (!(other instanceof Number)) {
                 return false;
             }
-            return this.id == ((LongId) other).id;
+            return this.id == ((Number) other).longValue();
         }
 
         @Override

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java

@@ -1324,7 +1324,7 @@ public class GraphTransaction extends IndexableTransaction {
             this.traverseVerticesByLabel(vertexLabel, vertex -> {
                 this.removeVertex((HugeVertex) vertex);
                 this.commitIfGtSize(COMMIT_BATCH);
-            });
+            }, true);
             this.commit();
         } catch (Exception e) {
             LOG.error("Failed to remove vertices", e);
@@ -1354,7 +1354,7 @@ public class GraphTransaction extends IndexableTransaction {
                 this.traverseEdgesByLabel(edgeLabel, edge -> {
                     this.removeEdge((HugeEdge) edge);
                     this.commitIfGtSize(COMMIT_BATCH);
-                });
+                }, true);
             }
             this.commit();
         } catch (Exception e) {
@@ -1366,18 +1366,19 @@ public class GraphTransaction extends IndexableTransaction {
     }
 
     public void traverseVerticesByLabel(VertexLabel label,
-                                        Consumer<Vertex> consumer) {
-        this.traverseByLabel(label, this::queryVertices, consumer);
+                                        Consumer<Vertex> consumer,
+                                        boolean remove) {
+        this.traverseByLabel(label, this::queryVertices, consumer, remove);
     }
 
-    public void traverseEdgesByLabel(EdgeLabel label,
-                                     Consumer<Edge> consumer) {
-        this.traverseByLabel(label, this::queryEdges, consumer);
+    public void traverseEdgesByLabel(EdgeLabel label, Consumer<Edge> consumer,
+                                     boolean remove) {
+        this.traverseByLabel(label, this::queryEdges, consumer, remove);
     }
 
     private <T> void traverseByLabel(SchemaLabel label,
                                      Function<Query, Iterator<T>> fetcher,
-                                     Consumer<T> consumer) {
+                                     Consumer<T> consumer, boolean remove) {
         HugeType type = label.type() == HugeType.VERTEX_LABEL ?
                         HugeType.VERTEX : HugeType.EDGE;
         ConditionQuery query = new ConditionQuery(type);
@@ -1405,12 +1406,14 @@ public class GraphTransaction extends IndexableTransaction {
          * Query.DEFAULT_CAPACITY to limit elements number per pass
          */
         query.limit(Query.DEFAULT_CAPACITY);
-        query.capacity(Query.DEFAULT_CAPACITY);
+        query.capacity(Query.NO_CAPACITY);
         query.eq(HugeKeys.LABEL, label.id());
         int pass = 0;
-        int counter = 0;
+        int counter;
         do {
-            query.offset(pass++ * Query.DEFAULT_CAPACITY);
+            if (!remove) {
+                query.offset(pass++ * Query.DEFAULT_CAPACITY);
+            }
             // Process every element in current batch
             Iterator<T> itor = fetcher.apply(query);
             for (counter = 0; itor.hasNext(); ++counter) {

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/IndexableTransaction.java

@@ -57,6 +57,15 @@ public abstract class IndexableTransaction extends AbstractTransaction {
         this.commitMutation2Backend(mutation, txMutation);
     }
 
+    @Override
+    public void commitIfGtSize(int size) throws BackendException {
+        int totalSize = this.mutationSize() +
+                        this.indexTransaction().mutationSize();
+        if (totalSize >= size) {
+            this.commit();
+        }
+    }
+
     @Override
     public void rollback() throws BackendException {
         try {

hugegraph-core/src/main/java/com/baidu/hugegraph/job/schema/RebuildIndexCallable.java

@@ -123,12 +123,14 @@ public class RebuildIndexCallable extends SchemaCallable {
             if (label.type() == HugeType.VERTEX_LABEL) {
                 @SuppressWarnings("unchecked")
                 Consumer<Vertex> consumer = (Consumer<Vertex>) indexUpdater;
-                graphTx.traverseVerticesByLabel((VertexLabel) label, consumer);
+                graphTx.traverseVerticesByLabel((VertexLabel) label,
+                                                consumer, false);
             } else {
                 assert label.type() == HugeType.EDGE_LABEL;
                 @SuppressWarnings("unchecked")
                 Consumer<Edge> consumer = (Consumer<Edge>) indexUpdater;
-                graphTx.traverseEdgesByLabel((EdgeLabel) label, consumer);
+                graphTx.traverseEdgesByLabel((EdgeLabel) label,
+                                             consumer, false);
             }
             graphTx.commit();
 

hugegraph-core/src/main/java/com/baidu/hugegraph/traversal/optimize/TraversalUtil.java

@@ -362,25 +362,31 @@ public final class TraversalUtil {
         assert bp instanceof Contains;
         List<?> values = (List<?>) has.getValue();
 
-        try {
-            String originKey = has.getKey();
-            if (values.size() > 1) {
-                E.checkArgument(!originKey.equals(T.key) &&
-                                !originKey.equals(T.value),
-                                "Not support hasKey() or hasValue() with " +
-                                "multiple values");
-            }
-            HugeKeys key = string2HugeKey(originKey);
-            values = convSysListValueIfNeeded(graph, type, key, values);
+        String originKey = has.getKey();
+        if (values.size() > 1) {
+            E.checkArgument(!originKey.equals(T.key) &&
+                            !originKey.equals(T.value),
+                            "Not support hasKey() or hasValue() with " +
+                            "multiple values");
+        }
 
+        HugeKeys hugeKey = null;
+        try {
+            hugeKey = string2HugeKey(originKey);
+        } catch (IllegalArgumentException ignored) {
+            // Ignore
+        }
+
+        if (hugeKey != null) {
+            values = convSysListValueIfNeeded(graph, type, hugeKey, values);
 
             switch ((Contains) bp) {
                 case within:
-                    return Condition.in(key, values);
+                    return Condition.in(hugeKey, values);
                 case without:
-                    return Condition.nin(key, values);
+                    return Condition.nin(hugeKey, values);
             }
-        } catch (IllegalArgumentException e) {
+        } else {
             String key = has.getKey();
             PropertyKey pkey = graph.propertyKey(key);
 

hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseSessions.java

@@ -479,11 +479,14 @@ public class HbaseSessions extends BackendSessionPool {
 
         public RowIterator(Result... results) {
             this.resultScanner = null;
-            if (results.length == 1 && results[0].isEmpty()) {
-                this.results = Collections.emptyIterator();
-            } else {
-                this.results = Arrays.asList(results).iterator();
+            List<Result> rs = new ArrayList<>(results.length);
+            for (Result result : results) {
+                // Get by Ids may return empty result
+                if (!result.isEmpty()) {
+                    rs.add(result);
+                }
             }
+            this.results = rs.iterator();
         }
 
         @Override

hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java

@@ -3302,6 +3302,101 @@ public class VertexCoreTest extends BaseCoreTest {
         });
     }
 
+    @Test
+    public void testQueryVerticesByIdsWithHasIdFilterAndNumberId() {
+        HugeGraph graph = graph();
+        SchemaManager schema = graph.schema();
+
+        schema.vertexLabel("user").useCustomizeNumberId().create();
+
+        graph.addVertex(T.label, "user", T.id, 123);
+        graph.addVertex(T.label, "user", T.id, 456);
+        graph.addVertex(T.label, "user", T.id, 789);
+        graph.tx().commit();
+
+        GraphTraversalSource g = graph.traversal();
+        List<Vertex> vertices;
+
+        vertices = g.V().hasId(P.within(123)).toList();
+        Assert.assertEquals(1, vertices.size());
+
+        vertices = g.V(123, 456).hasId(P.within(123)).toList();
+        Assert.assertEquals(1, vertices.size());
+
+        vertices = g.V(123, 456).hasId(123).toList();
+        Assert.assertEquals(1, vertices.size());
+
+        vertices = g.V(123, 456, 789).hasId(P.within(123, 456)).toList();
+        Assert.assertEquals(2, vertices.size());
+
+        vertices = g.V(123, 456, 789).hasId(456, 789).toList();
+        Assert.assertEquals(2, vertices.size());
+
+        vertices = g.V(123, 456, 789).hasId(P.within(123, 456, 789)).toList();
+        Assert.assertEquals(3, vertices.size());
+    }
+
+    @Test
+    public void testQueryVerticesByLabelsWithOneLabelNotExist() {
+        HugeGraph graph = graph();
+        SchemaManager schema = graph.schema();
+
+        schema.vertexLabel("user1").useCustomizeNumberId().create();
+        schema.vertexLabel("user2").useCustomizeNumberId().create();
+
+        graph.addVertex(T.label, "user1", T.id, 123);
+        graph.addVertex(T.label, "user2", T.id, 456);
+        graph.addVertex(T.label, "user2", T.id, 789);
+        graph.tx().commit();
+
+        GraphTraversalSource g = graph.traversal();
+        List<Vertex> vertices;
+
+        vertices = g.V().hasLabel("user1").toList();
+        Assert.assertEquals(1, vertices.size());
+
+        vertices = g.V().hasLabel("user2").toList();
+        Assert.assertEquals(2, vertices.size());
+
+        vertices = g.V().hasLabel("user1", "user2").toList();
+        Assert.assertEquals(3, vertices.size());
+
+        Assert.assertThrows(IllegalArgumentException.class, () -> {
+            g.V().hasLabel("user3").toList();
+        }, e -> {
+            Assert.assertEquals("Undefined vertex label: 'user3'",
+                                e.getMessage());
+        });
+
+        Assert.assertThrows(IllegalArgumentException.class, () -> {
+            g.V().hasLabel("user1", "user3").toList();
+        }, e -> {
+            Assert.assertEquals("Undefined vertex label: 'user3'",
+                                e.getMessage());
+        });
+
+        Assert.assertThrows(IllegalArgumentException.class, () -> {
+            g.V().hasLabel("user3", "user1").toList();
+        }, e -> {
+            Assert.assertEquals("Undefined vertex label: 'user3'",
+                                e.getMessage());
+        });
+
+        Assert.assertThrows(IllegalArgumentException.class, () -> {
+            g.V().hasLabel("user3", "user4").toList();
+        }, e -> {
+            Assert.assertEquals("Undefined vertex label: 'user3'",
+                                e.getMessage());
+        });
+
+        Assert.assertThrows(IllegalArgumentException.class, () -> {
+            g.V().hasLabel("user4", "user3").toList();
+        }, e -> {
+            Assert.assertEquals("Undefined vertex label: 'user4'",
+                                e.getMessage());
+        });
+    }
+
     private void init10Vertices() {
         HugeGraph graph = graph();