Compare commits

...

8 Commits

Author SHA1 Message Date
imbajin 5bf0290e54 fix: useless ring detection removed valid path (#1723)
* remove useless ring detection due to it will remove valid path
* limit max depth to 5000 to avoid stackoverflow
2022-01-12 22:42:10 +08:00
Jermy Li 35cffbc032 improve raft module and test (#1721)
* improve raft moddule
* add raft-server test
* set safe_read=true and use_snapshot=false
* fix leader not wait for apply-task
* add auth support to raft-tools.sh
* don't clear non-shared-storage backend
* set task timer interval from 3s to 1s
* improve CachedSchemaTransaction
* improve update schema status
* remove truncate from project test
* fix codecov: api test report not been uploaded
* improve task cancel() test with cancelled status

Change-Id: I016d3fcc4ab50614afdf452e7c9691ee3cc3c70b
2022-01-12 22:42:09 +08:00
dependabot[bot] df0a7b5dda Bump log4j-api from 2.17.0 to 2.17.1 (#1724)
Bumps log4j-api from 2.17.0 to 2.17.1.

---
updated-dependencies:
- dependency-name: org.apache.logging.log4j:log4j-api
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-01-12 22:42:09 +08:00
Jermy Li 35f78f4788 enhance schema job module (#1729)
Change-Id: I32938d4a5ed595ceb0f2dc02eb7ee75141f4133a
2022-01-12 22:42:09 +08:00
Jermy Li 3e80460a0e support hot set trace through /exception/trace API (#1727)
Change-Id: I78bbff56420be0a7c99dfc461913ed45d66725fc
2022-01-12 22:42:09 +08:00
Jermy Li 5e1efedb93 fix NoSuchMethodError Relations() (#1720)
Change-Id: I25b50a8d529d0292d700aaac0f63b4722a99160f
2022-01-12 22:42:09 +08:00
zhoney 2f726c9a75 upgrade rpc and common version to 2.0.1 (#1719)
Change-Id: If90b56013b04f1e72de18dcd733471ef42955f98
2022-01-12 22:42:09 +08:00
Jermy Li f17a68a510
HugeGraph-1358: update readme for release-0.12 (#1717)
Change-Id: I3e71ec79e926d0ba09aa6dc4bcb596ff71c15390
2021-12-31 15:09:58 +08:00
50 changed files with 979 additions and 317 deletions

View File

@ -15,6 +15,7 @@ jobs:
runs-on: ubuntu-20.04
env:
TRAVIS_DIR: hugegraph-dist/src/assembly/travis
REPORT_DIR: target/site/jacoco
BACKEND: ${{ matrix.BACKEND }}
TRIGGER_BRANCH_NAME: ${{ github.ref_name }}
HEAD_BRANCH_NAME: ${{ github.head_ref }}
@ -72,9 +73,14 @@ jobs:
- name: Run test
run: |
$TRAVIS_DIR/run-core-test.sh $BACKEND
$TRAVIS_DIR/run-api-test.sh $BACKEND
$TRAVIS_DIR/run-api-test.sh $BACKEND $REPORT_DIR
$TRAVIS_DIR/run-unit-test.sh $BACKEND
- name: Run Raft test
if: ${{ env.BACKEND == 'rocksdb' }}
run: |
$TRAVIS_DIR/run-api-test-for-raft.sh $BACKEND $REPORT_DIR
- name: Run TinkerPop test
if: ${{ env.RELEASE_BRANCH == 'true' }}
run: |
@ -83,4 +89,4 @@ jobs:
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v1
with:
file: target/site/jacoco/jacoco.xml
file: ${{ env.REPORT_DIR }}/*.xml

View File

@ -1,9 +1,9 @@
# HugeGraph
[![License](https://img.shields.io/badge/license-Apache%202-0E78BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
[![Build Status](https://github.com/hugegraph/hugegraph/actions/workflows/ci.yml/badge.svg)](https://github.com/hugegraph/hugegraph/actions/workflows/ci.yml)
[![Codecov](https://codecov.io/gh/hugegraph/hugegraph/branch/master/graph/badge.svg)](https://codecov.io/gh/hugegraph/hugegraph)
[![Github Releases Downloads](https://img.shields.io/github/downloads/hugegraph/hugegraph/total.svg)](https://github.com/hugegraph/hugegraph/releases)
[![Build Status](https://github.com/hugegraph/hugegraph/actions/workflows/ci.yml/badge.svg?branch=release-0.12)](https://github.com/hugegraph/hugegraph/actions/workflows/ci.yml)
[![Codecov](https://codecov.io/gh/hugegraph/hugegraph/branch/release-0.12/graph/badge.svg)](https://codecov.io/gh/hugegraph/hugegraph)
[![Github Releases Downloads](https://img.shields.io/github/downloads/hugegraph/hugegraph/v0.12.0/total.svg)](https://github.com/hugegraph/hugegraph/releases)
HugeGraph is a fast-speed and highly-scalable [graph database](https://en.wikipedia.org/wiki/Graph_database). Billions of vertices and edges can be easily stored into and queried from HugeGraph due to its excellent OLTP ability. As compliance to [Apache TinkerPop 3](https://tinkerpop.apache.org/) framework, various complicated graph queries can be accomplished through [Gremlin](https://tinkerpop.apache.org/gremlin.html)(a powerful graph traversal language).

View File

@ -21,7 +21,7 @@
<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-rpc</artifactId>
<version>1.0.0</version>
<version>2.0.1</version>
<exclusions>
<!-- conflict with jraft -->
<exclusion>

View File

@ -23,9 +23,16 @@ import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import javax.annotation.security.RolesAllowed;
import javax.inject.Singleton;
import javax.json.Json;
import javax.json.JsonArrayBuilder;
import javax.json.JsonObjectBuilder;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
@ -38,10 +45,13 @@ import org.apache.commons.lang3.StringUtils;
import org.glassfish.hk2.api.MultiException;
import com.baidu.hugegraph.HugeException;
import com.baidu.hugegraph.api.API;
import com.baidu.hugegraph.config.HugeConfig;
import com.baidu.hugegraph.config.ServerOptions;
import com.baidu.hugegraph.exception.HugeGremlinException;
import com.baidu.hugegraph.exception.NotFoundException;
import com.codahale.metrics.annotation.Timed;
import com.google.common.collect.ImmutableMap;
public class ExceptionFilter {
@ -52,12 +62,17 @@ public class ExceptionFilter {
private static final int INTERNAL_SERVER_ERROR =
Response.Status.INTERNAL_SERVER_ERROR.getStatusCode();
public static class TracedExceptionMapper {
public static class TracedExceptionMapper extends API {
private static boolean forcedTrace = false;
@Context
private javax.inject.Provider<HugeConfig> configProvider;
protected boolean trace() {
if (forcedTrace) {
return true;
}
HugeConfig config = this.configProvider.get();
if (config == null) {
return false;
@ -66,54 +81,81 @@ public class ExceptionFilter {
}
}
@Path("exception/trace")
@Singleton
public static class TracedExceptionAPI extends API {
@GET
@Timed
@Produces(APPLICATION_JSON_WITH_CHARSET)
@RolesAllowed({"admin"})
public Object get() {
return ImmutableMap.of("trace", TracedExceptionMapper.forcedTrace);
}
@PUT
@Timed
@Consumes(APPLICATION_JSON)
@Produces(APPLICATION_JSON_WITH_CHARSET)
@RolesAllowed({"admin"})
public Object trace(boolean trace) {
TracedExceptionMapper.forcedTrace = trace;
return ImmutableMap.of("trace", TracedExceptionMapper.forcedTrace);
}
}
@Provider
public static class HugeExceptionMapper
extends TracedExceptionMapper
implements ExceptionMapper<HugeException> {
@Override
public Response toResponse(HugeException exception) {
return Response.status(BAD_REQUEST_ERROR)
.type(MediaType.APPLICATION_JSON)
.entity(formatException(exception))
.entity(formatException(exception, this.trace()))
.build();
}
}
@Provider
public static class IllegalArgumentExceptionMapper
extends TracedExceptionMapper
implements ExceptionMapper<IllegalArgumentException> {
@Override
public Response toResponse(IllegalArgumentException exception) {
return Response.status(BAD_REQUEST_ERROR)
.type(MediaType.APPLICATION_JSON)
.entity(formatException(exception))
.entity(formatException(exception, this.trace()))
.build();
}
}
@Provider
public static class NotFoundExceptionMapper
extends TracedExceptionMapper
implements ExceptionMapper<NotFoundException> {
@Override
public Response toResponse(NotFoundException exception) {
return Response.status(NOT_FOUND_ERROR)
.type(MediaType.APPLICATION_JSON)
.entity(formatException(exception))
.entity(formatException(exception, this.trace()))
.build();
}
}
@Provider
public static class NoSuchElementExceptionMapper
extends TracedExceptionMapper
implements ExceptionMapper<NoSuchElementException> {
@Override
public Response toResponse(NoSuchElementException exception) {
return Response.status(NOT_FOUND_ERROR)
.type(MediaType.APPLICATION_JSON)
.entity(formatException(exception))
.entity(formatException(exception, this.trace()))
.build();
}
}
@ -189,10 +231,6 @@ public class ExceptionFilter {
}
}
public static String formatException(Throwable exception) {
return formatException(exception, false);
}
public static String formatException(Throwable exception, boolean trace) {
String clazz = exception.getClass().toString();
String message = exception.getMessage() != null ?

View File

@ -163,7 +163,7 @@ public class TaskAPI extends API {
HugeTask<?> task = scheduler.task(IdGenerator.of(id));
if (!task.completed() && !task.cancelling()) {
scheduler.cancel(task);
if (task.cancelling()) {
if (task.cancelling() || task.cancelled()) {
return task.asMap();
}
}

View File

@ -68,7 +68,7 @@ public class NeighborRankAPI extends API {
"The source of rank request can't be null");
E.checkArgument(request.steps != null && !request.steps.isEmpty(),
"The steps of rank request can't be empty");
E.checkArgument(request.steps.size() <= Long.parseLong(DEFAULT_MAX_DEPTH),
E.checkArgument(request.steps.size() <= DEFAULT_MAX_DEPTH,
"The steps length of rank request can't exceed %s",
DEFAULT_MAX_DEPTH);
E.checkArgument(request.alpha > 0 && request.alpha <= 1.0,

View File

@ -21,6 +21,7 @@ package com.baidu.hugegraph.api.traversers;
import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY;
import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEPTH;
import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT;
import java.util.Collection;
@ -114,9 +115,9 @@ public class PathsAPI extends TraverserAPI {
"The targets of request can't be null");
E.checkArgumentNotNull(request.step,
"The step of request can't be null");
E.checkArgument(request.depth > 0,
"The depth of request must be > 0, but got: %s",
request.depth);
E.checkArgument(request.depth > 0 && request.depth <= DEFAULT_MAX_DEPTH,
"The depth of request must be in (0, %s], " +
"but got: %s", DEFAULT_MAX_DEPTH, request.depth);
LOG.debug("Graph [{}] get paths from source vertices '{}', target " +
"vertices '{}', with step '{}', max depth '{}', " +

View File

@ -82,7 +82,7 @@ public class PersonalRankAPI extends API {
"The limit of rank request must be > 0 or == -1, " +
"but got: %s", request.limit);
E.checkArgument(request.maxDepth > 1L &&
request.maxDepth <= Long.parseLong(DEFAULT_MAX_DEPTH),
request.maxDepth <= DEFAULT_MAX_DEPTH,
"The max depth of rank request must be " +
"in range (1, %s], but got '%s'",
DEFAULT_MAX_DEPTH, request.maxDepth);

View File

@ -19,7 +19,7 @@
<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-common</artifactId>
<version>2.0.0</version>
<version>2.0.1</version>
</dependency>
<!-- tinkerpop -->
@ -54,7 +54,7 @@
<dependency>
<groupId>com.alipay.sofa</groupId>
<artifactId>jraft-core</artifactId>
<version>1.3.5</version>
<version>1.3.9</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

View File

@ -111,20 +111,7 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
event.checkArgs(String.class, HugeType.class, Id.class);
HugeType type = (HugeType) args[1];
Id id = (Id) args[2];
this.arrayCaches.remove(type, id);
id = generateId(type, id);
Object value = this.idCache.get(id);
if (value != null) {
// Invalidate id cache
this.idCache.invalidate(id);
// Invalidate name cache
SchemaElement schema = (SchemaElement) value;
Id prefixedName = generateId(schema.type(),
schema.name());
this.nameCache.invalidate(prefixedName);
}
this.invalidateCache(type, id);
this.resetCachedAll(type);
return true;
} else if (Cache.ACTION_CLEAR.equals(args[0])) {
@ -140,21 +127,6 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
}
}
private final void resetCachedAll(HugeType type) {
// Set the cache all flag of the schema type to false
this.cachedTypes().put(type, false);
}
private void clearCache(boolean notify) {
this.idCache.clear();
this.nameCache.clear();
this.arrayCaches.clear();
if (notify) {
this.notifyChanges(Cache.ACTION_CLEARED, null, null);
}
}
private void unlistenChanges() {
// Unlisten store event
this.store().provider().unlisten(this.storeEventListener);
@ -164,11 +136,16 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
schemaEventHub.unlisten(Events.CACHE, this.cacheEventListener);
}
private void notifyChanges(String action, HugeType type, Id id) {
private final void notifyChanges(String action, HugeType type, Id id) {
EventHub graphEventHub = this.params().schemaEventHub();
graphEventHub.notify(Events.CACHE, action, type, id);
}
private final void resetCachedAll(HugeType type) {
// Set the cache all flag of the schema type to false
this.cachedTypes().put(type, false);
}
private final void resetCachedAllIfReachedCapacity() {
if (this.idCache.size() >= this.idCache.capacity()) {
LOG.warn("Schema cache reached capacity({}): {}",
@ -181,6 +158,47 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
return this.arrayCaches.cachedTypes();
}
private final void clearCache(boolean notify) {
this.idCache.clear();
this.nameCache.clear();
this.arrayCaches.clear();
if (notify) {
this.notifyChanges(Cache.ACTION_CLEARED, null, null);
}
}
private final void updateCache(SchemaElement schema) {
this.resetCachedAllIfReachedCapacity();
// update id cache
Id prefixedId = generateId(schema.type(), schema.id());
this.idCache.update(prefixedId, schema);
// update name cache
Id prefixedName = generateId(schema.type(), schema.name());
this.nameCache.update(prefixedName, schema);
// update optimized array cache
this.arrayCaches.updateIfNeeded(schema);
}
private final void invalidateCache(HugeType type, Id id) {
// remove from id cache and name cache
Id prefixedId = generateId(type, id);
Object value = this.idCache.get(prefixedId);
if (value != null) {
this.idCache.invalidate(prefixedId);
SchemaElement schema = (SchemaElement) value;
Id prefixedName = generateId(schema.type(), schema.name());
this.nameCache.invalidate(prefixedName);
}
// remove from optimized array cache
this.arrayCaches.remove(type, id);
}
private static Id generateId(HugeType type, Id id) {
// NOTE: it's slower performance to use:
// String.format("%x-%s", type.code(), name)
@ -195,18 +213,7 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
protected void addSchema(SchemaElement schema) {
super.addSchema(schema);
this.resetCachedAllIfReachedCapacity();
// update id cache
Id prefixedId = generateId(schema.type(), schema.id());
this.idCache.update(prefixedId, schema);
// update name cache
Id prefixedName = generateId(schema.type(), schema.name());
this.nameCache.update(prefixedName, schema);
// update optimized array cache
this.arrayCaches.updateIfNeeded(schema);
this.updateCache(schema);
this.notifyChanges(Cache.ACTION_INVALIDED, schema.type(), schema.id());
}
@ -227,19 +234,15 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
if (value == null) {
value = super.getSchema(type, id);
if (value != null) {
this.resetCachedAllIfReachedCapacity();
this.idCache.update(prefixedId, value);
SchemaElement schema = (SchemaElement) value;
Id prefixedName = generateId(schema.type(), schema.name());
this.nameCache.update(prefixedName, schema);
// update id cache, name cache and optimized array cache
this.updateCache(schema);
}
} else {
// update optimized array cache for the result from id cache
this.arrayCaches.updateIfNeeded((SchemaElement) value);
}
// update optimized array cache
this.arrayCaches.updateIfNeeded((SchemaElement) value);
return (T) value;
}
@ -252,13 +255,8 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
if (value == null) {
value = super.getSchema(type, name);
if (value != null) {
this.resetCachedAllIfReachedCapacity();
this.nameCache.update(prefixedName, value);
SchemaElement schema = (SchemaElement) value;
Id prefixedId = generateId(schema.type(), schema.id());
this.idCache.update(prefixedId, schema);
this.updateCache(schema);
}
}
return (T) value;
@ -268,18 +266,7 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
protected void removeSchema(SchemaElement schema) {
super.removeSchema(schema);
Id prefixedId = generateId(schema.type(), schema.id());
Object value = this.idCache.get(prefixedId);
if (value != null) {
this.idCache.invalidate(prefixedId);
schema = (SchemaElement) value;
Id prefixedName = generateId(schema.type(), schema.name());
this.nameCache.invalidate(prefixedName);
}
// remove from optimized array cache
this.arrayCaches.remove(schema.type(), schema.id());
this.invalidateCache(schema.type(), schema.id());
this.notifyChanges(Cache.ACTION_INVALIDED, schema.type(), schema.id());
}
@ -299,16 +286,13 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
});
return results;
} else {
this.cachedTypes().remove(type);
List<T> results = super.getAllSchema(type);
long free = this.idCache.capacity() - this.idCache.size();
if (results.size() <= free) {
// Update cache
for (T schema : results) {
Id prefixedId = generateId(schema.type(), schema.id());
this.idCache.update(prefixedId, schema);
Id prefixedName = generateId(schema.type(), schema.name());
this.nameCache.update(prefixedName, schema);
this.updateCache(schema);
}
this.cachedTypes().putIfAbsent(type, true);
}

View File

@ -505,6 +505,10 @@ public final class ConditionQueryFlatten {
private static final long serialVersionUID = -2110811280408887334L;
public Relations() {
super();
}
public static Relations of(Relation... relations) {
Relations rs = new Relations();
rs.addAll(Arrays.asList(relations));

View File

@ -50,11 +50,13 @@ public class RaftBackendStore implements BackendStore {
private final BackendStore store;
private final RaftSharedContext context;
private final ThreadLocal<MutationBatch> mutationBatch;
private final boolean isSafeRead;
public RaftBackendStore(BackendStore store, RaftSharedContext context) {
this.store = store;
this.context = context;
this.mutationBatch = new ThreadLocal<>();
this.isSafeRead = this.context.isSafeRead();
}
public BackendStore originStore() {
@ -143,7 +145,8 @@ public class RaftBackendStore implements BackendStore {
@Override
public Number queryNumber(Query query) {
return (Number) this.queryByRaft(query, o -> this.store.queryNumber(query));
return (Number)
this.queryByRaft(query, o -> this.store.queryNumber(query));
}
@Override
@ -186,7 +189,10 @@ public class RaftBackendStore implements BackendStore {
@Override
public long getCounter(HugeType type) {
return (Long) this.queryByRaft(type, o -> this.store.getCounter(type));
Object counter = this.queryByRaft(type, true,
o -> this.store.getCounter(type));
assert counter instanceof Long;
return (Long) counter;
}
private Object submitAndWait(StoreAction action, byte[] data) {
@ -200,7 +206,12 @@ public class RaftBackendStore implements BackendStore {
}
private Object queryByRaft(Object query, Function<Object, Object> func) {
if (!this.context.isSafeRead()) {
return this.queryByRaft(query, this.isSafeRead, func);
}
private Object queryByRaft(Object query, boolean safeRead,
Function<Object, Object> func) {
if (!safeRead) {
return func.apply(query);
}
@ -212,8 +223,8 @@ public class RaftBackendStore implements BackendStore {
future.complete(status, () -> func.apply(query));
} else {
future.failure(status, new BackendException(
"Failed to execute query '%s' with read-index: %s",
query, status));
"Failed to do raft read-index: %s",
status));
}
}
};
@ -221,8 +232,8 @@ public class RaftBackendStore implements BackendStore {
try {
return future.waitFinished();
} catch (Throwable e) {
LOG.warn("Failed to execute query '{}' with read-index: {}",
query, future.status());
LOG.warn("Failed to execute query '{}': {}",
query, future.status(), e);
throw new BackendException("Failed to execute query: %s", e, query);
}
}

View File

@ -158,7 +158,7 @@ public class RaftBackendStoreProvider implements BackendStoreProvider {
for (RaftBackendStore store : this.stores()) {
store.init();
}
this.notifyAndWaitEvent(Events.STORE_INITED);
this.notifyAndWaitEvent(Events.STORE_INIT);
LOG.debug("Graph '{}' store has been initialized", this.graph());
}
@ -214,7 +214,8 @@ public class RaftBackendStoreProvider implements BackendStoreProvider {
@Override
public void createSnapshot() {
StoreCommand command = new StoreCommand(StoreType.ALL,
// TODO: snapshot for StoreType.ALL instead of StoreType.GRAPH
StoreCommand command = new StoreCommand(StoreType.GRAPH,
StoreAction.SNAPSHOT, null);
RaftStoreClosure closure = new RaftStoreClosure(command);
this.context.node().submitAndWait(command, closure);

View File

@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicReference;
import org.slf4j.Logger;
import com.alipay.sofa.jraft.Node;
import com.alipay.sofa.jraft.RaftGroupService;
import com.alipay.sofa.jraft.RaftServiceFactory;
import com.alipay.sofa.jraft.Status;
import com.alipay.sofa.jraft.closure.ReadIndexClosure;
import com.alipay.sofa.jraft.core.Replicator.ReplicatorStateListener;
@ -36,7 +36,6 @@ import com.alipay.sofa.jraft.entity.PeerId;
import com.alipay.sofa.jraft.entity.Task;
import com.alipay.sofa.jraft.error.RaftError;
import com.alipay.sofa.jraft.option.NodeOptions;
import com.alipay.sofa.jraft.rpc.RpcServer;
import com.alipay.sofa.jraft.util.BytesUtil;
import com.baidu.hugegraph.backend.BackendException;
import com.baidu.hugegraph.util.LZ4Util;
@ -58,6 +57,7 @@ public final class RaftNode {
this.stateMachine = new StoreStateMachine(context);
try {
this.node = this.initRaftNode();
LOG.info("Start raft node: {}", this);
} catch (IOException e) {
throw new BackendException("Failed to init raft node", e);
}
@ -94,6 +94,7 @@ public final class RaftNode {
}
public void shutdown() {
LOG.info("Shutdown raft node: {}", this);
this.node.shutdown();
}
@ -116,13 +117,14 @@ public final class RaftNode {
// TODO: When support sharding, groupId needs to be bound to shard Id
String groupId = this.context.group();
PeerId endpoint = this.context.endpoint();
RpcServer rpcServer = this.context.rpcServer();
RaftGroupService raftGroupService;
// Shared rpc server
raftGroupService = new RaftGroupService(groupId, endpoint, nodeOptions,
rpcServer, true);
// Start node
return raftGroupService.start(false);
/*
* Start raft node with shared rpc server:
* return new RaftGroupService(groupId, endpoint, nodeOptions,
* this.context.rpcServer(), true)
* .start(false)
*/
return RaftServiceFactory.createAndInitRaftNode(groupId, endpoint,
nodeOptions);
}
private void submitCommand(StoreCommand command, RaftStoreClosure closure) {

View File

@ -22,6 +22,7 @@ package com.baidu.hugegraph.backend.store.raft;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
@ -30,14 +31,15 @@ import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import com.alipay.sofa.jraft.option.ReadOnlyOption;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import com.alipay.sofa.jraft.NodeManager;
import com.alipay.sofa.jraft.conf.Configuration;
import com.alipay.sofa.jraft.entity.PeerId;
import com.alipay.sofa.jraft.option.NodeOptions;
import com.alipay.sofa.jraft.option.RaftOptions;
import com.alipay.sofa.jraft.option.ReadOnlyOption;
import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
import com.alipay.sofa.jraft.rpc.RpcServer;
import com.alipay.sofa.jraft.util.NamedThreadFactory;
@ -46,6 +48,9 @@ import com.baidu.hugegraph.HugeException;
import com.baidu.hugegraph.HugeGraphParams;
import com.baidu.hugegraph.backend.cache.Cache;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.store.BackendAction;
import com.baidu.hugegraph.backend.store.BackendMutation;
import com.baidu.hugegraph.backend.store.BackendStore;
import com.baidu.hugegraph.backend.store.raft.rpc.ListPeersProcessor;
import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType;
@ -101,7 +106,7 @@ public final class RaftSharedContext {
public RaftSharedContext(HugeGraphParams params) {
this.params = params;
HugeConfig config = params.configuration();
HugeConfig config = this.config();
this.schemaStoreName = config.get(CoreOptions.STORE_SCHEMA);
this.graphStoreName = config.get(CoreOptions.STORE_GRAPH);
@ -126,12 +131,7 @@ public final class RaftSharedContext {
this.raftGroupManager = null;
this.rpcForwarder = null;
this.registerRpcRequestProcessors();
}
private void registerRpcRequestProcessors() {
this.rpcServer.registerProcessor(new StoreCommandProcessor(this));
this.rpcServer.registerProcessor(new SetLeaderProcessor(this));
this.rpcServer.registerProcessor(new ListPeersProcessor(this));
LOG.info("Start raft server successfully: {}", this.endpoint());
}
public void initRaftNode() {
@ -149,8 +149,14 @@ public final class RaftSharedContext {
}
public void close() {
LOG.info("Stopping raft nodes");
this.rpcServer.shutdown();
LOG.info("Stop raft server: {}", this.endpoint());
RaftNode node = this.node();
if (node != null) {
node.shutdown();
}
this.shutdownRpcServer();
}
public RaftNode node() {
@ -168,10 +174,6 @@ public final class RaftSharedContext {
return this.raftGroupManager;
}
public RpcServer rpcServer() {
return this.rpcServer;
}
public String group() {
return DEFAULT_GROUP;
}
@ -266,12 +268,41 @@ public final class RaftSharedContext {
return nodeOptions;
}
public void clearCache() {
protected void clearCache() {
// Just choose two representatives used to represent schema and graph
this.notifyCache(Cache.ACTION_CLEAR, HugeType.VERTEX_LABEL, null);
this.notifyCache(Cache.ACTION_CLEAR, HugeType.VERTEX, null);
}
protected void updateCacheIfNeeded(BackendMutation mutation,
boolean forwarded) {
// Update cache only when graph run in general mode
if (this.graphMode() != GraphMode.NONE) {
return;
}
/*
* 1. If Follower, need to update cache from store to tx
* 3. If Leader, request is forwarded by follower, need to update cache
* 2. If Leader, request comes from leader, don't need to update cache,
* because the cache will be updated by upper layer
*/
if (!forwarded && this.node().selfIsLeader()) {
return;
}
for (HugeType type : mutation.types()) {
List<Id> ids = new ArrayList<>((int) Query.COMMIT_BATCH);
if (type.isSchema() || type.isGraph()) {
java.util.Iterator<BackendAction> it = mutation.mutation(type);
while (it.hasNext()) {
ids.add(it.next().entry().originId());
}
this.notifyCache(Cache.ACTION_INVALID, type, ids);
} else {
// Ignore other types due to not cached them
}
}
}
protected void notifyCache(String action, HugeType type, List<Id> ids) {
EventHub eventHub;
if (type.isGraph()) {
@ -340,14 +371,25 @@ public final class RaftSharedContext {
System.setProperty("bolt.channel_write_buf_high_water_mark",
String.valueOf(highWaterMark));
PeerId serverId = new PeerId();
serverId.parse(this.config().get(CoreOptions.RAFT_ENDPOINT));
PeerId endpoint = this.endpoint();
NodeManager.getInstance().addAddress(endpoint.getEndpoint());
RpcServer rpcServer = RaftRpcServerFactory.createAndStartRaftRpcServer(
serverId.getEndpoint());
LOG.info("RPC server is started successfully");
endpoint.getEndpoint());
return rpcServer;
}
private void shutdownRpcServer() {
this.rpcServer.shutdown();
PeerId endpoint = this.endpoint();
NodeManager.getInstance().removeAddress(endpoint.getEndpoint());
}
private void registerRpcRequestProcessors() {
this.rpcServer.registerProcessor(new StoreCommandProcessor(this));
this.rpcServer.registerProcessor(new SetLeaderProcessor(this));
this.rpcServer.registerProcessor(new ListPeersProcessor(this));
}
private ExecutorService createReadIndexExecutor(int coreThreads) {
int maxThreads = coreThreads << 2;
String name = "store-read-index-callback";

View File

@ -19,8 +19,10 @@
package com.baidu.hugegraph.backend.store.raft;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import org.slf4j.Logger;
@ -36,18 +38,12 @@ import com.alipay.sofa.jraft.error.RaftException;
import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
import com.baidu.hugegraph.backend.BackendException;
import com.baidu.hugegraph.backend.cache.Cache;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.serializer.BytesBuffer;
import com.baidu.hugegraph.backend.store.BackendAction;
import com.baidu.hugegraph.backend.store.BackendMutation;
import com.baidu.hugegraph.backend.store.BackendStore;
import com.baidu.hugegraph.backend.store.raft.RaftBackendStore.IncrCounter;
import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreAction;
import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.type.define.GraphMode;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.LZ4Util;
import com.baidu.hugegraph.util.Log;
@ -72,78 +68,24 @@ public final class StoreStateMachine extends StateMachineAdapter {
return this.context.node();
}
private void updateCacheIfNeeded(BackendMutation mutation,
boolean forwarded) {
// Update cache only when graph run in general mode
if (this.context.graphMode() != GraphMode.NONE) {
return;
}
/*
* 1. Follower need to update cache from store to tx
* 2. If request come from leader, cache will be updated by upper layer
* 3. If request is forwarded by follower, need to update cache
*/
if (!forwarded && this.node().selfIsLeader()) {
return;
}
for (HugeType type : mutation.types()) {
List<Id> ids = new ArrayList<>((int) Query.COMMIT_BATCH);
if (type.isSchema() || type.isGraph()) {
java.util.Iterator<BackendAction> it = mutation.mutation(type);
while (it.hasNext()) {
ids.add(it.next().entry().originId());
}
this.context.notifyCache(Cache.ACTION_INVALID, type, ids);
} else {
// Ignore other types due to not cached them
}
}
}
@Override
public void onApply(Iterator iter) {
LOG.debug("Node role: {}", this.node().selfIsLeader() ?
"leader" : "follower");
RaftStoreClosure closure = null;
List<Future<?>> futures = new ArrayList<>();
try {
// Apply all the logs
while (iter.hasNext()) {
closure = (RaftStoreClosure) iter.done();
RaftStoreClosure closure = (RaftStoreClosure) iter.done();
if (closure != null) {
// Leader just take it out from the closure
StoreCommand command = closure.command();
BytesBuffer buffer = BytesBuffer.wrap(command.data());
// The first two bytes are StoreType and StoreAction
StoreType type = StoreType.valueOf(buffer.read());
StoreAction action = StoreAction.valueOf(buffer.read());
boolean forwarded = command.forwarded();
// Let the producer thread to handle it
closure.complete(Status.OK(), () -> {
this.applyCommand(type, action, buffer, forwarded);
return null;
});
futures.add(this.onApplyLeader(closure));
} else {
// Follower need readMutation data
byte[] bytes = iter.getData().array();
// Let the backend thread do it directly
futures.add(this.context.backendExecutor().submit(() -> {
BytesBuffer buffer = LZ4Util.decompress(bytes,
RaftSharedContext.BLOCK_SIZE);
buffer.forReadWritten();
StoreType type = StoreType.valueOf(buffer.read());
StoreAction action = StoreAction.valueOf(buffer.read());
try {
this.applyCommand(type, action, buffer, false);
} catch (Throwable e) {
String title = "Failed to execute backend command";
LOG.error("{}: {}", title, action, e);
throw new BackendException(title, e);
}
}));
futures.add(this.onApplyFollower(iter.getData()));
}
iter.next();
}
// Follower wait tasks finished
// Wait for all tasks finished
for (Future<?> future : futures) {
future.get();
}
@ -152,17 +94,58 @@ public final class StoreStateMachine extends StateMachineAdapter {
LOG.error("{}", title, e);
Status status = new Status(RaftError.ESTATEMACHINE,
"%s: %s", title, e.getMessage());
if (closure != null) {
closure.failure(status, e);
}
// Will cause current node inactive
// TODO: rollback to correct index
iter.setErrorAndRollback(1L, status);
}
}
private void applyCommand(StoreType type, StoreAction action,
BytesBuffer buffer, boolean forwarded) {
private Future<?> onApplyLeader(RaftStoreClosure closure) {
// Leader just take the command out from the closure
StoreCommand command = closure.command();
BytesBuffer buffer = BytesBuffer.wrap(command.data());
// The first two bytes are StoreType and StoreAction
StoreType type = StoreType.valueOf(buffer.read());
StoreAction action = StoreAction.valueOf(buffer.read());
boolean forwarded = command.forwarded();
// Let the producer thread to handle it, and wait for it
CompletableFuture<Object> future = new CompletableFuture<>();
closure.complete(Status.OK(), () -> {
Object result;
try {
result = this.applyCommand(type, action, buffer, forwarded);
} catch (Throwable e) {
future.completeExceptionally(e);
throw e;
}
future.complete(result);
return result;
});
return future;
}
private Future<?> onApplyFollower(ByteBuffer data) {
// Follower need to read mutation data
byte[] bytes = data.array();
// Let the backend thread do it directly
return this.context.backendExecutor().submit(() -> {
BytesBuffer buffer = LZ4Util.decompress(bytes,
RaftSharedContext.BLOCK_SIZE);
buffer.forReadWritten();
StoreType type = StoreType.valueOf(buffer.read());
StoreAction action = StoreAction.valueOf(buffer.read());
try {
return this.applyCommand(type, action, buffer, false);
} catch (Throwable e) {
String title = "Failed to execute backend command";
LOG.error("{}: {}", title, action, e);
throw new BackendException(title, e);
}
});
}
private Object applyCommand(StoreType type, StoreAction action,
BytesBuffer buffer, boolean forwarded) {
E.checkState(type != StoreType.ALL,
"Can't apply command for all store at one time");
BackendStore store = this.store(type);
@ -190,7 +173,7 @@ public final class StoreStateMachine extends StateMachineAdapter {
store.beginTx();
for (BackendMutation mutation : mutations) {
store.mutate(mutation);
this.updateCacheIfNeeded(mutation, forwarded);
this.context.updateCacheIfNeeded(mutation, forwarded);
}
store.commitTx();
break;
@ -205,6 +188,7 @@ public final class StoreStateMachine extends StateMachineAdapter {
default:
throw new IllegalArgumentException("Invalid action " + action);
}
return null;
}
@Override

View File

@ -50,16 +50,20 @@ public class StoreCommandProcessor
@Override
public Message processRequest(StoreCommandRequest request,
RpcRequestClosure done) {
LOG.debug("Processing StoreCommandRequest");
LOG.debug("Processing StoreCommandRequest: {}", request.getAction());
RaftNode node = this.context.node();
try {
StoreCommand command = this.parseStoreCommand(request);
RaftStoreClosure closure = new RaftStoreClosure(command);
node.submitAndWait(command, closure);
// TODO: return the submitAndWait() result to rpc client
return StoreCommandResponse.newBuilder().setStatus(true).build();
} catch (Throwable e) {
StoreCommandResponse.Builder builder;
builder = StoreCommandResponse.newBuilder().setStatus(false);
LOG.warn("Failed to process StoreCommandRequest: {}",
request.getAction(), e);
StoreCommandResponse.Builder builder = StoreCommandResponse
.newBuilder()
.setStatus(false);
if (e.getMessage() != null) {
builder.setMessage(e.getMessage());
}

View File

@ -40,14 +40,14 @@ import com.baidu.hugegraph.backend.store.BackendStore;
import com.baidu.hugegraph.config.CoreOptions;
import com.baidu.hugegraph.exception.NotAllowException;
import com.baidu.hugegraph.job.JobBuilder;
import com.baidu.hugegraph.job.schema.EdgeLabelRemoveCallable;
import com.baidu.hugegraph.job.schema.IndexLabelRemoveCallable;
import com.baidu.hugegraph.job.schema.OlapPropertyKeyClearCallable;
import com.baidu.hugegraph.job.schema.OlapPropertyKeyCreateCallable;
import com.baidu.hugegraph.job.schema.OlapPropertyKeyRemoveCallable;
import com.baidu.hugegraph.job.schema.RebuildIndexCallable;
import com.baidu.hugegraph.job.schema.SchemaCallable;
import com.baidu.hugegraph.job.schema.VertexLabelRemoveCallable;
import com.baidu.hugegraph.job.schema.EdgeLabelRemoveJob;
import com.baidu.hugegraph.job.schema.IndexLabelRebuildJob;
import com.baidu.hugegraph.job.schema.IndexLabelRemoveJob;
import com.baidu.hugegraph.job.schema.OlapPropertyKeyClearJob;
import com.baidu.hugegraph.job.schema.OlapPropertyKeyCreateJob;
import com.baidu.hugegraph.job.schema.OlapPropertyKeyRemoveJob;
import com.baidu.hugegraph.job.schema.SchemaJob;
import com.baidu.hugegraph.job.schema.VertexLabelRemoveJob;
import com.baidu.hugegraph.perf.PerfUtil.Watched;
import com.baidu.hugegraph.schema.EdgeLabel;
import com.baidu.hugegraph.schema.IndexLabel;
@ -200,7 +200,7 @@ public class SchemaTransaction extends IndexableTransaction {
@Watched(prefix = "schema")
public Id removeVertexLabel(Id id) {
LOG.debug("SchemaTransaction remove vertex label '{}'", id);
SchemaCallable callable = new VertexLabelRemoveCallable();
SchemaJob callable = new VertexLabelRemoveJob();
VertexLabel schema = this.getVertexLabel(id);
return asyncRun(this.graph(), schema, callable);
}
@ -226,7 +226,7 @@ public class SchemaTransaction extends IndexableTransaction {
@Watched(prefix = "schema")
public Id removeEdgeLabel(Id id) {
LOG.debug("SchemaTransaction remove edge label '{}'", id);
SchemaCallable callable = new EdgeLabelRemoveCallable();
SchemaJob callable = new EdgeLabelRemoveJob();
EdgeLabel schema = this.getEdgeLabel(id);
return asyncRun(this.graph(), schema, callable);
}
@ -262,7 +262,7 @@ public class SchemaTransaction extends IndexableTransaction {
@Watched(prefix = "schema")
public Id removeIndexLabel(Id id) {
LOG.debug("SchemaTransaction remove index label '{}'", id);
SchemaCallable callable = new IndexLabelRemoveCallable();
SchemaJob callable = new IndexLabelRemoveJob();
IndexLabel schema = this.getIndexLabel(id);
return asyncRun(this.graph(), schema, callable);
}
@ -276,7 +276,7 @@ public class SchemaTransaction extends IndexableTransaction {
public Id rebuildIndex(SchemaElement schema, Set<Id> dependencies) {
LOG.debug("SchemaTransaction rebuild index for {} with id '{}'",
schema.type(), schema.id());
SchemaCallable callable = new RebuildIndexCallable();
SchemaJob callable = new IndexLabelRebuildJob();
return asyncRun(this.graph(), schema, callable, dependencies);
}
@ -305,26 +305,30 @@ public class SchemaTransaction extends IndexableTransaction {
public Id createOlapPk(PropertyKey propertyKey) {
LOG.debug("SchemaTransaction create olap property key {} with id '{}'",
propertyKey.name(), propertyKey.id());
SchemaCallable callable = new OlapPropertyKeyCreateCallable();
SchemaJob callable = new OlapPropertyKeyCreateJob();
return asyncRun(this.graph(), propertyKey, callable);
}
public Id clearOlapPk(PropertyKey propertyKey) {
LOG.debug("SchemaTransaction clear olap property key {} with id '{}'",
propertyKey.name(), propertyKey.id());
SchemaCallable callable = new OlapPropertyKeyClearCallable();
SchemaJob callable = new OlapPropertyKeyClearJob();
return asyncRun(this.graph(), propertyKey, callable);
}
public Id removeOlapPk(PropertyKey propertyKey) {
LOG.debug("SchemaTransaction remove olap property key {} with id '{}'",
propertyKey.name(), propertyKey.id());
SchemaCallable callable = new OlapPropertyKeyRemoveCallable();
SchemaJob callable = new OlapPropertyKeyRemoveJob();
return asyncRun(this.graph(), propertyKey, callable);
}
@Watched(prefix = "schema")
public void updateSchemaStatus(SchemaElement schema, SchemaStatus status) {
if (!this.existsSchemaId(schema.type(), schema.id())) {
LOG.warn("Can't update schema '{}', it may be deleted", schema);
return;
}
schema.status(status);
this.updateSchema(schema);
}
@ -543,17 +547,17 @@ public class SchemaTransaction extends IndexableTransaction {
}
private static Id asyncRun(HugeGraph graph, SchemaElement schema,
SchemaCallable callable) {
SchemaJob callable) {
return asyncRun(graph, schema, callable, ImmutableSet.of());
}
@Watched(prefix = "schema")
private static Id asyncRun(HugeGraph graph, SchemaElement schema,
SchemaCallable callable, Set<Id> dependencies) {
SchemaJob callable, Set<Id> dependencies) {
E.checkArgument(schema != null, "Schema can't be null");
String name = SchemaCallable.formatTaskName(schema.type(),
schema.id(),
schema.name());
String name = SchemaJob.formatTaskName(schema.type(),
schema.id(),
schema.name());
JobBuilder<Object> builder = JobBuilder.of(graph).name(name)
.job(callable)

View File

@ -30,7 +30,7 @@ import com.baidu.hugegraph.type.define.SchemaStatus;
import com.baidu.hugegraph.util.LockUtil;
import com.google.common.collect.ImmutableSet;
public class EdgeLabelRemoveCallable extends SchemaCallable {
public class EdgeLabelRemoveJob extends SchemaJob {
@Override
public String type() {
@ -43,7 +43,7 @@ public class EdgeLabelRemoveCallable extends SchemaCallable {
return null;
}
protected static void removeEdgeLabel(HugeGraphParams graph, Id id) {
private static void removeEdgeLabel(HugeGraphParams graph, Id id) {
GraphTransaction graphTx = graph.graphTransaction();
SchemaTransaction schemaTx = graph.schemaTransaction();
EdgeLabel edgeLabel = schemaTx.getEdgeLabel(id);
@ -51,7 +51,11 @@ public class EdgeLabelRemoveCallable extends SchemaCallable {
if (edgeLabel == null) {
return;
}
// TODO: use event to replace direct call
if (edgeLabel.status().deleting()) {
LOG.info("The edge label '{}' has been in {} status, " +
"please check if it's expected to delete it again",
edgeLabel, edgeLabel.status());
}
// Remove index related data(include schema) of this edge label
Set<Id> indexIds = ImmutableSet.copyOf(edgeLabel.indexLabels());
LockUtil.Locks locks = new LockUtil.Locks(graph.name());
@ -60,16 +64,18 @@ public class EdgeLabelRemoveCallable extends SchemaCallable {
schemaTx.updateSchemaStatus(edgeLabel, SchemaStatus.DELETING);
try {
for (Id indexId : indexIds) {
IndexLabelRemoveCallable.removeIndexLabel(graph, indexId);
IndexLabelRemoveJob.removeIndexLabel(graph, indexId);
}
// Remove all edges which has matched label
// TODO: use event to replace direct call
graphTx.removeEdges(edgeLabel);
removeSchema(schemaTx, edgeLabel);
/*
* Should commit changes to backend store before release
* delete lock
*/
graph.graph().tx().commit();
// Remove edge label
removeSchema(schemaTx, edgeLabel);
} catch (Throwable e) {
schemaTx.updateSchemaStatus(edgeLabel, SchemaStatus.UNDELETED);
throw e;

View File

@ -41,7 +41,7 @@ import com.baidu.hugegraph.type.define.SchemaStatus;
import com.baidu.hugegraph.util.LockUtil;
import com.google.common.collect.ImmutableSet;
public class RebuildIndexCallable extends SchemaCallable {
public class IndexLabelRebuildJob extends SchemaJob {
@Override
public String type() {

View File

@ -27,7 +27,7 @@ import com.baidu.hugegraph.schema.IndexLabel;
import com.baidu.hugegraph.type.define.SchemaStatus;
import com.baidu.hugegraph.util.LockUtil;
public class IndexLabelRemoveCallable extends SchemaCallable {
public class IndexLabelRemoveJob extends SchemaJob {
@Override
public String type() {
@ -48,6 +48,11 @@ public class IndexLabelRemoveCallable extends SchemaCallable {
if (indexLabel == null) {
return;
}
if (indexLabel.status().deleting()) {
LOG.info("The index label '{}' has been in {} status, " +
"please check if it's expected to delete it again",
indexLabel, indexLabel.status());
}
LockUtil.Locks locks = new LockUtil.Locks(graph.name());
try {
locks.lockWrites(LockUtil.INDEX_LABEL_DELETE, id);
@ -60,14 +65,15 @@ public class IndexLabelRemoveCallable extends SchemaCallable {
// Remove index data
// TODO: use event to replace direct call
graphTx.removeIndex(indexLabel);
removeSchema(schemaTx, indexLabel);
/*
* Should commit changes to backend store before release
* delete lock
*/
graph.graph().tx().commit();
// Remove index label
removeSchema(schemaTx, indexLabel);
} catch (Throwable e) {
schemaTx.updateSchemaStatus(indexLabel, SchemaStatus.INVALID);
schemaTx.updateSchemaStatus(indexLabel, SchemaStatus.UNDELETED);
throw e;
}
} finally {

View File

@ -27,7 +27,7 @@ import com.baidu.hugegraph.schema.IndexLabel;
import com.baidu.hugegraph.type.define.SchemaStatus;
import com.baidu.hugegraph.util.LockUtil;
public class OlapPropertyKeyClearCallable extends IndexLabelRemoveCallable {
public class OlapPropertyKeyClearJob extends IndexLabelRemoveJob {
@Override
public String type() {

View File

@ -22,7 +22,7 @@ package com.baidu.hugegraph.job.schema;
import com.baidu.hugegraph.backend.tx.SchemaTransaction;
import com.baidu.hugegraph.schema.PropertyKey;
public class OlapPropertyKeyCreateCallable extends SchemaCallable {
public class OlapPropertyKeyCreateJob extends SchemaJob {
@Override
public String type() {

View File

@ -23,8 +23,7 @@ import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.tx.SchemaTransaction;
import com.baidu.hugegraph.schema.PropertyKey;
public class OlapPropertyKeyRemoveCallable
extends OlapPropertyKeyClearCallable {
public class OlapPropertyKeyRemoveJob extends OlapPropertyKeyClearJob {
@Override
public String type() {

View File

@ -3,6 +3,8 @@ package com.baidu.hugegraph.job.schema;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import org.slf4j.Logger;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.id.IdGenerator;
import com.baidu.hugegraph.backend.tx.SchemaTransaction;
@ -13,8 +15,9 @@ import com.baidu.hugegraph.schema.SchemaLabel;
import com.baidu.hugegraph.schema.VertexLabel;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.Log;
public abstract class SchemaCallable extends SysJob<Object> {
public abstract class SchemaJob extends SysJob<Object> {
public static final String REMOVE_SCHEMA = "remove_schema";
public static final String REBUILD_INDEX = "rebuild_index";
@ -23,6 +26,8 @@ public abstract class SchemaCallable extends SysJob<Object> {
public static final String CLEAR_OLAP = "clear_olap";
public static final String REMOVE_OLAP = "remove_olap";
protected static final Logger LOG = Log.logger(SchemaJob.class);
private static final String SPLITOR = ":";
protected HugeType schemaType() {

View File

@ -33,7 +33,7 @@ import com.baidu.hugegraph.type.define.SchemaStatus;
import com.baidu.hugegraph.util.LockUtil;
import com.google.common.collect.ImmutableSet;
public class VertexLabelRemoveCallable extends SchemaCallable {
public class VertexLabelRemoveJob extends SchemaJob {
@Override
public String type() {
@ -54,7 +54,13 @@ public class VertexLabelRemoveCallable extends SchemaCallable {
if (vertexLabel == null) {
return;
}
if (vertexLabel.status().deleting()) {
LOG.info("The vertex label '{}' has been in {} status, " +
"please check if it's expected to delete it again",
vertexLabel, vertexLabel.status());
}
// Check no edge label use the vertex label
List<EdgeLabel> edgeLabels = schemaTx.getEdgeLabels();
for (EdgeLabel edgeLabel : edgeLabels) {
if (edgeLabel.linkWithLabel(id)) {
@ -76,19 +82,21 @@ public class VertexLabelRemoveCallable extends SchemaCallable {
schemaTx.updateSchemaStatus(vertexLabel, SchemaStatus.DELETING);
try {
for (Id ilId : indexLabelIds) {
IndexLabelRemoveCallable.removeIndexLabel(graph, ilId);
IndexLabelRemoveJob.removeIndexLabel(graph, ilId);
}
// TODO: use event to replace direct call
// Deleting a vertex will automatically deletes the held edge
graphTx.removeVertices(vertexLabel);
removeSchema(schemaTx, vertexLabel);
/*
* Should commit changes to backend store before release
* delete lock
*/
graph.graph().tx().commit();
// Remove vertex label
removeSchema(schemaTx, vertexLabel);
} catch (Throwable e) {
schemaTx.updateSchemaStatus(vertexLabel, SchemaStatus.UNDELETED);
schemaTx.updateSchemaStatus(vertexLabel,
SchemaStatus.UNDELETED);
throw e;
}
} finally {

View File

@ -50,7 +50,8 @@ public final class TaskManager {
"server-info-db-worker-%d";
public static final String TASK_SCHEDULER = "task-scheduler-%d";
protected static final int SCHEDULE_PERIOD = 3; // Unit second
protected static final int SCHEDULE_PERIOD = 1; // Unit second
private static final int THREADS = 4;
private static final TaskManager MANAGER = new TaskManager(THREADS);

View File

@ -82,8 +82,8 @@ public class HugeTraverser {
public static final String DEFAULT_MAX_DEGREE = "10000";
public static final String DEFAULT_SKIP_DEGREE = "100000";
public static final String DEFAULT_SAMPLE = "100";
public static final String DEFAULT_MAX_DEPTH = "50";
public static final String DEFAULT_WEIGHT = "0";
public static final int DEFAULT_MAX_DEPTH = 5000;
protected static final int MAX_VERTICES = 10;

View File

@ -20,7 +20,6 @@
package com.baidu.hugegraph.traversal.algorithm;
import java.util.Iterator;
import java.util.Objects;
import org.apache.tinkerpop.gremlin.structure.Edge;
@ -52,7 +51,9 @@ public class PathsTraverser extends HugeTraverser {
sourceDir == targetDir.opposite(),
"Source direction must equal to target direction" +
" or opposite to target direction");
checkPositive(depth, "max depth");
E.checkArgument(depth > 0 && depth <= DEFAULT_MAX_DEPTH,
"The depth must be in (0, %s], but got: %s",
DEFAULT_MAX_DEPTH, depth);
checkDegree(degree);
checkCapacity(capacity);
checkLimit(limit);
@ -124,9 +125,6 @@ public class PathsTraverser extends HugeTraverser {
PathSet results = this.record.findPath(target, null,
true, false);
for (Path path : results) {
if (Objects.equals(target, targetV)) {
continue;
}
this.paths.add(path);
if (this.reachLimit()) {
return;
@ -160,9 +158,6 @@ public class PathsTraverser extends HugeTraverser {
PathSet results = this.record.findPath(target, null,
true, false);
for (Path path : results) {
if (Objects.equals(target, sourceV)) {
continue;
}
this.paths.add(path);
if (this.reachLimit()) {
return;

View File

@ -14,6 +14,11 @@ LOG_PATH=${HOME_PATH}/logs
. ${BIN_PATH}/util.sh
#export HUGEGRAPH_URL=
#export HUGEGRAPH_GRAPH=
#export HUGEGRAPH_USERNAME=
#export HUGEGRAPH_PASSWORD=
function print_usage() {
echo "usage: raft-tools.sh [options]"
echo "options: "
@ -26,9 +31,6 @@ function print_usage() {
echo " -h,--help display help information"
}
GRAPH="hugegraph"
ENDPOINT=""
if [[ $# -lt 2 ]]; then
print_usage
exit 0
@ -36,56 +38,58 @@ fi
function list_peers() {
local graph=$1
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
local url=${rest_server_url}/graphs/${graph}/raft/list_peers
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/list_peers
curl ${url}
curl ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
}
function get_leader() {
local graph=$1
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
local url=${rest_server_url}/graphs/${graph}/raft/get_leader
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/get_leader
curl ${url}
curl ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
}
function set_leader() {
local graph=$1
local endpoint=$2
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
local url=${rest_server_url}/graphs/${graph}/raft/set_leader?endpoint=${endpoint}
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/set_leader?endpoint=${endpoint}
curl -X POST ${url}
curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
}
function transfer_leader() {
local graph=$1
local endpoint=$2
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
local url=${rest_server_url}/graphs/${graph}/raft/transfer_leader?endpoint=${endpoint}
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/transfer_leader?endpoint=${endpoint}
curl -X POST ${url}
curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
}
function add_peer() {
local graph=$1
local endpoint=$2
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
local url=${rest_server_url}/graphs/${graph}/raft/add_peer?endpoint=${endpoint}
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/add_peer?endpoint=${endpoint}
curl -X POST ${url}
curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
}
function remove_peer() {
local graph=$1
local endpoint=$2
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
local url=${rest_server_url}/graphs/${graph}/raft/remove_peer?endpoint=${endpoint}
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/remove_peer?endpoint=${endpoint}
curl -X POST ${url}
curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
}
if [ "${HUGEGRAPH_URL}" = "" ]; then
HUGEGRAPH_URL=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
fi
if [ "${HUGEGRAPH_GRAPH}" = "" ]; then
HUGEGRAPH_GRAPH="hugegraph"
fi
case $1 in
# help
--help|-h)

View File

@ -7,14 +7,15 @@ GC_OPTION=""
USER_OPTION=""
SERVER_STARTUP_TIMEOUT_S=30
while getopts "g:m:s:j:v" arg; do
while getopts "g:m:s:j:t:v" arg; do
case ${arg} in
g) GC_OPTION="$OPTARG" ;;
m) OPEN_MONITOR="$OPTARG" ;;
s) OPEN_SECURITY_CHECK="$OPTARG" ;;
j) USER_OPTION="$OPTARG" ;;
t) SERVER_STARTUP_TIMEOUT_S="$OPTARG" ;;
v) VERBOSE="verbose" ;;
?) echo "USAGE: $0 [-g g1] [-m true|false] [-s true|false] [-j xxx] [-v]" && exit 1 ;;
?) echo "USAGE: $0 [-g g1] [-m true|false] [-s true|false] [-j java_options] [-t timeout] [-v]" && exit 1 ;;
esac
done

View File

@ -136,7 +136,7 @@ function wait_for_startup() {
now_s=`date '+%s'`
done
echo "The operation timed out when attempting to connect to $server_url" >&2
echo "The operation timed out(${timeout_s}s) when attempting to connect to $server_url" >&2
return 1
}

View File

@ -3,6 +3,8 @@
set -ev
BACKEND=$1
JACOCO_PORT=$2
JACOCO_REPORT_FILE=$3
OPTION_CLASS_FILES_BACKEND="--classfiles hugegraph-$BACKEND/target/classes/com/baidu/hugegraph"
if [ "$BACKEND" == "memory" ]; then
@ -11,10 +13,10 @@ if [ "$BACKEND" == "memory" ]; then
fi
cd hugegraph-test
mvn jacoco:dump@pull-test-data -Dapp.host=localhost -Dapp.port=36320 -Dskip.dump=false
mvn jacoco:dump@pull-test-data -Dapp.host=localhost -Dapp.port=$JACOCO_PORT -Dskip.dump=false
cd ../
java -jar $TRAVIS_DIR/jacococli.jar report hugegraph-test/target/jacoco-it.exec \
--classfiles hugegraph-dist/target/classes/com/baidu/hugegraph \
--classfiles hugegraph-api/target/classes/com/baidu/hugegraph \
--classfiles hugegraph-core/target/classes/com/baidu/hugegraph \
$OPTION_CLASS_FILES_BACKEND --xml report.xml
${OPTION_CLASS_FILES_BACKEND} --xml "${JACOCO_REPORT_FILE}"

View File

@ -0,0 +1,29 @@
gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy
#gremlin.graph=com.baidu.hugegraph.HugeFactory
store=hugegraph
backend=rocksdb
serializer=binary
rocksdb.data_path=rocksdb-data-raft1
rocksdb.wal_path=rocksdb-data-raft1
raft.mode=true
raft.safe_read=true
raft.use_snapshot=false
raft.endpoint=127.0.0.1:8281
raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283
raft.path=rocksdb-raftlog1
raft.use_replicator_pipeline=true
raft.election_timeout=10000
raft.snapshot_interval=3600
raft.backend_threads=48
raft.read_index_threads=8
raft.read_strategy=ReadOnlyLeaseBased
raft.queue_size=16384
raft.queue_publish_timeout=60
raft.apply_batch=1
raft.rpc_threads=8
raft.rpc_connect_timeout=5000
raft.rpc_timeout=60000

View File

@ -0,0 +1,105 @@
# host and port of gremlin server, need to be consistent with host and port in rest-server.properties
#host: 127.0.0.1
port: 8181
# timeout in ms of gremlin query
scriptEvaluationTimeout: 30000
channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer
# don't set graph at here, this happens after support for dynamically adding graph
graphs: {
}
scriptEngines: {
gremlin-groovy: {
plugins: {
com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {},
org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {
classImports: [
java.lang.Math,
com.baidu.hugegraph.backend.id.IdGenerator,
com.baidu.hugegraph.type.define.Directions,
com.baidu.hugegraph.type.define.NodeRole,
com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser,
com.baidu.hugegraph.traversal.algorithm.CountTraverser,
com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser,
com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser,
com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser,
com.baidu.hugegraph.traversal.algorithm.HugeTraverser,
com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser,
com.baidu.hugegraph.traversal.algorithm.KneighborTraverser,
com.baidu.hugegraph.traversal.algorithm.KoutTraverser,
com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser,
com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser,
com.baidu.hugegraph.traversal.algorithm.PathsTraverser,
com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser,
com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser,
com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser,
com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser,
com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser,
com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser,
com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep,
com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep,
com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep,
com.baidu.hugegraph.traversal.optimize.Text,
com.baidu.hugegraph.traversal.optimize.TraversalUtil,
com.baidu.hugegraph.util.DateUtil
],
methodImports: [java.lang.Math#*]
},
org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {
files: [scripts/empty-sample.groovy]
}
}
}
}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
metrics: {
consoleReporter: {enabled: false, interval: 180000},
csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv},
jmxReporter: {enabled: false},
slf4jReporter: {enabled: false, interval: 180000},
gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST},
graphiteReporter: {enabled: false, interval: 180000}
}
maxInitialLineLength: 4096
maxHeaderSize: 8192
maxChunkSize: 8192
maxContentLength: 65536
maxAccumulationBufferComponents: 1024
resultIterationBatchSize: 64
writeBufferLowWaterMark: 32768
writeBufferHighWaterMark: 65536
ssl: {
enabled: false
}
authentication: {
authenticator: com.baidu.hugegraph.auth.StandardAuthenticator,
#authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler,
authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler,
config: {tokens: conf/rest-server.properties}
}

View File

@ -0,0 +1,11 @@
restserver.url=http://127.0.0.1:8080
gremlinserver.url=http://127.0.0.1:8181
graphs=conf/graphs
auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator
rpc.server_host=127.0.0.1
rpc.server_port=8091
rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093
server.id=server1
server.role=master

View File

@ -0,0 +1,29 @@
gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy
#gremlin.graph=com.baidu.hugegraph.HugeFactory
store=hugegraph
backend=rocksdb
serializer=binary
rocksdb.data_path=rocksdb-data-raft2
rocksdb.wal_path=rocksdb-data-raft2
raft.mode=true
raft.safe_read=true
raft.use_snapshot=false
raft.endpoint=127.0.0.1:8282
raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283
raft.path=rocksdb-raftlog2
raft.use_replicator_pipeline=true
raft.election_timeout=10000
raft.snapshot_interval=3600
raft.backend_threads=48
raft.read_index_threads=8
raft.read_strategy=ReadOnlyLeaseBased
raft.queue_size=16384
raft.queue_publish_timeout=60
raft.apply_batch=1
raft.rpc_threads=8
raft.rpc_connect_timeout=5000
raft.rpc_timeout=60000

View File

@ -0,0 +1,105 @@
# host and port of gremlin server
#host: 127.0.0.1
port: 8182
# timeout in ms of gremlin query
scriptEvaluationTimeout: 30000
channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer
# don't set graph at here, this happens after support for dynamically adding graph
graphs: {
}
scriptEngines: {
gremlin-groovy: {
plugins: {
com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {},
org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {
classImports: [
java.lang.Math,
com.baidu.hugegraph.backend.id.IdGenerator,
com.baidu.hugegraph.type.define.Directions,
com.baidu.hugegraph.type.define.NodeRole,
com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser,
com.baidu.hugegraph.traversal.algorithm.CountTraverser,
com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser,
com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser,
com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser,
com.baidu.hugegraph.traversal.algorithm.HugeTraverser,
com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser,
com.baidu.hugegraph.traversal.algorithm.KneighborTraverser,
com.baidu.hugegraph.traversal.algorithm.KoutTraverser,
com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser,
com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser,
com.baidu.hugegraph.traversal.algorithm.PathsTraverser,
com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser,
com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser,
com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser,
com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser,
com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser,
com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser,
com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep,
com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep,
com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep,
com.baidu.hugegraph.traversal.optimize.Text,
com.baidu.hugegraph.traversal.optimize.TraversalUtil,
com.baidu.hugegraph.util.DateUtil
],
methodImports: [java.lang.Math#*]
},
org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {
files: [scripts/empty-sample.groovy]
}
}
}
}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
metrics: {
consoleReporter: {enabled: false, interval: 180000},
csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv},
jmxReporter: {enabled: false},
slf4jReporter: {enabled: false, interval: 180000},
gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST},
graphiteReporter: {enabled: false, interval: 180000}
}
maxInitialLineLength: 4096
maxHeaderSize: 8192
maxChunkSize: 8192
maxContentLength: 65536
maxAccumulationBufferComponents: 1024
resultIterationBatchSize: 64
writeBufferLowWaterMark: 32768
writeBufferHighWaterMark: 65536
ssl: {
enabled: false
}
authentication: {
authenticator: com.baidu.hugegraph.auth.StandardAuthenticator,
#authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler,
authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler,
config: {tokens: conf/rest-server.properties}
}

View File

@ -0,0 +1,11 @@
restserver.url=http://127.0.0.1:8082
gremlinserver.url=http://127.0.0.1:8182
graphs=conf/graphs
auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator
rpc.server_host=127.0.0.1
rpc.server_port=8092
rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093
server.id=server2
server.role=worker

View File

@ -0,0 +1,29 @@
gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy
#gremlin.graph=com.baidu.hugegraph.HugeFactory
store=hugegraph
backend=rocksdb
serializer=binary
rocksdb.data_path=rocksdb-data-raft3
rocksdb.wal_path=rocksdb-data-raft3
raft.mode=true
raft.safe_read=true
raft.use_snapshot=false
raft.endpoint=127.0.0.1:8283
raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283
raft.path=rocksdb-raftlog3
raft.use_replicator_pipeline=true
raft.election_timeout=10000
raft.snapshot_interval=3600
raft.backend_threads=48
raft.read_index_threads=8
raft.read_strategy=ReadOnlyLeaseBased
raft.queue_size=16384
raft.queue_publish_timeout=60
raft.apply_batch=1
raft.rpc_threads=8
raft.rpc_connect_timeout=5000
raft.rpc_timeout=60000

View File

@ -0,0 +1,105 @@
# host and port of gremlin server
#host: 127.0.0.1
port: 8183
# timeout in ms of gremlin query
scriptEvaluationTimeout: 30000
channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer
# don't set graph at here, this happens after support for dynamically adding graph
graphs: {
}
scriptEngines: {
gremlin-groovy: {
plugins: {
com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {},
org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {
classImports: [
java.lang.Math,
com.baidu.hugegraph.backend.id.IdGenerator,
com.baidu.hugegraph.type.define.Directions,
com.baidu.hugegraph.type.define.NodeRole,
com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser,
com.baidu.hugegraph.traversal.algorithm.CountTraverser,
com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser,
com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser,
com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser,
com.baidu.hugegraph.traversal.algorithm.HugeTraverser,
com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser,
com.baidu.hugegraph.traversal.algorithm.KneighborTraverser,
com.baidu.hugegraph.traversal.algorithm.KoutTraverser,
com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser,
com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser,
com.baidu.hugegraph.traversal.algorithm.PathsTraverser,
com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser,
com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser,
com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser,
com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser,
com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser,
com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser,
com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep,
com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep,
com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep,
com.baidu.hugegraph.traversal.optimize.Text,
com.baidu.hugegraph.traversal.optimize.TraversalUtil,
com.baidu.hugegraph.util.DateUtil
],
methodImports: [java.lang.Math#*]
},
org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {
files: [scripts/empty-sample.groovy]
}
}
}
}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0,
config: {
serializeResultToString: false,
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
}
}
metrics: {
consoleReporter: {enabled: false, interval: 180000},
csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv},
jmxReporter: {enabled: false},
slf4jReporter: {enabled: false, interval: 180000},
gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST},
graphiteReporter: {enabled: false, interval: 180000}
}
maxInitialLineLength: 4096
maxHeaderSize: 8192
maxChunkSize: 8192
maxContentLength: 65536
maxAccumulationBufferComponents: 1024
resultIterationBatchSize: 64
writeBufferLowWaterMark: 32768
writeBufferHighWaterMark: 65536
ssl: {
enabled: false
}
authentication: {
authenticator: com.baidu.hugegraph.auth.StandardAuthenticator,
#authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler,
authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler,
config: {tokens: conf/rest-server.properties}
}

View File

@ -0,0 +1,11 @@
restserver.url=http://127.0.0.1:8083
gremlinserver.url=http://127.0.0.1:8183
graphs=conf/graphs
auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator
rpc.server_host=127.0.0.1
rpc.server_port=8093
rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093
server.id=server3
server.role=worker

View File

@ -0,0 +1,52 @@
#!/bin/bash
set -ev
BACKEND=$1
REPORT_DIR=$2
REPORT_FILE=$REPORT_DIR/jacoco-api-test.xml
TRAVIS_DIR=`dirname $0`
VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
SERVER_DIR=hugegraph-$VERSION
RAFT1_DIR=hugegraph-raft1
RAFT2_DIR=hugegraph-raft2
RAFT3_DIR=hugegraph-raft3
CONF=$SERVER_DIR/conf/graphs/hugegraph.properties
REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties
GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml
JACOCO_PORT=36320
RAFT_TOOLS=$RAFT1_DIR/bin/raft-tools.sh
RAFT_LEADER="127.0.0.1:8281"
mvn package -DskipTests
# mkdir for each raft-server
cp -r $SERVER_DIR $RAFT1_DIR
cp -r $SERVER_DIR $RAFT2_DIR
cp -r $SERVER_DIR $RAFT3_DIR
# config raft-server (must keep '/.')
cp -rf $TRAVIS_DIR/conf-raft1/. $RAFT1_DIR/conf/
cp -rf $TRAVIS_DIR/conf-raft2/. $RAFT2_DIR/conf/
cp -rf $TRAVIS_DIR/conf-raft3/. $RAFT3_DIR/conf/
# start server
$TRAVIS_DIR/start-server.sh $RAFT1_DIR $BACKEND $JACOCO_PORT || (cat $RAFT1_DIR/logs/hugegraph-server.log && exit 1) &
$TRAVIS_DIR/start-server.sh $RAFT2_DIR $BACKEND || (cat $RAFT2_DIR/logs/hugegraph-server.log && exit 1) &
$TRAVIS_DIR/start-server.sh $RAFT3_DIR $BACKEND || (cat $RAFT3_DIR/logs/hugegraph-server.log && exit 1)
export HUGEGRAPH_USERNAME=admin
export HUGEGRAPH_PASSWORD=pa
$RAFT_TOOLS --set-leader "hugegraph" "$RAFT_LEADER"
# run api-test
mvn test -P api-test,$BACKEND || (cat $RAFT1_DIR/logs/hugegraph-server.log && exit 1)
$TRAVIS_DIR/build-report.sh $BACKEND $JACOCO_PORT $REPORT_FILE
# stop server
$TRAVIS_DIR/stop-server.sh $RAFT1_DIR
$TRAVIS_DIR/stop-server.sh $RAFT2_DIR
$TRAVIS_DIR/stop-server.sh $RAFT3_DIR

View File

@ -3,6 +3,8 @@
set -ev
BACKEND=$1
REPORT_DIR=$2
REPORT_FILE=$REPORT_DIR/jacoco-api-test-for-raft.xml
TRAVIS_DIR=`dirname $0`
VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
@ -10,6 +12,7 @@ SERVER_DIR=hugegraph-$VERSION
CONF=$SERVER_DIR/conf/graphs/hugegraph.properties
REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties
GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml
JACOCO_PORT=36320
mvn package -DskipTests
@ -28,9 +31,13 @@ authentication: {
config: {tokens: conf/rest-server.properties}
}" >> $GREMLIN_SERVER_CONF
$TRAVIS_DIR/start-server.sh $SERVER_DIR $BACKEND || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1)
# start server
$TRAVIS_DIR/start-server.sh $SERVER_DIR $BACKEND $JACOCO_PORT || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1)
# run api-test
mvn test -P api-test,$BACKEND || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1)
$TRAVIS_DIR/build-report.sh $BACKEND
$TRAVIS_DIR/stop-server.sh
$TRAVIS_DIR/build-report.sh $BACKEND $JACOCO_PORT $REPORT_FILE
# stop server
$TRAVIS_DIR/stop-server.sh $SERVER_DIR

View File

@ -6,14 +6,21 @@ HOME_DIR=$(pwd)
TRAVIS_DIR=$(dirname $0)
BASE_DIR=$1
BACKEND=$2
JACOCO_PORT=$3
JACOCO_JAR=${HOME_DIR}/${TRAVIS_DIR}/jacocoagent.jar
BIN=$BASE_DIR/bin
CONF=$BASE_DIR/conf/graphs/hugegraph.properties
REST_CONF=$BASE_DIR/conf/rest-server.properties
GREMLIN_CONF=$BASE_DIR/conf/gremlin-server.yaml
declare -A backend_serializer_map=(["memory"]="text" ["cassandra"]="cassandra" \
["scylladb"]="scylladb" ["mysql"]="mysql" \
["hbase"]="hbase" ["rocksdb"]="binary" \
declare -A backend_serializer_map=(["memory"]="text" \
["cassandra"]="cassandra" \
["scylladb"]="scylladb" \
["mysql"]="mysql" \
["hbase"]="hbase" \
["rocksdb"]="binary" \
["postgresql"]="postgresql")
SERIALIZER=${backend_serializer_map[$BACKEND]}
@ -37,5 +44,9 @@ fi
# Append schema.sync_deletion=true to config file
echo "schema.sync_deletion=true" >> $CONF
AGENT_JAR=${HOME_DIR}/${TRAVIS_DIR}/jacocoagent.jar
echo -e "pa" | $BIN/init-store.sh && $BIN/start-hugegraph.sh -j "-javaagent:${AGENT_JAR}=includes=*,port=36320,destfile=jacoco-it.exec,output=tcpserver" -v
JACOCO_OPTION=""
if [ -n "$JACOCO_PORT" ]; then
JACOCO_OPTION="-javaagent:${JACOCO_JAR}=includes=*,port=${JACOCO_PORT},destfile=jacoco-it.exec,output=tcpserver"
fi
echo -e "pa" | $BIN/init-store.sh && $BIN/start-hugegraph.sh -j "$JACOCO_OPTION" -t 60 -v

View File

@ -2,8 +2,8 @@
set -ev
VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
BASE_DIR=hugegraph-$VERSION
BASE_DIR=$1
BIN=$BASE_DIR/bin
VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
$BIN/stop-hugegraph.sh

View File

@ -24,6 +24,7 @@ import java.util.Map;
import javax.ws.rs.core.Response;
import org.junit.Assume;
import org.junit.Test;
import com.baidu.hugegraph.testutil.Assert;
@ -86,12 +87,29 @@ public class GremlinApiTest extends BaseApiTest {
@Test
public void testClearAndInit() {
String body = "{"
+ "\"gremlin\":\"hugegraph.backendStoreFeatures()"
+ " .supportsSharedStorage();\","
+ "\"bindings\":{},"
+ "\"language\":\"gremlin-groovy\","
+ "\"aliases\":{\"g\":\"__g_hugegraph\"}}";
String content = assertResponseStatus(200, client().post(path, body));
Map<?, ?> result = assertJsonContains(content, "result");
@SuppressWarnings({ "unchecked" })
Object data = ((List<Object>) assertMapContains(result, "data")).get(0);
boolean supportsSharedStorage = (boolean) data;
Assume.assumeTrue("Can't clear non-shared-storage backend",
supportsSharedStorage);
body = "{"
+ "\"gremlin\":\""
+ "def auth = hugegraph.hugegraph().authManager();"
+ "def admin = auth.findUser('admin');"
+ "hugegraph.clearBackend();"
+ "hugegraph.initBackend();"
+ "auth.createUser(admin);\","
+ " if (!hugegraph.backendStoreFeatures()"
+ " .supportsSharedStorage())"
+ " return;"
+ " def auth = hugegraph.hugegraph().authManager();"
+ " def admin = auth.findUser('admin');"
+ " hugegraph.clearBackend();"
+ " hugegraph.initBackend();"
+ " auth.createUser(admin);\","
+ "\"bindings\":{},"
+ "\"language\":\"gremlin-groovy\","
+ "\"aliases\":{\"g\":\"__g_hugegraph\"}}";
@ -99,7 +117,7 @@ public class GremlinApiTest extends BaseApiTest {
body = "{"
+ "\"gremlin\":\"hugegraph.serverStarted("
+ "IdGenerator.of('server1'), NodeRole.MASTER)\","
+ " IdGenerator.of('server1'), NodeRole.MASTER)\","
+ "\"bindings\":{},"
+ "\"language\":\"gremlin-groovy\","
+ "\"aliases\":{\"g\":\"__g_hugegraph\"}}";

View File

@ -27,17 +27,48 @@ import javax.ws.rs.core.Response;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import com.baidu.hugegraph.util.JsonUtil;
import com.google.common.collect.ImmutableMap;
public class ProjectApiTest extends BaseApiTest {
private final static String path = "graphs/hugegraph/auth/projects";
@Before
public void setup() {
BaseApiTest.truncate();
@Override
@After
public void teardown() throws Exception {
Response resp = client().get(path);
String respBody = assertResponseStatus(200, resp);
List<?> projects = readList(respBody, "projects", Map.class);
for (Object project : projects) {
@SuppressWarnings("unchecked")
Map<String, Object> projectMap = ((Map<String, Object>) project);
String projectId = (String) projectMap.get("id");
// remove graphs from project if needed
List<?> projectGraphs = (List<?>) projectMap.get("project_graphs");
if (projectGraphs != null && projectGraphs.size() > 0) {
Map<String, Object> graphs = ImmutableMap.of("project_graphs",
projectGraphs);
resp = client().target()
.path(path)
.path(projectId)
.queryParam("action", "remove_graph")
.request()
.put(Entity.json(JsonUtil.toJson(graphs)));
assertResponseStatus(200, resp);
}
// delete project
resp = client().target()
.path(path)
.path(projectId)
.request()
.delete();
assertResponseStatus(204, resp);
}
}
@Test
@ -141,32 +172,32 @@ public class ProjectApiTest extends BaseApiTest {
}
@Test
public void testremoveGraphs() {
public void testRemoveGraphs() {
String projectId = this.createProjectAndAddGraph("project_test",
"graph_test");
String graph = "{\"project_graphs\":[\"graph_test\"]}";
String graphs = "{\"project_graphs\":[\"graph_test\"]}";
Response resp = client().target()
.path(path)
.path(projectId)
.queryParam("action", "remove_graph")
.request()
.put(Entity.json(graph));
.put(Entity.json(graphs));
assertResponseStatus(200, resp);
String project = this.getProject(projectId);
Assert.assertFalse(project.contains("project_graphs"));
this.addGraphs(projectId, "graph_test1", "graph_test2");
this.addGraphsToProject(projectId, "graph_test1", "graph_test2");
graph = "{\"project_graphs\":[\"graph_test1\"]}";
graphs = "{\"project_graphs\":[\"graph_test1\"]}";
resp = client().target()
.path(path)
.path(projectId)
.queryParam("action", "remove_graph")
.request()
.put(Entity.json(graph));
.put(Entity.json(graphs));
assertResponseStatus(200, resp);
project = this.getProject(projectId);
List<String> graphs1 = assertJsonContains(project, "project_graphs");
Assert.assertEquals(1, graphs1.size());
@ -199,19 +230,18 @@ public class ProjectApiTest extends BaseApiTest {
String graph) {
String projectId = assertJsonContains(createProject(projectName, null),
"id");
this.addGraphs(projectId, graph);
this.addGraphsToProject(projectId, graph);
return projectId;
}
private void addGraphs(String projectId, String... graphNames) {
private void addGraphsToProject(String projectId, String... graphNames) {
Assert.assertFalse(ArrayUtils.isEmpty(graphNames));
StringBuilder graphNamesBuilder = new StringBuilder();
for (int i = 0; i < graphNames.length - 1; i++) {
graphNamesBuilder.append(String.format("\"%s\",", graphNames[i]));
}
graphNamesBuilder.append(
String.format("\"%s\"",
graphNames[graphNames.length - 1]));
graphNamesBuilder.append(String.format("\"%s\"",
graphNames[graphNames.length - 1]));
String graphs = String.format("{\"project_graphs\":[%s]}",
graphNamesBuilder);
Response resp = client().target()

View File

@ -79,7 +79,8 @@ public class TaskApiTest extends BaseApiTest {
r.getStatus() == 202 || r.getStatus() == 400);
if (r.getStatus() == 202) {
String status = assertJsonContains(content, "task_status");
Assert.assertEquals("cancelling", status);
Assert.assertTrue(status, status.equals("cancelling") ||
status.equals("cancelled"));
} else {
assert r.getStatus() == 400;
String error = String.format(

View File

@ -96,7 +96,7 @@
<compiler.target>1.8</compiler.target>
<slf4j.version>1.7.5</slf4j.version>
<log4j.version>1.2.17</log4j.version>
<log4j2.version>2.17.0</log4j2.version>
<log4j2.version>2.17.1</log4j2.version>
<junit.version>4.12</junit.version>
<tinkerpop.version>3.4.3</tinkerpop.version>
<commons.io.version>2.7</commons.io.version>