forked from hugegraph/hugegraph-sync
Compare commits
8 Commits
master
...
release-0.
| Author | SHA1 | Date |
|---|---|---|
|
|
5bf0290e54 | |
|
|
35cffbc032 | |
|
|
df0a7b5dda | |
|
|
35f78f4788 | |
|
|
3e80460a0e | |
|
|
5e1efedb93 | |
|
|
2f726c9a75 | |
|
|
f17a68a510 |
|
|
@ -15,6 +15,7 @@ jobs:
|
|||
runs-on: ubuntu-20.04
|
||||
env:
|
||||
TRAVIS_DIR: hugegraph-dist/src/assembly/travis
|
||||
REPORT_DIR: target/site/jacoco
|
||||
BACKEND: ${{ matrix.BACKEND }}
|
||||
TRIGGER_BRANCH_NAME: ${{ github.ref_name }}
|
||||
HEAD_BRANCH_NAME: ${{ github.head_ref }}
|
||||
|
|
@ -72,9 +73,14 @@ jobs:
|
|||
- name: Run test
|
||||
run: |
|
||||
$TRAVIS_DIR/run-core-test.sh $BACKEND
|
||||
$TRAVIS_DIR/run-api-test.sh $BACKEND
|
||||
$TRAVIS_DIR/run-api-test.sh $BACKEND $REPORT_DIR
|
||||
$TRAVIS_DIR/run-unit-test.sh $BACKEND
|
||||
|
||||
- name: Run Raft test
|
||||
if: ${{ env.BACKEND == 'rocksdb' }}
|
||||
run: |
|
||||
$TRAVIS_DIR/run-api-test-for-raft.sh $BACKEND $REPORT_DIR
|
||||
|
||||
- name: Run TinkerPop test
|
||||
if: ${{ env.RELEASE_BRANCH == 'true' }}
|
||||
run: |
|
||||
|
|
@ -83,4 +89,4 @@ jobs:
|
|||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v1
|
||||
with:
|
||||
file: target/site/jacoco/jacoco.xml
|
||||
file: ${{ env.REPORT_DIR }}/*.xml
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
# HugeGraph
|
||||
|
||||
[](https://www.apache.org/licenses/LICENSE-2.0.html)
|
||||
[](https://github.com/hugegraph/hugegraph/actions/workflows/ci.yml)
|
||||
[](https://codecov.io/gh/hugegraph/hugegraph)
|
||||
[](https://github.com/hugegraph/hugegraph/releases)
|
||||
[](https://github.com/hugegraph/hugegraph/actions/workflows/ci.yml)
|
||||
[](https://codecov.io/gh/hugegraph/hugegraph)
|
||||
[](https://github.com/hugegraph/hugegraph/releases)
|
||||
|
||||
HugeGraph is a fast-speed and highly-scalable [graph database](https://en.wikipedia.org/wiki/Graph_database). Billions of vertices and edges can be easily stored into and queried from HugeGraph due to its excellent OLTP ability. As compliance to [Apache TinkerPop 3](https://tinkerpop.apache.org/) framework, various complicated graph queries can be accomplished through [Gremlin](https://tinkerpop.apache.org/gremlin.html)(a powerful graph traversal language).
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
<dependency>
|
||||
<groupId>com.baidu.hugegraph</groupId>
|
||||
<artifactId>hugegraph-rpc</artifactId>
|
||||
<version>1.0.0</version>
|
||||
<version>2.0.1</version>
|
||||
<exclusions>
|
||||
<!-- conflict with jraft -->
|
||||
<exclusion>
|
||||
|
|
|
|||
|
|
@ -23,9 +23,16 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
|
||||
import javax.annotation.security.RolesAllowed;
|
||||
import javax.inject.Singleton;
|
||||
import javax.json.Json;
|
||||
import javax.json.JsonArrayBuilder;
|
||||
import javax.json.JsonObjectBuilder;
|
||||
import javax.ws.rs.Consumes;
|
||||
import javax.ws.rs.GET;
|
||||
import javax.ws.rs.PUT;
|
||||
import javax.ws.rs.Path;
|
||||
import javax.ws.rs.Produces;
|
||||
import javax.ws.rs.WebApplicationException;
|
||||
import javax.ws.rs.core.Context;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
|
|
@ -38,10 +45,13 @@ import org.apache.commons.lang3.StringUtils;
|
|||
import org.glassfish.hk2.api.MultiException;
|
||||
|
||||
import com.baidu.hugegraph.HugeException;
|
||||
import com.baidu.hugegraph.api.API;
|
||||
import com.baidu.hugegraph.config.HugeConfig;
|
||||
import com.baidu.hugegraph.config.ServerOptions;
|
||||
import com.baidu.hugegraph.exception.HugeGremlinException;
|
||||
import com.baidu.hugegraph.exception.NotFoundException;
|
||||
import com.codahale.metrics.annotation.Timed;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
public class ExceptionFilter {
|
||||
|
||||
|
|
@ -52,12 +62,17 @@ public class ExceptionFilter {
|
|||
private static final int INTERNAL_SERVER_ERROR =
|
||||
Response.Status.INTERNAL_SERVER_ERROR.getStatusCode();
|
||||
|
||||
public static class TracedExceptionMapper {
|
||||
public static class TracedExceptionMapper extends API {
|
||||
|
||||
private static boolean forcedTrace = false;
|
||||
|
||||
@Context
|
||||
private javax.inject.Provider<HugeConfig> configProvider;
|
||||
|
||||
protected boolean trace() {
|
||||
if (forcedTrace) {
|
||||
return true;
|
||||
}
|
||||
HugeConfig config = this.configProvider.get();
|
||||
if (config == null) {
|
||||
return false;
|
||||
|
|
@ -66,54 +81,81 @@ public class ExceptionFilter {
|
|||
}
|
||||
}
|
||||
|
||||
@Path("exception/trace")
|
||||
@Singleton
|
||||
public static class TracedExceptionAPI extends API {
|
||||
|
||||
@GET
|
||||
@Timed
|
||||
@Produces(APPLICATION_JSON_WITH_CHARSET)
|
||||
@RolesAllowed({"admin"})
|
||||
public Object get() {
|
||||
return ImmutableMap.of("trace", TracedExceptionMapper.forcedTrace);
|
||||
}
|
||||
|
||||
@PUT
|
||||
@Timed
|
||||
@Consumes(APPLICATION_JSON)
|
||||
@Produces(APPLICATION_JSON_WITH_CHARSET)
|
||||
@RolesAllowed({"admin"})
|
||||
public Object trace(boolean trace) {
|
||||
TracedExceptionMapper.forcedTrace = trace;
|
||||
return ImmutableMap.of("trace", TracedExceptionMapper.forcedTrace);
|
||||
}
|
||||
}
|
||||
|
||||
@Provider
|
||||
public static class HugeExceptionMapper
|
||||
extends TracedExceptionMapper
|
||||
implements ExceptionMapper<HugeException> {
|
||||
|
||||
@Override
|
||||
public Response toResponse(HugeException exception) {
|
||||
return Response.status(BAD_REQUEST_ERROR)
|
||||
.type(MediaType.APPLICATION_JSON)
|
||||
.entity(formatException(exception))
|
||||
.entity(formatException(exception, this.trace()))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@Provider
|
||||
public static class IllegalArgumentExceptionMapper
|
||||
extends TracedExceptionMapper
|
||||
implements ExceptionMapper<IllegalArgumentException> {
|
||||
|
||||
@Override
|
||||
public Response toResponse(IllegalArgumentException exception) {
|
||||
return Response.status(BAD_REQUEST_ERROR)
|
||||
.type(MediaType.APPLICATION_JSON)
|
||||
.entity(formatException(exception))
|
||||
.entity(formatException(exception, this.trace()))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@Provider
|
||||
public static class NotFoundExceptionMapper
|
||||
extends TracedExceptionMapper
|
||||
implements ExceptionMapper<NotFoundException> {
|
||||
|
||||
@Override
|
||||
public Response toResponse(NotFoundException exception) {
|
||||
return Response.status(NOT_FOUND_ERROR)
|
||||
.type(MediaType.APPLICATION_JSON)
|
||||
.entity(formatException(exception))
|
||||
.entity(formatException(exception, this.trace()))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@Provider
|
||||
public static class NoSuchElementExceptionMapper
|
||||
extends TracedExceptionMapper
|
||||
implements ExceptionMapper<NoSuchElementException> {
|
||||
|
||||
@Override
|
||||
public Response toResponse(NoSuchElementException exception) {
|
||||
return Response.status(NOT_FOUND_ERROR)
|
||||
.type(MediaType.APPLICATION_JSON)
|
||||
.entity(formatException(exception))
|
||||
.entity(formatException(exception, this.trace()))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -189,10 +231,6 @@ public class ExceptionFilter {
|
|||
}
|
||||
}
|
||||
|
||||
public static String formatException(Throwable exception) {
|
||||
return formatException(exception, false);
|
||||
}
|
||||
|
||||
public static String formatException(Throwable exception, boolean trace) {
|
||||
String clazz = exception.getClass().toString();
|
||||
String message = exception.getMessage() != null ?
|
||||
|
|
|
|||
|
|
@ -163,7 +163,7 @@ public class TaskAPI extends API {
|
|||
HugeTask<?> task = scheduler.task(IdGenerator.of(id));
|
||||
if (!task.completed() && !task.cancelling()) {
|
||||
scheduler.cancel(task);
|
||||
if (task.cancelling()) {
|
||||
if (task.cancelling() || task.cancelled()) {
|
||||
return task.asMap();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ public class NeighborRankAPI extends API {
|
|||
"The source of rank request can't be null");
|
||||
E.checkArgument(request.steps != null && !request.steps.isEmpty(),
|
||||
"The steps of rank request can't be empty");
|
||||
E.checkArgument(request.steps.size() <= Long.parseLong(DEFAULT_MAX_DEPTH),
|
||||
E.checkArgument(request.steps.size() <= DEFAULT_MAX_DEPTH,
|
||||
"The steps length of rank request can't exceed %s",
|
||||
DEFAULT_MAX_DEPTH);
|
||||
E.checkArgument(request.alpha > 0 && request.alpha <= 1.0,
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ package com.baidu.hugegraph.api.traversers;
|
|||
|
||||
import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY;
|
||||
import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
|
||||
import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEPTH;
|
||||
import static com.baidu.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT;
|
||||
|
||||
import java.util.Collection;
|
||||
|
|
@ -114,9 +115,9 @@ public class PathsAPI extends TraverserAPI {
|
|||
"The targets of request can't be null");
|
||||
E.checkArgumentNotNull(request.step,
|
||||
"The step of request can't be null");
|
||||
E.checkArgument(request.depth > 0,
|
||||
"The depth of request must be > 0, but got: %s",
|
||||
request.depth);
|
||||
E.checkArgument(request.depth > 0 && request.depth <= DEFAULT_MAX_DEPTH,
|
||||
"The depth of request must be in (0, %s], " +
|
||||
"but got: %s", DEFAULT_MAX_DEPTH, request.depth);
|
||||
|
||||
LOG.debug("Graph [{}] get paths from source vertices '{}', target " +
|
||||
"vertices '{}', with step '{}', max depth '{}', " +
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ public class PersonalRankAPI extends API {
|
|||
"The limit of rank request must be > 0 or == -1, " +
|
||||
"but got: %s", request.limit);
|
||||
E.checkArgument(request.maxDepth > 1L &&
|
||||
request.maxDepth <= Long.parseLong(DEFAULT_MAX_DEPTH),
|
||||
request.maxDepth <= DEFAULT_MAX_DEPTH,
|
||||
"The max depth of rank request must be " +
|
||||
"in range (1, %s], but got '%s'",
|
||||
DEFAULT_MAX_DEPTH, request.maxDepth);
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
<dependency>
|
||||
<groupId>com.baidu.hugegraph</groupId>
|
||||
<artifactId>hugegraph-common</artifactId>
|
||||
<version>2.0.0</version>
|
||||
<version>2.0.1</version>
|
||||
</dependency>
|
||||
|
||||
<!-- tinkerpop -->
|
||||
|
|
@ -54,7 +54,7 @@
|
|||
<dependency>
|
||||
<groupId>com.alipay.sofa</groupId>
|
||||
<artifactId>jraft-core</artifactId>
|
||||
<version>1.3.5</version>
|
||||
<version>1.3.9</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
|
|
|
|||
|
|
@ -111,20 +111,7 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
|
|||
event.checkArgs(String.class, HugeType.class, Id.class);
|
||||
HugeType type = (HugeType) args[1];
|
||||
Id id = (Id) args[2];
|
||||
this.arrayCaches.remove(type, id);
|
||||
|
||||
id = generateId(type, id);
|
||||
Object value = this.idCache.get(id);
|
||||
if (value != null) {
|
||||
// Invalidate id cache
|
||||
this.idCache.invalidate(id);
|
||||
|
||||
// Invalidate name cache
|
||||
SchemaElement schema = (SchemaElement) value;
|
||||
Id prefixedName = generateId(schema.type(),
|
||||
schema.name());
|
||||
this.nameCache.invalidate(prefixedName);
|
||||
}
|
||||
this.invalidateCache(type, id);
|
||||
this.resetCachedAll(type);
|
||||
return true;
|
||||
} else if (Cache.ACTION_CLEAR.equals(args[0])) {
|
||||
|
|
@ -140,21 +127,6 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
|
|||
}
|
||||
}
|
||||
|
||||
private final void resetCachedAll(HugeType type) {
|
||||
// Set the cache all flag of the schema type to false
|
||||
this.cachedTypes().put(type, false);
|
||||
}
|
||||
|
||||
private void clearCache(boolean notify) {
|
||||
this.idCache.clear();
|
||||
this.nameCache.clear();
|
||||
this.arrayCaches.clear();
|
||||
|
||||
if (notify) {
|
||||
this.notifyChanges(Cache.ACTION_CLEARED, null, null);
|
||||
}
|
||||
}
|
||||
|
||||
private void unlistenChanges() {
|
||||
// Unlisten store event
|
||||
this.store().provider().unlisten(this.storeEventListener);
|
||||
|
|
@ -164,11 +136,16 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
|
|||
schemaEventHub.unlisten(Events.CACHE, this.cacheEventListener);
|
||||
}
|
||||
|
||||
private void notifyChanges(String action, HugeType type, Id id) {
|
||||
private final void notifyChanges(String action, HugeType type, Id id) {
|
||||
EventHub graphEventHub = this.params().schemaEventHub();
|
||||
graphEventHub.notify(Events.CACHE, action, type, id);
|
||||
}
|
||||
|
||||
private final void resetCachedAll(HugeType type) {
|
||||
// Set the cache all flag of the schema type to false
|
||||
this.cachedTypes().put(type, false);
|
||||
}
|
||||
|
||||
private final void resetCachedAllIfReachedCapacity() {
|
||||
if (this.idCache.size() >= this.idCache.capacity()) {
|
||||
LOG.warn("Schema cache reached capacity({}): {}",
|
||||
|
|
@ -181,6 +158,47 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
|
|||
return this.arrayCaches.cachedTypes();
|
||||
}
|
||||
|
||||
private final void clearCache(boolean notify) {
|
||||
this.idCache.clear();
|
||||
this.nameCache.clear();
|
||||
this.arrayCaches.clear();
|
||||
|
||||
if (notify) {
|
||||
this.notifyChanges(Cache.ACTION_CLEARED, null, null);
|
||||
}
|
||||
}
|
||||
|
||||
private final void updateCache(SchemaElement schema) {
|
||||
this.resetCachedAllIfReachedCapacity();
|
||||
|
||||
// update id cache
|
||||
Id prefixedId = generateId(schema.type(), schema.id());
|
||||
this.idCache.update(prefixedId, schema);
|
||||
|
||||
// update name cache
|
||||
Id prefixedName = generateId(schema.type(), schema.name());
|
||||
this.nameCache.update(prefixedName, schema);
|
||||
|
||||
// update optimized array cache
|
||||
this.arrayCaches.updateIfNeeded(schema);
|
||||
}
|
||||
|
||||
private final void invalidateCache(HugeType type, Id id) {
|
||||
// remove from id cache and name cache
|
||||
Id prefixedId = generateId(type, id);
|
||||
Object value = this.idCache.get(prefixedId);
|
||||
if (value != null) {
|
||||
this.idCache.invalidate(prefixedId);
|
||||
|
||||
SchemaElement schema = (SchemaElement) value;
|
||||
Id prefixedName = generateId(schema.type(), schema.name());
|
||||
this.nameCache.invalidate(prefixedName);
|
||||
}
|
||||
|
||||
// remove from optimized array cache
|
||||
this.arrayCaches.remove(type, id);
|
||||
}
|
||||
|
||||
private static Id generateId(HugeType type, Id id) {
|
||||
// NOTE: it's slower performance to use:
|
||||
// String.format("%x-%s", type.code(), name)
|
||||
|
|
@ -195,18 +213,7 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
|
|||
protected void addSchema(SchemaElement schema) {
|
||||
super.addSchema(schema);
|
||||
|
||||
this.resetCachedAllIfReachedCapacity();
|
||||
|
||||
// update id cache
|
||||
Id prefixedId = generateId(schema.type(), schema.id());
|
||||
this.idCache.update(prefixedId, schema);
|
||||
|
||||
// update name cache
|
||||
Id prefixedName = generateId(schema.type(), schema.name());
|
||||
this.nameCache.update(prefixedName, schema);
|
||||
|
||||
// update optimized array cache
|
||||
this.arrayCaches.updateIfNeeded(schema);
|
||||
this.updateCache(schema);
|
||||
|
||||
this.notifyChanges(Cache.ACTION_INVALIDED, schema.type(), schema.id());
|
||||
}
|
||||
|
|
@ -227,19 +234,15 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
|
|||
if (value == null) {
|
||||
value = super.getSchema(type, id);
|
||||
if (value != null) {
|
||||
this.resetCachedAllIfReachedCapacity();
|
||||
|
||||
this.idCache.update(prefixedId, value);
|
||||
|
||||
SchemaElement schema = (SchemaElement) value;
|
||||
Id prefixedName = generateId(schema.type(), schema.name());
|
||||
this.nameCache.update(prefixedName, schema);
|
||||
// update id cache, name cache and optimized array cache
|
||||
this.updateCache(schema);
|
||||
}
|
||||
} else {
|
||||
// update optimized array cache for the result from id cache
|
||||
this.arrayCaches.updateIfNeeded((SchemaElement) value);
|
||||
}
|
||||
|
||||
// update optimized array cache
|
||||
this.arrayCaches.updateIfNeeded((SchemaElement) value);
|
||||
|
||||
return (T) value;
|
||||
}
|
||||
|
||||
|
|
@ -252,13 +255,8 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
|
|||
if (value == null) {
|
||||
value = super.getSchema(type, name);
|
||||
if (value != null) {
|
||||
this.resetCachedAllIfReachedCapacity();
|
||||
|
||||
this.nameCache.update(prefixedName, value);
|
||||
|
||||
SchemaElement schema = (SchemaElement) value;
|
||||
Id prefixedId = generateId(schema.type(), schema.id());
|
||||
this.idCache.update(prefixedId, schema);
|
||||
this.updateCache(schema);
|
||||
}
|
||||
}
|
||||
return (T) value;
|
||||
|
|
@ -268,18 +266,7 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
|
|||
protected void removeSchema(SchemaElement schema) {
|
||||
super.removeSchema(schema);
|
||||
|
||||
Id prefixedId = generateId(schema.type(), schema.id());
|
||||
Object value = this.idCache.get(prefixedId);
|
||||
if (value != null) {
|
||||
this.idCache.invalidate(prefixedId);
|
||||
|
||||
schema = (SchemaElement) value;
|
||||
Id prefixedName = generateId(schema.type(), schema.name());
|
||||
this.nameCache.invalidate(prefixedName);
|
||||
}
|
||||
|
||||
// remove from optimized array cache
|
||||
this.arrayCaches.remove(schema.type(), schema.id());
|
||||
this.invalidateCache(schema.type(), schema.id());
|
||||
|
||||
this.notifyChanges(Cache.ACTION_INVALIDED, schema.type(), schema.id());
|
||||
}
|
||||
|
|
@ -299,16 +286,13 @@ public final class CachedSchemaTransaction extends SchemaTransaction {
|
|||
});
|
||||
return results;
|
||||
} else {
|
||||
this.cachedTypes().remove(type);
|
||||
List<T> results = super.getAllSchema(type);
|
||||
long free = this.idCache.capacity() - this.idCache.size();
|
||||
if (results.size() <= free) {
|
||||
// Update cache
|
||||
for (T schema : results) {
|
||||
Id prefixedId = generateId(schema.type(), schema.id());
|
||||
this.idCache.update(prefixedId, schema);
|
||||
|
||||
Id prefixedName = generateId(schema.type(), schema.name());
|
||||
this.nameCache.update(prefixedName, schema);
|
||||
this.updateCache(schema);
|
||||
}
|
||||
this.cachedTypes().putIfAbsent(type, true);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -505,6 +505,10 @@ public final class ConditionQueryFlatten {
|
|||
|
||||
private static final long serialVersionUID = -2110811280408887334L;
|
||||
|
||||
public Relations() {
|
||||
super();
|
||||
}
|
||||
|
||||
public static Relations of(Relation... relations) {
|
||||
Relations rs = new Relations();
|
||||
rs.addAll(Arrays.asList(relations));
|
||||
|
|
|
|||
|
|
@ -50,11 +50,13 @@ public class RaftBackendStore implements BackendStore {
|
|||
private final BackendStore store;
|
||||
private final RaftSharedContext context;
|
||||
private final ThreadLocal<MutationBatch> mutationBatch;
|
||||
private final boolean isSafeRead;
|
||||
|
||||
public RaftBackendStore(BackendStore store, RaftSharedContext context) {
|
||||
this.store = store;
|
||||
this.context = context;
|
||||
this.mutationBatch = new ThreadLocal<>();
|
||||
this.isSafeRead = this.context.isSafeRead();
|
||||
}
|
||||
|
||||
public BackendStore originStore() {
|
||||
|
|
@ -143,7 +145,8 @@ public class RaftBackendStore implements BackendStore {
|
|||
|
||||
@Override
|
||||
public Number queryNumber(Query query) {
|
||||
return (Number) this.queryByRaft(query, o -> this.store.queryNumber(query));
|
||||
return (Number)
|
||||
this.queryByRaft(query, o -> this.store.queryNumber(query));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -186,7 +189,10 @@ public class RaftBackendStore implements BackendStore {
|
|||
|
||||
@Override
|
||||
public long getCounter(HugeType type) {
|
||||
return (Long) this.queryByRaft(type, o -> this.store.getCounter(type));
|
||||
Object counter = this.queryByRaft(type, true,
|
||||
o -> this.store.getCounter(type));
|
||||
assert counter instanceof Long;
|
||||
return (Long) counter;
|
||||
}
|
||||
|
||||
private Object submitAndWait(StoreAction action, byte[] data) {
|
||||
|
|
@ -200,7 +206,12 @@ public class RaftBackendStore implements BackendStore {
|
|||
}
|
||||
|
||||
private Object queryByRaft(Object query, Function<Object, Object> func) {
|
||||
if (!this.context.isSafeRead()) {
|
||||
return this.queryByRaft(query, this.isSafeRead, func);
|
||||
}
|
||||
|
||||
private Object queryByRaft(Object query, boolean safeRead,
|
||||
Function<Object, Object> func) {
|
||||
if (!safeRead) {
|
||||
return func.apply(query);
|
||||
}
|
||||
|
||||
|
|
@ -212,8 +223,8 @@ public class RaftBackendStore implements BackendStore {
|
|||
future.complete(status, () -> func.apply(query));
|
||||
} else {
|
||||
future.failure(status, new BackendException(
|
||||
"Failed to execute query '%s' with read-index: %s",
|
||||
query, status));
|
||||
"Failed to do raft read-index: %s",
|
||||
status));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -221,8 +232,8 @@ public class RaftBackendStore implements BackendStore {
|
|||
try {
|
||||
return future.waitFinished();
|
||||
} catch (Throwable e) {
|
||||
LOG.warn("Failed to execute query '{}' with read-index: {}",
|
||||
query, future.status());
|
||||
LOG.warn("Failed to execute query '{}': {}",
|
||||
query, future.status(), e);
|
||||
throw new BackendException("Failed to execute query: %s", e, query);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ public class RaftBackendStoreProvider implements BackendStoreProvider {
|
|||
for (RaftBackendStore store : this.stores()) {
|
||||
store.init();
|
||||
}
|
||||
this.notifyAndWaitEvent(Events.STORE_INITED);
|
||||
this.notifyAndWaitEvent(Events.STORE_INIT);
|
||||
|
||||
LOG.debug("Graph '{}' store has been initialized", this.graph());
|
||||
}
|
||||
|
|
@ -214,7 +214,8 @@ public class RaftBackendStoreProvider implements BackendStoreProvider {
|
|||
|
||||
@Override
|
||||
public void createSnapshot() {
|
||||
StoreCommand command = new StoreCommand(StoreType.ALL,
|
||||
// TODO: snapshot for StoreType.ALL instead of StoreType.GRAPH
|
||||
StoreCommand command = new StoreCommand(StoreType.GRAPH,
|
||||
StoreAction.SNAPSHOT, null);
|
||||
RaftStoreClosure closure = new RaftStoreClosure(command);
|
||||
this.context.node().submitAndWait(command, closure);
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicReference;
|
|||
import org.slf4j.Logger;
|
||||
|
||||
import com.alipay.sofa.jraft.Node;
|
||||
import com.alipay.sofa.jraft.RaftGroupService;
|
||||
import com.alipay.sofa.jraft.RaftServiceFactory;
|
||||
import com.alipay.sofa.jraft.Status;
|
||||
import com.alipay.sofa.jraft.closure.ReadIndexClosure;
|
||||
import com.alipay.sofa.jraft.core.Replicator.ReplicatorStateListener;
|
||||
|
|
@ -36,7 +36,6 @@ import com.alipay.sofa.jraft.entity.PeerId;
|
|||
import com.alipay.sofa.jraft.entity.Task;
|
||||
import com.alipay.sofa.jraft.error.RaftError;
|
||||
import com.alipay.sofa.jraft.option.NodeOptions;
|
||||
import com.alipay.sofa.jraft.rpc.RpcServer;
|
||||
import com.alipay.sofa.jraft.util.BytesUtil;
|
||||
import com.baidu.hugegraph.backend.BackendException;
|
||||
import com.baidu.hugegraph.util.LZ4Util;
|
||||
|
|
@ -58,6 +57,7 @@ public final class RaftNode {
|
|||
this.stateMachine = new StoreStateMachine(context);
|
||||
try {
|
||||
this.node = this.initRaftNode();
|
||||
LOG.info("Start raft node: {}", this);
|
||||
} catch (IOException e) {
|
||||
throw new BackendException("Failed to init raft node", e);
|
||||
}
|
||||
|
|
@ -94,6 +94,7 @@ public final class RaftNode {
|
|||
}
|
||||
|
||||
public void shutdown() {
|
||||
LOG.info("Shutdown raft node: {}", this);
|
||||
this.node.shutdown();
|
||||
}
|
||||
|
||||
|
|
@ -116,13 +117,14 @@ public final class RaftNode {
|
|||
// TODO: When support sharding, groupId needs to be bound to shard Id
|
||||
String groupId = this.context.group();
|
||||
PeerId endpoint = this.context.endpoint();
|
||||
RpcServer rpcServer = this.context.rpcServer();
|
||||
RaftGroupService raftGroupService;
|
||||
// Shared rpc server
|
||||
raftGroupService = new RaftGroupService(groupId, endpoint, nodeOptions,
|
||||
rpcServer, true);
|
||||
// Start node
|
||||
return raftGroupService.start(false);
|
||||
/*
|
||||
* Start raft node with shared rpc server:
|
||||
* return new RaftGroupService(groupId, endpoint, nodeOptions,
|
||||
* this.context.rpcServer(), true)
|
||||
* .start(false)
|
||||
*/
|
||||
return RaftServiceFactory.createAndInitRaftNode(groupId, endpoint,
|
||||
nodeOptions);
|
||||
}
|
||||
|
||||
private void submitCommand(StoreCommand command, RaftStoreClosure closure) {
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ package com.baidu.hugegraph.backend.store.raft;
|
|||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
|
|
@ -30,14 +31,15 @@ import java.util.concurrent.RejectedExecutionException;
|
|||
import java.util.concurrent.RejectedExecutionHandler;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
|
||||
import com.alipay.sofa.jraft.option.ReadOnlyOption;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import com.alipay.sofa.jraft.NodeManager;
|
||||
import com.alipay.sofa.jraft.conf.Configuration;
|
||||
import com.alipay.sofa.jraft.entity.PeerId;
|
||||
import com.alipay.sofa.jraft.option.NodeOptions;
|
||||
import com.alipay.sofa.jraft.option.RaftOptions;
|
||||
import com.alipay.sofa.jraft.option.ReadOnlyOption;
|
||||
import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
|
||||
import com.alipay.sofa.jraft.rpc.RpcServer;
|
||||
import com.alipay.sofa.jraft.util.NamedThreadFactory;
|
||||
|
|
@ -46,6 +48,9 @@ import com.baidu.hugegraph.HugeException;
|
|||
import com.baidu.hugegraph.HugeGraphParams;
|
||||
import com.baidu.hugegraph.backend.cache.Cache;
|
||||
import com.baidu.hugegraph.backend.id.Id;
|
||||
import com.baidu.hugegraph.backend.query.Query;
|
||||
import com.baidu.hugegraph.backend.store.BackendAction;
|
||||
import com.baidu.hugegraph.backend.store.BackendMutation;
|
||||
import com.baidu.hugegraph.backend.store.BackendStore;
|
||||
import com.baidu.hugegraph.backend.store.raft.rpc.ListPeersProcessor;
|
||||
import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType;
|
||||
|
|
@ -101,7 +106,7 @@ public final class RaftSharedContext {
|
|||
|
||||
public RaftSharedContext(HugeGraphParams params) {
|
||||
this.params = params;
|
||||
HugeConfig config = params.configuration();
|
||||
HugeConfig config = this.config();
|
||||
|
||||
this.schemaStoreName = config.get(CoreOptions.STORE_SCHEMA);
|
||||
this.graphStoreName = config.get(CoreOptions.STORE_GRAPH);
|
||||
|
|
@ -126,12 +131,7 @@ public final class RaftSharedContext {
|
|||
this.raftGroupManager = null;
|
||||
this.rpcForwarder = null;
|
||||
this.registerRpcRequestProcessors();
|
||||
}
|
||||
|
||||
private void registerRpcRequestProcessors() {
|
||||
this.rpcServer.registerProcessor(new StoreCommandProcessor(this));
|
||||
this.rpcServer.registerProcessor(new SetLeaderProcessor(this));
|
||||
this.rpcServer.registerProcessor(new ListPeersProcessor(this));
|
||||
LOG.info("Start raft server successfully: {}", this.endpoint());
|
||||
}
|
||||
|
||||
public void initRaftNode() {
|
||||
|
|
@ -149,8 +149,14 @@ public final class RaftSharedContext {
|
|||
}
|
||||
|
||||
public void close() {
|
||||
LOG.info("Stopping raft nodes");
|
||||
this.rpcServer.shutdown();
|
||||
LOG.info("Stop raft server: {}", this.endpoint());
|
||||
|
||||
RaftNode node = this.node();
|
||||
if (node != null) {
|
||||
node.shutdown();
|
||||
}
|
||||
|
||||
this.shutdownRpcServer();
|
||||
}
|
||||
|
||||
public RaftNode node() {
|
||||
|
|
@ -168,10 +174,6 @@ public final class RaftSharedContext {
|
|||
return this.raftGroupManager;
|
||||
}
|
||||
|
||||
public RpcServer rpcServer() {
|
||||
return this.rpcServer;
|
||||
}
|
||||
|
||||
public String group() {
|
||||
return DEFAULT_GROUP;
|
||||
}
|
||||
|
|
@ -266,12 +268,41 @@ public final class RaftSharedContext {
|
|||
return nodeOptions;
|
||||
}
|
||||
|
||||
public void clearCache() {
|
||||
protected void clearCache() {
|
||||
// Just choose two representatives used to represent schema and graph
|
||||
this.notifyCache(Cache.ACTION_CLEAR, HugeType.VERTEX_LABEL, null);
|
||||
this.notifyCache(Cache.ACTION_CLEAR, HugeType.VERTEX, null);
|
||||
}
|
||||
|
||||
protected void updateCacheIfNeeded(BackendMutation mutation,
|
||||
boolean forwarded) {
|
||||
// Update cache only when graph run in general mode
|
||||
if (this.graphMode() != GraphMode.NONE) {
|
||||
return;
|
||||
}
|
||||
/*
|
||||
* 1. If Follower, need to update cache from store to tx
|
||||
* 3. If Leader, request is forwarded by follower, need to update cache
|
||||
* 2. If Leader, request comes from leader, don't need to update cache,
|
||||
* because the cache will be updated by upper layer
|
||||
*/
|
||||
if (!forwarded && this.node().selfIsLeader()) {
|
||||
return;
|
||||
}
|
||||
for (HugeType type : mutation.types()) {
|
||||
List<Id> ids = new ArrayList<>((int) Query.COMMIT_BATCH);
|
||||
if (type.isSchema() || type.isGraph()) {
|
||||
java.util.Iterator<BackendAction> it = mutation.mutation(type);
|
||||
while (it.hasNext()) {
|
||||
ids.add(it.next().entry().originId());
|
||||
}
|
||||
this.notifyCache(Cache.ACTION_INVALID, type, ids);
|
||||
} else {
|
||||
// Ignore other types due to not cached them
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void notifyCache(String action, HugeType type, List<Id> ids) {
|
||||
EventHub eventHub;
|
||||
if (type.isGraph()) {
|
||||
|
|
@ -340,14 +371,25 @@ public final class RaftSharedContext {
|
|||
System.setProperty("bolt.channel_write_buf_high_water_mark",
|
||||
String.valueOf(highWaterMark));
|
||||
|
||||
PeerId serverId = new PeerId();
|
||||
serverId.parse(this.config().get(CoreOptions.RAFT_ENDPOINT));
|
||||
PeerId endpoint = this.endpoint();
|
||||
NodeManager.getInstance().addAddress(endpoint.getEndpoint());
|
||||
RpcServer rpcServer = RaftRpcServerFactory.createAndStartRaftRpcServer(
|
||||
serverId.getEndpoint());
|
||||
LOG.info("RPC server is started successfully");
|
||||
endpoint.getEndpoint());
|
||||
return rpcServer;
|
||||
}
|
||||
|
||||
private void shutdownRpcServer() {
|
||||
this.rpcServer.shutdown();
|
||||
PeerId endpoint = this.endpoint();
|
||||
NodeManager.getInstance().removeAddress(endpoint.getEndpoint());
|
||||
}
|
||||
|
||||
private void registerRpcRequestProcessors() {
|
||||
this.rpcServer.registerProcessor(new StoreCommandProcessor(this));
|
||||
this.rpcServer.registerProcessor(new SetLeaderProcessor(this));
|
||||
this.rpcServer.registerProcessor(new ListPeersProcessor(this));
|
||||
}
|
||||
|
||||
private ExecutorService createReadIndexExecutor(int coreThreads) {
|
||||
int maxThreads = coreThreads << 2;
|
||||
String name = "store-read-index-callback";
|
||||
|
|
|
|||
|
|
@ -19,8 +19,10 @@
|
|||
|
||||
package com.baidu.hugegraph.backend.store.raft;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -36,18 +38,12 @@ import com.alipay.sofa.jraft.error.RaftException;
|
|||
import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
|
||||
import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
|
||||
import com.baidu.hugegraph.backend.BackendException;
|
||||
import com.baidu.hugegraph.backend.cache.Cache;
|
||||
import com.baidu.hugegraph.backend.id.Id;
|
||||
import com.baidu.hugegraph.backend.query.Query;
|
||||
import com.baidu.hugegraph.backend.serializer.BytesBuffer;
|
||||
import com.baidu.hugegraph.backend.store.BackendAction;
|
||||
import com.baidu.hugegraph.backend.store.BackendMutation;
|
||||
import com.baidu.hugegraph.backend.store.BackendStore;
|
||||
import com.baidu.hugegraph.backend.store.raft.RaftBackendStore.IncrCounter;
|
||||
import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreAction;
|
||||
import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType;
|
||||
import com.baidu.hugegraph.type.HugeType;
|
||||
import com.baidu.hugegraph.type.define.GraphMode;
|
||||
import com.baidu.hugegraph.util.E;
|
||||
import com.baidu.hugegraph.util.LZ4Util;
|
||||
import com.baidu.hugegraph.util.Log;
|
||||
|
|
@ -72,78 +68,24 @@ public final class StoreStateMachine extends StateMachineAdapter {
|
|||
return this.context.node();
|
||||
}
|
||||
|
||||
private void updateCacheIfNeeded(BackendMutation mutation,
|
||||
boolean forwarded) {
|
||||
// Update cache only when graph run in general mode
|
||||
if (this.context.graphMode() != GraphMode.NONE) {
|
||||
return;
|
||||
}
|
||||
/*
|
||||
* 1. Follower need to update cache from store to tx
|
||||
* 2. If request come from leader, cache will be updated by upper layer
|
||||
* 3. If request is forwarded by follower, need to update cache
|
||||
*/
|
||||
if (!forwarded && this.node().selfIsLeader()) {
|
||||
return;
|
||||
}
|
||||
for (HugeType type : mutation.types()) {
|
||||
List<Id> ids = new ArrayList<>((int) Query.COMMIT_BATCH);
|
||||
if (type.isSchema() || type.isGraph()) {
|
||||
java.util.Iterator<BackendAction> it = mutation.mutation(type);
|
||||
while (it.hasNext()) {
|
||||
ids.add(it.next().entry().originId());
|
||||
}
|
||||
this.context.notifyCache(Cache.ACTION_INVALID, type, ids);
|
||||
} else {
|
||||
// Ignore other types due to not cached them
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onApply(Iterator iter) {
|
||||
LOG.debug("Node role: {}", this.node().selfIsLeader() ?
|
||||
"leader" : "follower");
|
||||
RaftStoreClosure closure = null;
|
||||
List<Future<?>> futures = new ArrayList<>();
|
||||
try {
|
||||
// Apply all the logs
|
||||
while (iter.hasNext()) {
|
||||
closure = (RaftStoreClosure) iter.done();
|
||||
RaftStoreClosure closure = (RaftStoreClosure) iter.done();
|
||||
if (closure != null) {
|
||||
// Leader just take it out from the closure
|
||||
StoreCommand command = closure.command();
|
||||
BytesBuffer buffer = BytesBuffer.wrap(command.data());
|
||||
// The first two bytes are StoreType and StoreAction
|
||||
StoreType type = StoreType.valueOf(buffer.read());
|
||||
StoreAction action = StoreAction.valueOf(buffer.read());
|
||||
boolean forwarded = command.forwarded();
|
||||
// Let the producer thread to handle it
|
||||
closure.complete(Status.OK(), () -> {
|
||||
this.applyCommand(type, action, buffer, forwarded);
|
||||
return null;
|
||||
});
|
||||
futures.add(this.onApplyLeader(closure));
|
||||
} else {
|
||||
// Follower need readMutation data
|
||||
byte[] bytes = iter.getData().array();
|
||||
// Let the backend thread do it directly
|
||||
futures.add(this.context.backendExecutor().submit(() -> {
|
||||
BytesBuffer buffer = LZ4Util.decompress(bytes,
|
||||
RaftSharedContext.BLOCK_SIZE);
|
||||
buffer.forReadWritten();
|
||||
StoreType type = StoreType.valueOf(buffer.read());
|
||||
StoreAction action = StoreAction.valueOf(buffer.read());
|
||||
try {
|
||||
this.applyCommand(type, action, buffer, false);
|
||||
} catch (Throwable e) {
|
||||
String title = "Failed to execute backend command";
|
||||
LOG.error("{}: {}", title, action, e);
|
||||
throw new BackendException(title, e);
|
||||
}
|
||||
}));
|
||||
futures.add(this.onApplyFollower(iter.getData()));
|
||||
}
|
||||
iter.next();
|
||||
}
|
||||
// Follower wait tasks finished
|
||||
|
||||
// Wait for all tasks finished
|
||||
for (Future<?> future : futures) {
|
||||
future.get();
|
||||
}
|
||||
|
|
@ -152,17 +94,58 @@ public final class StoreStateMachine extends StateMachineAdapter {
|
|||
LOG.error("{}", title, e);
|
||||
Status status = new Status(RaftError.ESTATEMACHINE,
|
||||
"%s: %s", title, e.getMessage());
|
||||
if (closure != null) {
|
||||
closure.failure(status, e);
|
||||
}
|
||||
// Will cause current node inactive
|
||||
// TODO: rollback to correct index
|
||||
iter.setErrorAndRollback(1L, status);
|
||||
}
|
||||
}
|
||||
|
||||
private void applyCommand(StoreType type, StoreAction action,
|
||||
BytesBuffer buffer, boolean forwarded) {
|
||||
private Future<?> onApplyLeader(RaftStoreClosure closure) {
|
||||
// Leader just take the command out from the closure
|
||||
StoreCommand command = closure.command();
|
||||
BytesBuffer buffer = BytesBuffer.wrap(command.data());
|
||||
// The first two bytes are StoreType and StoreAction
|
||||
StoreType type = StoreType.valueOf(buffer.read());
|
||||
StoreAction action = StoreAction.valueOf(buffer.read());
|
||||
boolean forwarded = command.forwarded();
|
||||
// Let the producer thread to handle it, and wait for it
|
||||
CompletableFuture<Object> future = new CompletableFuture<>();
|
||||
closure.complete(Status.OK(), () -> {
|
||||
Object result;
|
||||
try {
|
||||
result = this.applyCommand(type, action, buffer, forwarded);
|
||||
} catch (Throwable e) {
|
||||
future.completeExceptionally(e);
|
||||
throw e;
|
||||
}
|
||||
future.complete(result);
|
||||
return result;
|
||||
});
|
||||
return future;
|
||||
}
|
||||
|
||||
private Future<?> onApplyFollower(ByteBuffer data) {
|
||||
// Follower need to read mutation data
|
||||
byte[] bytes = data.array();
|
||||
// Let the backend thread do it directly
|
||||
return this.context.backendExecutor().submit(() -> {
|
||||
BytesBuffer buffer = LZ4Util.decompress(bytes,
|
||||
RaftSharedContext.BLOCK_SIZE);
|
||||
buffer.forReadWritten();
|
||||
StoreType type = StoreType.valueOf(buffer.read());
|
||||
StoreAction action = StoreAction.valueOf(buffer.read());
|
||||
try {
|
||||
return this.applyCommand(type, action, buffer, false);
|
||||
} catch (Throwable e) {
|
||||
String title = "Failed to execute backend command";
|
||||
LOG.error("{}: {}", title, action, e);
|
||||
throw new BackendException(title, e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private Object applyCommand(StoreType type, StoreAction action,
|
||||
BytesBuffer buffer, boolean forwarded) {
|
||||
E.checkState(type != StoreType.ALL,
|
||||
"Can't apply command for all store at one time");
|
||||
BackendStore store = this.store(type);
|
||||
|
|
@ -190,7 +173,7 @@ public final class StoreStateMachine extends StateMachineAdapter {
|
|||
store.beginTx();
|
||||
for (BackendMutation mutation : mutations) {
|
||||
store.mutate(mutation);
|
||||
this.updateCacheIfNeeded(mutation, forwarded);
|
||||
this.context.updateCacheIfNeeded(mutation, forwarded);
|
||||
}
|
||||
store.commitTx();
|
||||
break;
|
||||
|
|
@ -205,6 +188,7 @@ public final class StoreStateMachine extends StateMachineAdapter {
|
|||
default:
|
||||
throw new IllegalArgumentException("Invalid action " + action);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -50,16 +50,20 @@ public class StoreCommandProcessor
|
|||
@Override
|
||||
public Message processRequest(StoreCommandRequest request,
|
||||
RpcRequestClosure done) {
|
||||
LOG.debug("Processing StoreCommandRequest");
|
||||
LOG.debug("Processing StoreCommandRequest: {}", request.getAction());
|
||||
RaftNode node = this.context.node();
|
||||
try {
|
||||
StoreCommand command = this.parseStoreCommand(request);
|
||||
RaftStoreClosure closure = new RaftStoreClosure(command);
|
||||
node.submitAndWait(command, closure);
|
||||
// TODO: return the submitAndWait() result to rpc client
|
||||
return StoreCommandResponse.newBuilder().setStatus(true).build();
|
||||
} catch (Throwable e) {
|
||||
StoreCommandResponse.Builder builder;
|
||||
builder = StoreCommandResponse.newBuilder().setStatus(false);
|
||||
LOG.warn("Failed to process StoreCommandRequest: {}",
|
||||
request.getAction(), e);
|
||||
StoreCommandResponse.Builder builder = StoreCommandResponse
|
||||
.newBuilder()
|
||||
.setStatus(false);
|
||||
if (e.getMessage() != null) {
|
||||
builder.setMessage(e.getMessage());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,14 +40,14 @@ import com.baidu.hugegraph.backend.store.BackendStore;
|
|||
import com.baidu.hugegraph.config.CoreOptions;
|
||||
import com.baidu.hugegraph.exception.NotAllowException;
|
||||
import com.baidu.hugegraph.job.JobBuilder;
|
||||
import com.baidu.hugegraph.job.schema.EdgeLabelRemoveCallable;
|
||||
import com.baidu.hugegraph.job.schema.IndexLabelRemoveCallable;
|
||||
import com.baidu.hugegraph.job.schema.OlapPropertyKeyClearCallable;
|
||||
import com.baidu.hugegraph.job.schema.OlapPropertyKeyCreateCallable;
|
||||
import com.baidu.hugegraph.job.schema.OlapPropertyKeyRemoveCallable;
|
||||
import com.baidu.hugegraph.job.schema.RebuildIndexCallable;
|
||||
import com.baidu.hugegraph.job.schema.SchemaCallable;
|
||||
import com.baidu.hugegraph.job.schema.VertexLabelRemoveCallable;
|
||||
import com.baidu.hugegraph.job.schema.EdgeLabelRemoveJob;
|
||||
import com.baidu.hugegraph.job.schema.IndexLabelRebuildJob;
|
||||
import com.baidu.hugegraph.job.schema.IndexLabelRemoveJob;
|
||||
import com.baidu.hugegraph.job.schema.OlapPropertyKeyClearJob;
|
||||
import com.baidu.hugegraph.job.schema.OlapPropertyKeyCreateJob;
|
||||
import com.baidu.hugegraph.job.schema.OlapPropertyKeyRemoveJob;
|
||||
import com.baidu.hugegraph.job.schema.SchemaJob;
|
||||
import com.baidu.hugegraph.job.schema.VertexLabelRemoveJob;
|
||||
import com.baidu.hugegraph.perf.PerfUtil.Watched;
|
||||
import com.baidu.hugegraph.schema.EdgeLabel;
|
||||
import com.baidu.hugegraph.schema.IndexLabel;
|
||||
|
|
@ -200,7 +200,7 @@ public class SchemaTransaction extends IndexableTransaction {
|
|||
@Watched(prefix = "schema")
|
||||
public Id removeVertexLabel(Id id) {
|
||||
LOG.debug("SchemaTransaction remove vertex label '{}'", id);
|
||||
SchemaCallable callable = new VertexLabelRemoveCallable();
|
||||
SchemaJob callable = new VertexLabelRemoveJob();
|
||||
VertexLabel schema = this.getVertexLabel(id);
|
||||
return asyncRun(this.graph(), schema, callable);
|
||||
}
|
||||
|
|
@ -226,7 +226,7 @@ public class SchemaTransaction extends IndexableTransaction {
|
|||
@Watched(prefix = "schema")
|
||||
public Id removeEdgeLabel(Id id) {
|
||||
LOG.debug("SchemaTransaction remove edge label '{}'", id);
|
||||
SchemaCallable callable = new EdgeLabelRemoveCallable();
|
||||
SchemaJob callable = new EdgeLabelRemoveJob();
|
||||
EdgeLabel schema = this.getEdgeLabel(id);
|
||||
return asyncRun(this.graph(), schema, callable);
|
||||
}
|
||||
|
|
@ -262,7 +262,7 @@ public class SchemaTransaction extends IndexableTransaction {
|
|||
@Watched(prefix = "schema")
|
||||
public Id removeIndexLabel(Id id) {
|
||||
LOG.debug("SchemaTransaction remove index label '{}'", id);
|
||||
SchemaCallable callable = new IndexLabelRemoveCallable();
|
||||
SchemaJob callable = new IndexLabelRemoveJob();
|
||||
IndexLabel schema = this.getIndexLabel(id);
|
||||
return asyncRun(this.graph(), schema, callable);
|
||||
}
|
||||
|
|
@ -276,7 +276,7 @@ public class SchemaTransaction extends IndexableTransaction {
|
|||
public Id rebuildIndex(SchemaElement schema, Set<Id> dependencies) {
|
||||
LOG.debug("SchemaTransaction rebuild index for {} with id '{}'",
|
||||
schema.type(), schema.id());
|
||||
SchemaCallable callable = new RebuildIndexCallable();
|
||||
SchemaJob callable = new IndexLabelRebuildJob();
|
||||
return asyncRun(this.graph(), schema, callable, dependencies);
|
||||
}
|
||||
|
||||
|
|
@ -305,26 +305,30 @@ public class SchemaTransaction extends IndexableTransaction {
|
|||
public Id createOlapPk(PropertyKey propertyKey) {
|
||||
LOG.debug("SchemaTransaction create olap property key {} with id '{}'",
|
||||
propertyKey.name(), propertyKey.id());
|
||||
SchemaCallable callable = new OlapPropertyKeyCreateCallable();
|
||||
SchemaJob callable = new OlapPropertyKeyCreateJob();
|
||||
return asyncRun(this.graph(), propertyKey, callable);
|
||||
}
|
||||
|
||||
public Id clearOlapPk(PropertyKey propertyKey) {
|
||||
LOG.debug("SchemaTransaction clear olap property key {} with id '{}'",
|
||||
propertyKey.name(), propertyKey.id());
|
||||
SchemaCallable callable = new OlapPropertyKeyClearCallable();
|
||||
SchemaJob callable = new OlapPropertyKeyClearJob();
|
||||
return asyncRun(this.graph(), propertyKey, callable);
|
||||
}
|
||||
|
||||
public Id removeOlapPk(PropertyKey propertyKey) {
|
||||
LOG.debug("SchemaTransaction remove olap property key {} with id '{}'",
|
||||
propertyKey.name(), propertyKey.id());
|
||||
SchemaCallable callable = new OlapPropertyKeyRemoveCallable();
|
||||
SchemaJob callable = new OlapPropertyKeyRemoveJob();
|
||||
return asyncRun(this.graph(), propertyKey, callable);
|
||||
}
|
||||
|
||||
@Watched(prefix = "schema")
|
||||
public void updateSchemaStatus(SchemaElement schema, SchemaStatus status) {
|
||||
if (!this.existsSchemaId(schema.type(), schema.id())) {
|
||||
LOG.warn("Can't update schema '{}', it may be deleted", schema);
|
||||
return;
|
||||
}
|
||||
schema.status(status);
|
||||
this.updateSchema(schema);
|
||||
}
|
||||
|
|
@ -543,17 +547,17 @@ public class SchemaTransaction extends IndexableTransaction {
|
|||
}
|
||||
|
||||
private static Id asyncRun(HugeGraph graph, SchemaElement schema,
|
||||
SchemaCallable callable) {
|
||||
SchemaJob callable) {
|
||||
return asyncRun(graph, schema, callable, ImmutableSet.of());
|
||||
}
|
||||
|
||||
@Watched(prefix = "schema")
|
||||
private static Id asyncRun(HugeGraph graph, SchemaElement schema,
|
||||
SchemaCallable callable, Set<Id> dependencies) {
|
||||
SchemaJob callable, Set<Id> dependencies) {
|
||||
E.checkArgument(schema != null, "Schema can't be null");
|
||||
String name = SchemaCallable.formatTaskName(schema.type(),
|
||||
schema.id(),
|
||||
schema.name());
|
||||
String name = SchemaJob.formatTaskName(schema.type(),
|
||||
schema.id(),
|
||||
schema.name());
|
||||
|
||||
JobBuilder<Object> builder = JobBuilder.of(graph).name(name)
|
||||
.job(callable)
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import com.baidu.hugegraph.type.define.SchemaStatus;
|
|||
import com.baidu.hugegraph.util.LockUtil;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
|
||||
public class EdgeLabelRemoveCallable extends SchemaCallable {
|
||||
public class EdgeLabelRemoveJob extends SchemaJob {
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
|
|
@ -43,7 +43,7 @@ public class EdgeLabelRemoveCallable extends SchemaCallable {
|
|||
return null;
|
||||
}
|
||||
|
||||
protected static void removeEdgeLabel(HugeGraphParams graph, Id id) {
|
||||
private static void removeEdgeLabel(HugeGraphParams graph, Id id) {
|
||||
GraphTransaction graphTx = graph.graphTransaction();
|
||||
SchemaTransaction schemaTx = graph.schemaTransaction();
|
||||
EdgeLabel edgeLabel = schemaTx.getEdgeLabel(id);
|
||||
|
|
@ -51,7 +51,11 @@ public class EdgeLabelRemoveCallable extends SchemaCallable {
|
|||
if (edgeLabel == null) {
|
||||
return;
|
||||
}
|
||||
// TODO: use event to replace direct call
|
||||
if (edgeLabel.status().deleting()) {
|
||||
LOG.info("The edge label '{}' has been in {} status, " +
|
||||
"please check if it's expected to delete it again",
|
||||
edgeLabel, edgeLabel.status());
|
||||
}
|
||||
// Remove index related data(include schema) of this edge label
|
||||
Set<Id> indexIds = ImmutableSet.copyOf(edgeLabel.indexLabels());
|
||||
LockUtil.Locks locks = new LockUtil.Locks(graph.name());
|
||||
|
|
@ -60,16 +64,18 @@ public class EdgeLabelRemoveCallable extends SchemaCallable {
|
|||
schemaTx.updateSchemaStatus(edgeLabel, SchemaStatus.DELETING);
|
||||
try {
|
||||
for (Id indexId : indexIds) {
|
||||
IndexLabelRemoveCallable.removeIndexLabel(graph, indexId);
|
||||
IndexLabelRemoveJob.removeIndexLabel(graph, indexId);
|
||||
}
|
||||
// Remove all edges which has matched label
|
||||
// TODO: use event to replace direct call
|
||||
graphTx.removeEdges(edgeLabel);
|
||||
removeSchema(schemaTx, edgeLabel);
|
||||
/*
|
||||
* Should commit changes to backend store before release
|
||||
* delete lock
|
||||
*/
|
||||
graph.graph().tx().commit();
|
||||
// Remove edge label
|
||||
removeSchema(schemaTx, edgeLabel);
|
||||
} catch (Throwable e) {
|
||||
schemaTx.updateSchemaStatus(edgeLabel, SchemaStatus.UNDELETED);
|
||||
throw e;
|
||||
|
|
@ -41,7 +41,7 @@ import com.baidu.hugegraph.type.define.SchemaStatus;
|
|||
import com.baidu.hugegraph.util.LockUtil;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
|
||||
public class RebuildIndexCallable extends SchemaCallable {
|
||||
public class IndexLabelRebuildJob extends SchemaJob {
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
|
|
@ -27,7 +27,7 @@ import com.baidu.hugegraph.schema.IndexLabel;
|
|||
import com.baidu.hugegraph.type.define.SchemaStatus;
|
||||
import com.baidu.hugegraph.util.LockUtil;
|
||||
|
||||
public class IndexLabelRemoveCallable extends SchemaCallable {
|
||||
public class IndexLabelRemoveJob extends SchemaJob {
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
|
|
@ -48,6 +48,11 @@ public class IndexLabelRemoveCallable extends SchemaCallable {
|
|||
if (indexLabel == null) {
|
||||
return;
|
||||
}
|
||||
if (indexLabel.status().deleting()) {
|
||||
LOG.info("The index label '{}' has been in {} status, " +
|
||||
"please check if it's expected to delete it again",
|
||||
indexLabel, indexLabel.status());
|
||||
}
|
||||
LockUtil.Locks locks = new LockUtil.Locks(graph.name());
|
||||
try {
|
||||
locks.lockWrites(LockUtil.INDEX_LABEL_DELETE, id);
|
||||
|
|
@ -60,14 +65,15 @@ public class IndexLabelRemoveCallable extends SchemaCallable {
|
|||
// Remove index data
|
||||
// TODO: use event to replace direct call
|
||||
graphTx.removeIndex(indexLabel);
|
||||
removeSchema(schemaTx, indexLabel);
|
||||
/*
|
||||
* Should commit changes to backend store before release
|
||||
* delete lock
|
||||
*/
|
||||
graph.graph().tx().commit();
|
||||
// Remove index label
|
||||
removeSchema(schemaTx, indexLabel);
|
||||
} catch (Throwable e) {
|
||||
schemaTx.updateSchemaStatus(indexLabel, SchemaStatus.INVALID);
|
||||
schemaTx.updateSchemaStatus(indexLabel, SchemaStatus.UNDELETED);
|
||||
throw e;
|
||||
}
|
||||
} finally {
|
||||
|
|
@ -27,7 +27,7 @@ import com.baidu.hugegraph.schema.IndexLabel;
|
|||
import com.baidu.hugegraph.type.define.SchemaStatus;
|
||||
import com.baidu.hugegraph.util.LockUtil;
|
||||
|
||||
public class OlapPropertyKeyClearCallable extends IndexLabelRemoveCallable {
|
||||
public class OlapPropertyKeyClearJob extends IndexLabelRemoveJob {
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
|
|
@ -22,7 +22,7 @@ package com.baidu.hugegraph.job.schema;
|
|||
import com.baidu.hugegraph.backend.tx.SchemaTransaction;
|
||||
import com.baidu.hugegraph.schema.PropertyKey;
|
||||
|
||||
public class OlapPropertyKeyCreateCallable extends SchemaCallable {
|
||||
public class OlapPropertyKeyCreateJob extends SchemaJob {
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
|
|
@ -23,8 +23,7 @@ import com.baidu.hugegraph.backend.id.Id;
|
|||
import com.baidu.hugegraph.backend.tx.SchemaTransaction;
|
||||
import com.baidu.hugegraph.schema.PropertyKey;
|
||||
|
||||
public class OlapPropertyKeyRemoveCallable
|
||||
extends OlapPropertyKeyClearCallable {
|
||||
public class OlapPropertyKeyRemoveJob extends OlapPropertyKeyClearJob {
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
|
|
@ -3,6 +3,8 @@ package com.baidu.hugegraph.job.schema;
|
|||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import com.baidu.hugegraph.backend.id.Id;
|
||||
import com.baidu.hugegraph.backend.id.IdGenerator;
|
||||
import com.baidu.hugegraph.backend.tx.SchemaTransaction;
|
||||
|
|
@ -13,8 +15,9 @@ import com.baidu.hugegraph.schema.SchemaLabel;
|
|||
import com.baidu.hugegraph.schema.VertexLabel;
|
||||
import com.baidu.hugegraph.type.HugeType;
|
||||
import com.baidu.hugegraph.util.E;
|
||||
import com.baidu.hugegraph.util.Log;
|
||||
|
||||
public abstract class SchemaCallable extends SysJob<Object> {
|
||||
public abstract class SchemaJob extends SysJob<Object> {
|
||||
|
||||
public static final String REMOVE_SCHEMA = "remove_schema";
|
||||
public static final String REBUILD_INDEX = "rebuild_index";
|
||||
|
|
@ -23,6 +26,8 @@ public abstract class SchemaCallable extends SysJob<Object> {
|
|||
public static final String CLEAR_OLAP = "clear_olap";
|
||||
public static final String REMOVE_OLAP = "remove_olap";
|
||||
|
||||
protected static final Logger LOG = Log.logger(SchemaJob.class);
|
||||
|
||||
private static final String SPLITOR = ":";
|
||||
|
||||
protected HugeType schemaType() {
|
||||
|
|
@ -33,7 +33,7 @@ import com.baidu.hugegraph.type.define.SchemaStatus;
|
|||
import com.baidu.hugegraph.util.LockUtil;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
|
||||
public class VertexLabelRemoveCallable extends SchemaCallable {
|
||||
public class VertexLabelRemoveJob extends SchemaJob {
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
|
|
@ -54,7 +54,13 @@ public class VertexLabelRemoveCallable extends SchemaCallable {
|
|||
if (vertexLabel == null) {
|
||||
return;
|
||||
}
|
||||
if (vertexLabel.status().deleting()) {
|
||||
LOG.info("The vertex label '{}' has been in {} status, " +
|
||||
"please check if it's expected to delete it again",
|
||||
vertexLabel, vertexLabel.status());
|
||||
}
|
||||
|
||||
// Check no edge label use the vertex label
|
||||
List<EdgeLabel> edgeLabels = schemaTx.getEdgeLabels();
|
||||
for (EdgeLabel edgeLabel : edgeLabels) {
|
||||
if (edgeLabel.linkWithLabel(id)) {
|
||||
|
|
@ -76,19 +82,21 @@ public class VertexLabelRemoveCallable extends SchemaCallable {
|
|||
schemaTx.updateSchemaStatus(vertexLabel, SchemaStatus.DELETING);
|
||||
try {
|
||||
for (Id ilId : indexLabelIds) {
|
||||
IndexLabelRemoveCallable.removeIndexLabel(graph, ilId);
|
||||
IndexLabelRemoveJob.removeIndexLabel(graph, ilId);
|
||||
}
|
||||
// TODO: use event to replace direct call
|
||||
// Deleting a vertex will automatically deletes the held edge
|
||||
graphTx.removeVertices(vertexLabel);
|
||||
removeSchema(schemaTx, vertexLabel);
|
||||
/*
|
||||
* Should commit changes to backend store before release
|
||||
* delete lock
|
||||
*/
|
||||
graph.graph().tx().commit();
|
||||
// Remove vertex label
|
||||
removeSchema(schemaTx, vertexLabel);
|
||||
} catch (Throwable e) {
|
||||
schemaTx.updateSchemaStatus(vertexLabel, SchemaStatus.UNDELETED);
|
||||
schemaTx.updateSchemaStatus(vertexLabel,
|
||||
SchemaStatus.UNDELETED);
|
||||
throw e;
|
||||
}
|
||||
} finally {
|
||||
|
|
@ -50,7 +50,8 @@ public final class TaskManager {
|
|||
"server-info-db-worker-%d";
|
||||
public static final String TASK_SCHEDULER = "task-scheduler-%d";
|
||||
|
||||
protected static final int SCHEDULE_PERIOD = 3; // Unit second
|
||||
protected static final int SCHEDULE_PERIOD = 1; // Unit second
|
||||
|
||||
private static final int THREADS = 4;
|
||||
private static final TaskManager MANAGER = new TaskManager(THREADS);
|
||||
|
||||
|
|
|
|||
|
|
@ -82,8 +82,8 @@ public class HugeTraverser {
|
|||
public static final String DEFAULT_MAX_DEGREE = "10000";
|
||||
public static final String DEFAULT_SKIP_DEGREE = "100000";
|
||||
public static final String DEFAULT_SAMPLE = "100";
|
||||
public static final String DEFAULT_MAX_DEPTH = "50";
|
||||
public static final String DEFAULT_WEIGHT = "0";
|
||||
public static final int DEFAULT_MAX_DEPTH = 5000;
|
||||
|
||||
protected static final int MAX_VERTICES = 10;
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@
|
|||
package com.baidu.hugegraph.traversal.algorithm;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.tinkerpop.gremlin.structure.Edge;
|
||||
|
||||
|
|
@ -52,7 +51,9 @@ public class PathsTraverser extends HugeTraverser {
|
|||
sourceDir == targetDir.opposite(),
|
||||
"Source direction must equal to target direction" +
|
||||
" or opposite to target direction");
|
||||
checkPositive(depth, "max depth");
|
||||
E.checkArgument(depth > 0 && depth <= DEFAULT_MAX_DEPTH,
|
||||
"The depth must be in (0, %s], but got: %s",
|
||||
DEFAULT_MAX_DEPTH, depth);
|
||||
checkDegree(degree);
|
||||
checkCapacity(capacity);
|
||||
checkLimit(limit);
|
||||
|
|
@ -124,9 +125,6 @@ public class PathsTraverser extends HugeTraverser {
|
|||
PathSet results = this.record.findPath(target, null,
|
||||
true, false);
|
||||
for (Path path : results) {
|
||||
if (Objects.equals(target, targetV)) {
|
||||
continue;
|
||||
}
|
||||
this.paths.add(path);
|
||||
if (this.reachLimit()) {
|
||||
return;
|
||||
|
|
@ -160,9 +158,6 @@ public class PathsTraverser extends HugeTraverser {
|
|||
PathSet results = this.record.findPath(target, null,
|
||||
true, false);
|
||||
for (Path path : results) {
|
||||
if (Objects.equals(target, sourceV)) {
|
||||
continue;
|
||||
}
|
||||
this.paths.add(path);
|
||||
if (this.reachLimit()) {
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,11 @@ LOG_PATH=${HOME_PATH}/logs
|
|||
|
||||
. ${BIN_PATH}/util.sh
|
||||
|
||||
#export HUGEGRAPH_URL=
|
||||
#export HUGEGRAPH_GRAPH=
|
||||
#export HUGEGRAPH_USERNAME=
|
||||
#export HUGEGRAPH_PASSWORD=
|
||||
|
||||
function print_usage() {
|
||||
echo "usage: raft-tools.sh [options]"
|
||||
echo "options: "
|
||||
|
|
@ -26,9 +31,6 @@ function print_usage() {
|
|||
echo " -h,--help display help information"
|
||||
}
|
||||
|
||||
GRAPH="hugegraph"
|
||||
ENDPOINT=""
|
||||
|
||||
if [[ $# -lt 2 ]]; then
|
||||
print_usage
|
||||
exit 0
|
||||
|
|
@ -36,56 +38,58 @@ fi
|
|||
|
||||
function list_peers() {
|
||||
local graph=$1
|
||||
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
|
||||
local url=${rest_server_url}/graphs/${graph}/raft/list_peers
|
||||
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/list_peers
|
||||
|
||||
curl ${url}
|
||||
curl ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
|
||||
}
|
||||
|
||||
function get_leader() {
|
||||
local graph=$1
|
||||
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
|
||||
local url=${rest_server_url}/graphs/${graph}/raft/get_leader
|
||||
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/get_leader
|
||||
|
||||
curl ${url}
|
||||
curl ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
|
||||
}
|
||||
|
||||
function set_leader() {
|
||||
local graph=$1
|
||||
local endpoint=$2
|
||||
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
|
||||
local url=${rest_server_url}/graphs/${graph}/raft/set_leader?endpoint=${endpoint}
|
||||
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/set_leader?endpoint=${endpoint}
|
||||
|
||||
curl -X POST ${url}
|
||||
curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
|
||||
}
|
||||
|
||||
function transfer_leader() {
|
||||
local graph=$1
|
||||
local endpoint=$2
|
||||
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
|
||||
local url=${rest_server_url}/graphs/${graph}/raft/transfer_leader?endpoint=${endpoint}
|
||||
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/transfer_leader?endpoint=${endpoint}
|
||||
|
||||
curl -X POST ${url}
|
||||
curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
|
||||
}
|
||||
|
||||
function add_peer() {
|
||||
local graph=$1
|
||||
local endpoint=$2
|
||||
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
|
||||
local url=${rest_server_url}/graphs/${graph}/raft/add_peer?endpoint=${endpoint}
|
||||
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/add_peer?endpoint=${endpoint}
|
||||
|
||||
curl -X POST ${url}
|
||||
curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
|
||||
}
|
||||
|
||||
function remove_peer() {
|
||||
local graph=$1
|
||||
local endpoint=$2
|
||||
local rest_server_url=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
|
||||
local url=${rest_server_url}/graphs/${graph}/raft/remove_peer?endpoint=${endpoint}
|
||||
local url=${HUGEGRAPH_URL}/graphs/${graph}/raft/remove_peer?endpoint=${endpoint}
|
||||
|
||||
curl -X POST ${url}
|
||||
curl -X POST ${url} --user ${HUGEGRAPH_USERNAME}:${HUGEGRAPH_PASSWORD}
|
||||
}
|
||||
|
||||
if [ "${HUGEGRAPH_URL}" = "" ]; then
|
||||
HUGEGRAPH_URL=`read_property ${CONF_PATH}/rest-server.properties restserver.url`
|
||||
fi
|
||||
|
||||
if [ "${HUGEGRAPH_GRAPH}" = "" ]; then
|
||||
HUGEGRAPH_GRAPH="hugegraph"
|
||||
fi
|
||||
|
||||
case $1 in
|
||||
# help
|
||||
--help|-h)
|
||||
|
|
|
|||
|
|
@ -7,14 +7,15 @@ GC_OPTION=""
|
|||
USER_OPTION=""
|
||||
SERVER_STARTUP_TIMEOUT_S=30
|
||||
|
||||
while getopts "g:m:s:j:v" arg; do
|
||||
while getopts "g:m:s:j:t:v" arg; do
|
||||
case ${arg} in
|
||||
g) GC_OPTION="$OPTARG" ;;
|
||||
m) OPEN_MONITOR="$OPTARG" ;;
|
||||
s) OPEN_SECURITY_CHECK="$OPTARG" ;;
|
||||
j) USER_OPTION="$OPTARG" ;;
|
||||
t) SERVER_STARTUP_TIMEOUT_S="$OPTARG" ;;
|
||||
v) VERBOSE="verbose" ;;
|
||||
?) echo "USAGE: $0 [-g g1] [-m true|false] [-s true|false] [-j xxx] [-v]" && exit 1 ;;
|
||||
?) echo "USAGE: $0 [-g g1] [-m true|false] [-s true|false] [-j java_options] [-t timeout] [-v]" && exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ function wait_for_startup() {
|
|||
now_s=`date '+%s'`
|
||||
done
|
||||
|
||||
echo "The operation timed out when attempting to connect to $server_url" >&2
|
||||
echo "The operation timed out(${timeout_s}s) when attempting to connect to $server_url" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
set -ev
|
||||
|
||||
BACKEND=$1
|
||||
JACOCO_PORT=$2
|
||||
JACOCO_REPORT_FILE=$3
|
||||
|
||||
OPTION_CLASS_FILES_BACKEND="--classfiles hugegraph-$BACKEND/target/classes/com/baidu/hugegraph"
|
||||
if [ "$BACKEND" == "memory" ]; then
|
||||
|
|
@ -11,10 +13,10 @@ if [ "$BACKEND" == "memory" ]; then
|
|||
fi
|
||||
|
||||
cd hugegraph-test
|
||||
mvn jacoco:dump@pull-test-data -Dapp.host=localhost -Dapp.port=36320 -Dskip.dump=false
|
||||
mvn jacoco:dump@pull-test-data -Dapp.host=localhost -Dapp.port=$JACOCO_PORT -Dskip.dump=false
|
||||
cd ../
|
||||
java -jar $TRAVIS_DIR/jacococli.jar report hugegraph-test/target/jacoco-it.exec \
|
||||
--classfiles hugegraph-dist/target/classes/com/baidu/hugegraph \
|
||||
--classfiles hugegraph-api/target/classes/com/baidu/hugegraph \
|
||||
--classfiles hugegraph-core/target/classes/com/baidu/hugegraph \
|
||||
$OPTION_CLASS_FILES_BACKEND --xml report.xml
|
||||
${OPTION_CLASS_FILES_BACKEND} --xml "${JACOCO_REPORT_FILE}"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy
|
||||
#gremlin.graph=com.baidu.hugegraph.HugeFactory
|
||||
|
||||
store=hugegraph
|
||||
|
||||
backend=rocksdb
|
||||
serializer=binary
|
||||
|
||||
rocksdb.data_path=rocksdb-data-raft1
|
||||
rocksdb.wal_path=rocksdb-data-raft1
|
||||
|
||||
raft.mode=true
|
||||
raft.safe_read=true
|
||||
raft.use_snapshot=false
|
||||
raft.endpoint=127.0.0.1:8281
|
||||
raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283
|
||||
raft.path=rocksdb-raftlog1
|
||||
raft.use_replicator_pipeline=true
|
||||
raft.election_timeout=10000
|
||||
raft.snapshot_interval=3600
|
||||
raft.backend_threads=48
|
||||
raft.read_index_threads=8
|
||||
raft.read_strategy=ReadOnlyLeaseBased
|
||||
raft.queue_size=16384
|
||||
raft.queue_publish_timeout=60
|
||||
raft.apply_batch=1
|
||||
raft.rpc_threads=8
|
||||
raft.rpc_connect_timeout=5000
|
||||
raft.rpc_timeout=60000
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
# host and port of gremlin server, need to be consistent with host and port in rest-server.properties
|
||||
#host: 127.0.0.1
|
||||
port: 8181
|
||||
|
||||
# timeout in ms of gremlin query
|
||||
scriptEvaluationTimeout: 30000
|
||||
|
||||
channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer
|
||||
# don't set graph at here, this happens after support for dynamically adding graph
|
||||
graphs: {
|
||||
}
|
||||
scriptEngines: {
|
||||
gremlin-groovy: {
|
||||
plugins: {
|
||||
com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {},
|
||||
org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
|
||||
org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {
|
||||
classImports: [
|
||||
java.lang.Math,
|
||||
com.baidu.hugegraph.backend.id.IdGenerator,
|
||||
com.baidu.hugegraph.type.define.Directions,
|
||||
com.baidu.hugegraph.type.define.NodeRole,
|
||||
com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.CountTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.HugeTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.KneighborTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.KoutTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.PathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep,
|
||||
com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep,
|
||||
com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep,
|
||||
com.baidu.hugegraph.traversal.optimize.Text,
|
||||
com.baidu.hugegraph.traversal.optimize.TraversalUtil,
|
||||
com.baidu.hugegraph.util.DateUtil
|
||||
],
|
||||
methodImports: [java.lang.Math#*]
|
||||
},
|
||||
org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {
|
||||
files: [scripts/empty-sample.groovy]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
serializers:
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
metrics: {
|
||||
consoleReporter: {enabled: false, interval: 180000},
|
||||
csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv},
|
||||
jmxReporter: {enabled: false},
|
||||
slf4jReporter: {enabled: false, interval: 180000},
|
||||
gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST},
|
||||
graphiteReporter: {enabled: false, interval: 180000}
|
||||
}
|
||||
maxInitialLineLength: 4096
|
||||
maxHeaderSize: 8192
|
||||
maxChunkSize: 8192
|
||||
maxContentLength: 65536
|
||||
maxAccumulationBufferComponents: 1024
|
||||
resultIterationBatchSize: 64
|
||||
writeBufferLowWaterMark: 32768
|
||||
writeBufferHighWaterMark: 65536
|
||||
ssl: {
|
||||
enabled: false
|
||||
}
|
||||
authentication: {
|
||||
authenticator: com.baidu.hugegraph.auth.StandardAuthenticator,
|
||||
#authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler,
|
||||
authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler,
|
||||
config: {tokens: conf/rest-server.properties}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
restserver.url=http://127.0.0.1:8080
|
||||
gremlinserver.url=http://127.0.0.1:8181
|
||||
graphs=conf/graphs
|
||||
auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator
|
||||
|
||||
rpc.server_host=127.0.0.1
|
||||
rpc.server_port=8091
|
||||
rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093
|
||||
|
||||
server.id=server1
|
||||
server.role=master
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy
|
||||
#gremlin.graph=com.baidu.hugegraph.HugeFactory
|
||||
|
||||
store=hugegraph
|
||||
|
||||
backend=rocksdb
|
||||
serializer=binary
|
||||
|
||||
rocksdb.data_path=rocksdb-data-raft2
|
||||
rocksdb.wal_path=rocksdb-data-raft2
|
||||
|
||||
raft.mode=true
|
||||
raft.safe_read=true
|
||||
raft.use_snapshot=false
|
||||
raft.endpoint=127.0.0.1:8282
|
||||
raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283
|
||||
raft.path=rocksdb-raftlog2
|
||||
raft.use_replicator_pipeline=true
|
||||
raft.election_timeout=10000
|
||||
raft.snapshot_interval=3600
|
||||
raft.backend_threads=48
|
||||
raft.read_index_threads=8
|
||||
raft.read_strategy=ReadOnlyLeaseBased
|
||||
raft.queue_size=16384
|
||||
raft.queue_publish_timeout=60
|
||||
raft.apply_batch=1
|
||||
raft.rpc_threads=8
|
||||
raft.rpc_connect_timeout=5000
|
||||
raft.rpc_timeout=60000
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
# host and port of gremlin server
|
||||
#host: 127.0.0.1
|
||||
port: 8182
|
||||
|
||||
# timeout in ms of gremlin query
|
||||
scriptEvaluationTimeout: 30000
|
||||
|
||||
channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer
|
||||
# don't set graph at here, this happens after support for dynamically adding graph
|
||||
graphs: {
|
||||
}
|
||||
scriptEngines: {
|
||||
gremlin-groovy: {
|
||||
plugins: {
|
||||
com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {},
|
||||
org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
|
||||
org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {
|
||||
classImports: [
|
||||
java.lang.Math,
|
||||
com.baidu.hugegraph.backend.id.IdGenerator,
|
||||
com.baidu.hugegraph.type.define.Directions,
|
||||
com.baidu.hugegraph.type.define.NodeRole,
|
||||
com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.CountTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.HugeTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.KneighborTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.KoutTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.PathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep,
|
||||
com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep,
|
||||
com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep,
|
||||
com.baidu.hugegraph.traversal.optimize.Text,
|
||||
com.baidu.hugegraph.traversal.optimize.TraversalUtil,
|
||||
com.baidu.hugegraph.util.DateUtil
|
||||
],
|
||||
methodImports: [java.lang.Math#*]
|
||||
},
|
||||
org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {
|
||||
files: [scripts/empty-sample.groovy]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
serializers:
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
metrics: {
|
||||
consoleReporter: {enabled: false, interval: 180000},
|
||||
csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv},
|
||||
jmxReporter: {enabled: false},
|
||||
slf4jReporter: {enabled: false, interval: 180000},
|
||||
gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST},
|
||||
graphiteReporter: {enabled: false, interval: 180000}
|
||||
}
|
||||
maxInitialLineLength: 4096
|
||||
maxHeaderSize: 8192
|
||||
maxChunkSize: 8192
|
||||
maxContentLength: 65536
|
||||
maxAccumulationBufferComponents: 1024
|
||||
resultIterationBatchSize: 64
|
||||
writeBufferLowWaterMark: 32768
|
||||
writeBufferHighWaterMark: 65536
|
||||
ssl: {
|
||||
enabled: false
|
||||
}
|
||||
authentication: {
|
||||
authenticator: com.baidu.hugegraph.auth.StandardAuthenticator,
|
||||
#authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler,
|
||||
authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler,
|
||||
config: {tokens: conf/rest-server.properties}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
restserver.url=http://127.0.0.1:8082
|
||||
gremlinserver.url=http://127.0.0.1:8182
|
||||
graphs=conf/graphs
|
||||
auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator
|
||||
|
||||
rpc.server_host=127.0.0.1
|
||||
rpc.server_port=8092
|
||||
rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093
|
||||
|
||||
server.id=server2
|
||||
server.role=worker
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy
|
||||
#gremlin.graph=com.baidu.hugegraph.HugeFactory
|
||||
|
||||
store=hugegraph
|
||||
|
||||
backend=rocksdb
|
||||
serializer=binary
|
||||
|
||||
rocksdb.data_path=rocksdb-data-raft3
|
||||
rocksdb.wal_path=rocksdb-data-raft3
|
||||
|
||||
raft.mode=true
|
||||
raft.safe_read=true
|
||||
raft.use_snapshot=false
|
||||
raft.endpoint=127.0.0.1:8283
|
||||
raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283
|
||||
raft.path=rocksdb-raftlog3
|
||||
raft.use_replicator_pipeline=true
|
||||
raft.election_timeout=10000
|
||||
raft.snapshot_interval=3600
|
||||
raft.backend_threads=48
|
||||
raft.read_index_threads=8
|
||||
raft.read_strategy=ReadOnlyLeaseBased
|
||||
raft.queue_size=16384
|
||||
raft.queue_publish_timeout=60
|
||||
raft.apply_batch=1
|
||||
raft.rpc_threads=8
|
||||
raft.rpc_connect_timeout=5000
|
||||
raft.rpc_timeout=60000
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
# host and port of gremlin server
|
||||
#host: 127.0.0.1
|
||||
port: 8183
|
||||
|
||||
# timeout in ms of gremlin query
|
||||
scriptEvaluationTimeout: 30000
|
||||
|
||||
channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer
|
||||
# don't set graph at here, this happens after support for dynamically adding graph
|
||||
graphs: {
|
||||
}
|
||||
scriptEngines: {
|
||||
gremlin-groovy: {
|
||||
plugins: {
|
||||
com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {},
|
||||
org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
|
||||
org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {
|
||||
classImports: [
|
||||
java.lang.Math,
|
||||
com.baidu.hugegraph.backend.id.IdGenerator,
|
||||
com.baidu.hugegraph.type.define.Directions,
|
||||
com.baidu.hugegraph.type.define.NodeRole,
|
||||
com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.CountTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.HugeTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.KneighborTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.KoutTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.PathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser,
|
||||
com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep,
|
||||
com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep,
|
||||
com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep,
|
||||
com.baidu.hugegraph.traversal.optimize.Text,
|
||||
com.baidu.hugegraph.traversal.optimize.TraversalUtil,
|
||||
com.baidu.hugegraph.util.DateUtil
|
||||
],
|
||||
methodImports: [java.lang.Math#*]
|
||||
},
|
||||
org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {
|
||||
files: [scripts/empty-sample.groovy]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
serializers:
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0,
|
||||
config: {
|
||||
serializeResultToString: false,
|
||||
ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
|
||||
}
|
||||
}
|
||||
metrics: {
|
||||
consoleReporter: {enabled: false, interval: 180000},
|
||||
csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv},
|
||||
jmxReporter: {enabled: false},
|
||||
slf4jReporter: {enabled: false, interval: 180000},
|
||||
gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST},
|
||||
graphiteReporter: {enabled: false, interval: 180000}
|
||||
}
|
||||
maxInitialLineLength: 4096
|
||||
maxHeaderSize: 8192
|
||||
maxChunkSize: 8192
|
||||
maxContentLength: 65536
|
||||
maxAccumulationBufferComponents: 1024
|
||||
resultIterationBatchSize: 64
|
||||
writeBufferLowWaterMark: 32768
|
||||
writeBufferHighWaterMark: 65536
|
||||
ssl: {
|
||||
enabled: false
|
||||
}
|
||||
authentication: {
|
||||
authenticator: com.baidu.hugegraph.auth.StandardAuthenticator,
|
||||
#authenticationHandler: org.apache.tinkerpop.gremlin.server.handler.SaslAndHttpBasicAuthenticationHandler,
|
||||
authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler,
|
||||
config: {tokens: conf/rest-server.properties}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
restserver.url=http://127.0.0.1:8083
|
||||
gremlinserver.url=http://127.0.0.1:8183
|
||||
graphs=conf/graphs
|
||||
auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator
|
||||
|
||||
rpc.server_host=127.0.0.1
|
||||
rpc.server_port=8093
|
||||
rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093
|
||||
|
||||
server.id=server3
|
||||
server.role=worker
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ev
|
||||
|
||||
BACKEND=$1
|
||||
REPORT_DIR=$2
|
||||
REPORT_FILE=$REPORT_DIR/jacoco-api-test.xml
|
||||
|
||||
TRAVIS_DIR=`dirname $0`
|
||||
VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
|
||||
SERVER_DIR=hugegraph-$VERSION
|
||||
RAFT1_DIR=hugegraph-raft1
|
||||
RAFT2_DIR=hugegraph-raft2
|
||||
RAFT3_DIR=hugegraph-raft3
|
||||
CONF=$SERVER_DIR/conf/graphs/hugegraph.properties
|
||||
REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties
|
||||
GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml
|
||||
|
||||
JACOCO_PORT=36320
|
||||
RAFT_TOOLS=$RAFT1_DIR/bin/raft-tools.sh
|
||||
RAFT_LEADER="127.0.0.1:8281"
|
||||
|
||||
mvn package -DskipTests
|
||||
|
||||
# mkdir for each raft-server
|
||||
cp -r $SERVER_DIR $RAFT1_DIR
|
||||
cp -r $SERVER_DIR $RAFT2_DIR
|
||||
cp -r $SERVER_DIR $RAFT3_DIR
|
||||
|
||||
# config raft-server (must keep '/.')
|
||||
cp -rf $TRAVIS_DIR/conf-raft1/. $RAFT1_DIR/conf/
|
||||
cp -rf $TRAVIS_DIR/conf-raft2/. $RAFT2_DIR/conf/
|
||||
cp -rf $TRAVIS_DIR/conf-raft3/. $RAFT3_DIR/conf/
|
||||
|
||||
# start server
|
||||
$TRAVIS_DIR/start-server.sh $RAFT1_DIR $BACKEND $JACOCO_PORT || (cat $RAFT1_DIR/logs/hugegraph-server.log && exit 1) &
|
||||
$TRAVIS_DIR/start-server.sh $RAFT2_DIR $BACKEND || (cat $RAFT2_DIR/logs/hugegraph-server.log && exit 1) &
|
||||
$TRAVIS_DIR/start-server.sh $RAFT3_DIR $BACKEND || (cat $RAFT3_DIR/logs/hugegraph-server.log && exit 1)
|
||||
|
||||
export HUGEGRAPH_USERNAME=admin
|
||||
export HUGEGRAPH_PASSWORD=pa
|
||||
$RAFT_TOOLS --set-leader "hugegraph" "$RAFT_LEADER"
|
||||
|
||||
# run api-test
|
||||
mvn test -P api-test,$BACKEND || (cat $RAFT1_DIR/logs/hugegraph-server.log && exit 1)
|
||||
|
||||
$TRAVIS_DIR/build-report.sh $BACKEND $JACOCO_PORT $REPORT_FILE
|
||||
|
||||
# stop server
|
||||
$TRAVIS_DIR/stop-server.sh $RAFT1_DIR
|
||||
$TRAVIS_DIR/stop-server.sh $RAFT2_DIR
|
||||
$TRAVIS_DIR/stop-server.sh $RAFT3_DIR
|
||||
|
|
@ -3,6 +3,8 @@
|
|||
set -ev
|
||||
|
||||
BACKEND=$1
|
||||
REPORT_DIR=$2
|
||||
REPORT_FILE=$REPORT_DIR/jacoco-api-test-for-raft.xml
|
||||
|
||||
TRAVIS_DIR=`dirname $0`
|
||||
VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
|
||||
|
|
@ -10,6 +12,7 @@ SERVER_DIR=hugegraph-$VERSION
|
|||
CONF=$SERVER_DIR/conf/graphs/hugegraph.properties
|
||||
REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties
|
||||
GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml
|
||||
JACOCO_PORT=36320
|
||||
|
||||
mvn package -DskipTests
|
||||
|
||||
|
|
@ -28,9 +31,13 @@ authentication: {
|
|||
config: {tokens: conf/rest-server.properties}
|
||||
}" >> $GREMLIN_SERVER_CONF
|
||||
|
||||
$TRAVIS_DIR/start-server.sh $SERVER_DIR $BACKEND || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1)
|
||||
# start server
|
||||
$TRAVIS_DIR/start-server.sh $SERVER_DIR $BACKEND $JACOCO_PORT || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1)
|
||||
|
||||
# run api-test
|
||||
mvn test -P api-test,$BACKEND || (cat $SERVER_DIR/logs/hugegraph-server.log && exit 1)
|
||||
$TRAVIS_DIR/build-report.sh $BACKEND
|
||||
$TRAVIS_DIR/stop-server.sh
|
||||
|
||||
$TRAVIS_DIR/build-report.sh $BACKEND $JACOCO_PORT $REPORT_FILE
|
||||
|
||||
# stop server
|
||||
$TRAVIS_DIR/stop-server.sh $SERVER_DIR
|
||||
|
|
|
|||
|
|
@ -6,14 +6,21 @@ HOME_DIR=$(pwd)
|
|||
TRAVIS_DIR=$(dirname $0)
|
||||
BASE_DIR=$1
|
||||
BACKEND=$2
|
||||
JACOCO_PORT=$3
|
||||
|
||||
JACOCO_JAR=${HOME_DIR}/${TRAVIS_DIR}/jacocoagent.jar
|
||||
|
||||
BIN=$BASE_DIR/bin
|
||||
CONF=$BASE_DIR/conf/graphs/hugegraph.properties
|
||||
REST_CONF=$BASE_DIR/conf/rest-server.properties
|
||||
GREMLIN_CONF=$BASE_DIR/conf/gremlin-server.yaml
|
||||
|
||||
declare -A backend_serializer_map=(["memory"]="text" ["cassandra"]="cassandra" \
|
||||
["scylladb"]="scylladb" ["mysql"]="mysql" \
|
||||
["hbase"]="hbase" ["rocksdb"]="binary" \
|
||||
declare -A backend_serializer_map=(["memory"]="text" \
|
||||
["cassandra"]="cassandra" \
|
||||
["scylladb"]="scylladb" \
|
||||
["mysql"]="mysql" \
|
||||
["hbase"]="hbase" \
|
||||
["rocksdb"]="binary" \
|
||||
["postgresql"]="postgresql")
|
||||
|
||||
SERIALIZER=${backend_serializer_map[$BACKEND]}
|
||||
|
|
@ -37,5 +44,9 @@ fi
|
|||
# Append schema.sync_deletion=true to config file
|
||||
echo "schema.sync_deletion=true" >> $CONF
|
||||
|
||||
AGENT_JAR=${HOME_DIR}/${TRAVIS_DIR}/jacocoagent.jar
|
||||
echo -e "pa" | $BIN/init-store.sh && $BIN/start-hugegraph.sh -j "-javaagent:${AGENT_JAR}=includes=*,port=36320,destfile=jacoco-it.exec,output=tcpserver" -v
|
||||
JACOCO_OPTION=""
|
||||
if [ -n "$JACOCO_PORT" ]; then
|
||||
JACOCO_OPTION="-javaagent:${JACOCO_JAR}=includes=*,port=${JACOCO_PORT},destfile=jacoco-it.exec,output=tcpserver"
|
||||
fi
|
||||
|
||||
echo -e "pa" | $BIN/init-store.sh && $BIN/start-hugegraph.sh -j "$JACOCO_OPTION" -t 60 -v
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@
|
|||
|
||||
set -ev
|
||||
|
||||
VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
|
||||
BASE_DIR=hugegraph-$VERSION
|
||||
BASE_DIR=$1
|
||||
BIN=$BASE_DIR/bin
|
||||
VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
|
||||
|
||||
$BIN/stop-hugegraph.sh
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import java.util.Map;
|
|||
|
||||
import javax.ws.rs.core.Response;
|
||||
|
||||
import org.junit.Assume;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.baidu.hugegraph.testutil.Assert;
|
||||
|
|
@ -86,12 +87,29 @@ public class GremlinApiTest extends BaseApiTest {
|
|||
@Test
|
||||
public void testClearAndInit() {
|
||||
String body = "{"
|
||||
+ "\"gremlin\":\"hugegraph.backendStoreFeatures()"
|
||||
+ " .supportsSharedStorage();\","
|
||||
+ "\"bindings\":{},"
|
||||
+ "\"language\":\"gremlin-groovy\","
|
||||
+ "\"aliases\":{\"g\":\"__g_hugegraph\"}}";
|
||||
String content = assertResponseStatus(200, client().post(path, body));
|
||||
Map<?, ?> result = assertJsonContains(content, "result");
|
||||
@SuppressWarnings({ "unchecked" })
|
||||
Object data = ((List<Object>) assertMapContains(result, "data")).get(0);
|
||||
boolean supportsSharedStorage = (boolean) data;
|
||||
Assume.assumeTrue("Can't clear non-shared-storage backend",
|
||||
supportsSharedStorage);
|
||||
|
||||
body = "{"
|
||||
+ "\"gremlin\":\""
|
||||
+ "def auth = hugegraph.hugegraph().authManager();"
|
||||
+ "def admin = auth.findUser('admin');"
|
||||
+ "hugegraph.clearBackend();"
|
||||
+ "hugegraph.initBackend();"
|
||||
+ "auth.createUser(admin);\","
|
||||
+ " if (!hugegraph.backendStoreFeatures()"
|
||||
+ " .supportsSharedStorage())"
|
||||
+ " return;"
|
||||
+ " def auth = hugegraph.hugegraph().authManager();"
|
||||
+ " def admin = auth.findUser('admin');"
|
||||
+ " hugegraph.clearBackend();"
|
||||
+ " hugegraph.initBackend();"
|
||||
+ " auth.createUser(admin);\","
|
||||
+ "\"bindings\":{},"
|
||||
+ "\"language\":\"gremlin-groovy\","
|
||||
+ "\"aliases\":{\"g\":\"__g_hugegraph\"}}";
|
||||
|
|
@ -99,7 +117,7 @@ public class GremlinApiTest extends BaseApiTest {
|
|||
|
||||
body = "{"
|
||||
+ "\"gremlin\":\"hugegraph.serverStarted("
|
||||
+ "IdGenerator.of('server1'), NodeRole.MASTER)\","
|
||||
+ " IdGenerator.of('server1'), NodeRole.MASTER)\","
|
||||
+ "\"bindings\":{},"
|
||||
+ "\"language\":\"gremlin-groovy\","
|
||||
+ "\"aliases\":{\"g\":\"__g_hugegraph\"}}";
|
||||
|
|
|
|||
|
|
@ -27,17 +27,48 @@ import javax.ws.rs.core.Response;
|
|||
|
||||
import org.apache.commons.lang.ArrayUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.baidu.hugegraph.util.JsonUtil;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
public class ProjectApiTest extends BaseApiTest {
|
||||
|
||||
private final static String path = "graphs/hugegraph/auth/projects";
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
BaseApiTest.truncate();
|
||||
@Override
|
||||
@After
|
||||
public void teardown() throws Exception {
|
||||
Response resp = client().get(path);
|
||||
String respBody = assertResponseStatus(200, resp);
|
||||
List<?> projects = readList(respBody, "projects", Map.class);
|
||||
for (Object project : projects) {
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> projectMap = ((Map<String, Object>) project);
|
||||
String projectId = (String) projectMap.get("id");
|
||||
// remove graphs from project if needed
|
||||
List<?> projectGraphs = (List<?>) projectMap.get("project_graphs");
|
||||
if (projectGraphs != null && projectGraphs.size() > 0) {
|
||||
Map<String, Object> graphs = ImmutableMap.of("project_graphs",
|
||||
projectGraphs);
|
||||
resp = client().target()
|
||||
.path(path)
|
||||
.path(projectId)
|
||||
.queryParam("action", "remove_graph")
|
||||
.request()
|
||||
.put(Entity.json(JsonUtil.toJson(graphs)));
|
||||
assertResponseStatus(200, resp);
|
||||
}
|
||||
// delete project
|
||||
resp = client().target()
|
||||
.path(path)
|
||||
.path(projectId)
|
||||
.request()
|
||||
.delete();
|
||||
assertResponseStatus(204, resp);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -141,32 +172,32 @@ public class ProjectApiTest extends BaseApiTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testremoveGraphs() {
|
||||
public void testRemoveGraphs() {
|
||||
String projectId = this.createProjectAndAddGraph("project_test",
|
||||
"graph_test");
|
||||
String graph = "{\"project_graphs\":[\"graph_test\"]}";
|
||||
String graphs = "{\"project_graphs\":[\"graph_test\"]}";
|
||||
Response resp = client().target()
|
||||
.path(path)
|
||||
.path(projectId)
|
||||
.queryParam("action", "remove_graph")
|
||||
.request()
|
||||
.put(Entity.json(graph));
|
||||
.put(Entity.json(graphs));
|
||||
assertResponseStatus(200, resp);
|
||||
|
||||
String project = this.getProject(projectId);
|
||||
Assert.assertFalse(project.contains("project_graphs"));
|
||||
|
||||
this.addGraphs(projectId, "graph_test1", "graph_test2");
|
||||
this.addGraphsToProject(projectId, "graph_test1", "graph_test2");
|
||||
|
||||
graph = "{\"project_graphs\":[\"graph_test1\"]}";
|
||||
graphs = "{\"project_graphs\":[\"graph_test1\"]}";
|
||||
resp = client().target()
|
||||
.path(path)
|
||||
.path(projectId)
|
||||
.queryParam("action", "remove_graph")
|
||||
.request()
|
||||
.put(Entity.json(graph));
|
||||
|
||||
.put(Entity.json(graphs));
|
||||
assertResponseStatus(200, resp);
|
||||
|
||||
project = this.getProject(projectId);
|
||||
List<String> graphs1 = assertJsonContains(project, "project_graphs");
|
||||
Assert.assertEquals(1, graphs1.size());
|
||||
|
|
@ -199,19 +230,18 @@ public class ProjectApiTest extends BaseApiTest {
|
|||
String graph) {
|
||||
String projectId = assertJsonContains(createProject(projectName, null),
|
||||
"id");
|
||||
this.addGraphs(projectId, graph);
|
||||
this.addGraphsToProject(projectId, graph);
|
||||
return projectId;
|
||||
}
|
||||
|
||||
private void addGraphs(String projectId, String... graphNames) {
|
||||
private void addGraphsToProject(String projectId, String... graphNames) {
|
||||
Assert.assertFalse(ArrayUtils.isEmpty(graphNames));
|
||||
StringBuilder graphNamesBuilder = new StringBuilder();
|
||||
for (int i = 0; i < graphNames.length - 1; i++) {
|
||||
graphNamesBuilder.append(String.format("\"%s\",", graphNames[i]));
|
||||
}
|
||||
graphNamesBuilder.append(
|
||||
String.format("\"%s\"",
|
||||
graphNames[graphNames.length - 1]));
|
||||
graphNamesBuilder.append(String.format("\"%s\"",
|
||||
graphNames[graphNames.length - 1]));
|
||||
String graphs = String.format("{\"project_graphs\":[%s]}",
|
||||
graphNamesBuilder);
|
||||
Response resp = client().target()
|
||||
|
|
|
|||
|
|
@ -79,7 +79,8 @@ public class TaskApiTest extends BaseApiTest {
|
|||
r.getStatus() == 202 || r.getStatus() == 400);
|
||||
if (r.getStatus() == 202) {
|
||||
String status = assertJsonContains(content, "task_status");
|
||||
Assert.assertEquals("cancelling", status);
|
||||
Assert.assertTrue(status, status.equals("cancelling") ||
|
||||
status.equals("cancelled"));
|
||||
} else {
|
||||
assert r.getStatus() == 400;
|
||||
String error = String.format(
|
||||
|
|
|
|||
2
pom.xml
2
pom.xml
|
|
@ -96,7 +96,7 @@
|
|||
<compiler.target>1.8</compiler.target>
|
||||
<slf4j.version>1.7.5</slf4j.version>
|
||||
<log4j.version>1.2.17</log4j.version>
|
||||
<log4j2.version>2.17.0</log4j2.version>
|
||||
<log4j2.version>2.17.1</log4j2.version>
|
||||
<junit.version>4.12</junit.version>
|
||||
<tinkerpop.version>3.4.3</tinkerpop.version>
|
||||
<commons.io.version>2.7</commons.io.version>
|
||||
|
|
|
|||
Loading…
Reference in New Issue