refact: use a slim way to build docker image on latest code & support zgc (#2118) (#2297)

* also modify shell

Co-authored-by: imbajin <jin@apache.org>

Dockerfile

@@ -15,54 +15,46 @@
 # limitations under the License.
 #
 
-FROM ubuntu:xenial
+# Dockerfile for HugeGraph Server
+# 1st stage: build source code
+FROM maven:3.9.0-eclipse-temurin-11 AS build
 
-LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
+COPY . /pkg
+WORKDIR /pkg
+RUN mvn package -e -B -ntp -DskipTests -Dmaven.javadoc.skip=true && pwd && ls -l
 
-ENV PKG_URL https://github.com/hugegraph
+# 2nd stage: runtime env
+FROM openjdk:11-slim
+# TODO: get the version from the pom.xml
+ENV version=1.0.0
+COPY --from=build /pkg/apache-hugegraph-incubating-$version/ /hugegraph
+LABEL maintainer="HugeGraph Docker Maintainers <dev@hugegraph.apache.org>"
 
-# 1. Install needed dependencies of GraphServer & RocksDB
+# TODO: use g1gc or zgc as default
+ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm"
+
+#COPY . /hugegraph/hugegraph-server
+WORKDIR /hugegraph/
+
+# 1. Install environment
 RUN set -x \
     && apt-get -q update \
     && apt-get -q install -y --no-install-recommends --no-install-suggests \
+       dumb-init \
+       procps \
        curl \
        lsof \
-       g++ \
-       gcc \
-       openjdk-8-jdk \
-    && apt-get clean
-    # && rm -rf /var/lib/apt/lists/*
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # 2. Init HugeGraph Sever
-# (Optional) You can set the ip of github to speed up the local build
-# && echo "192.30.253.112 github.com\n151.101.44.249 github.global.ssl.fastly.net" >> /etc/hosts \
-ENV SERVER_VERSION 0.12.0
 RUN set -e \
-    && mkdir -p /root/hugegraph-server \
-    && curl -L -S ${PKG_URL}/hugegraph/releases/download/v${SERVER_VERSION}/hugegraph-${SERVER_VERSION}.tar.gz -o /root/server.tar.gz \
-    && tar xzf /root/server.tar.gz --strip-components 1 -C /root/hugegraph-server \
-    && rm /root/server.tar.gz \
-    && cd /root/hugegraph-server/ \
+    && pwd && cd /hugegraph/ \
     && sed -i "s/^restserver.url.*$/restserver.url=http:\/\/0.0.0.0:8080/g" ./conf/rest-server.properties \
-    && sed -n '65p' ./bin/start-hugegraph.sh | grep "&" > /dev/null && sed -i 65{s/\&$/#/g} ./bin/start-hugegraph.sh \
-    && sed -n '75p' ./bin/start-hugegraph.sh | grep "exit" > /dev/null && sed -i 75{s/^/#/g} ./bin/start-hugegraph.sh \
     && ./bin/init-store.sh
 
-# 3. Prepare for HugeGraph Studio
-ENV STUDIO_VERSION 0.10.0
-# (Optional) You can set the ip of github to speed up the local build
-# && echo "192.30.253.112 github.com\n151.101.44.249 github.global.ssl.fastly.net" >> /etc/hosts \
-RUN set -e \
-    && mkdir -p /root/hugegraph-studio \
-    && curl -L -S ${PKG_URL}/hugegraph-studio/releases/download/v${STUDIO_VERSION}/hugegraph-studio-${STUDIO_VERSION}.tar.gz -o /root/studio.tar.gz \
-    && tar xzf /root/studio.tar.gz --strip-components 1 -C /root/hugegraph-studio \
-    && rm /root/studio.tar.gz \
-    && cd /root/hugegraph-studio/ \
-    && sed -i "s/^studio.server.host.*$/studio.server.host=0.0.0.0/g" ./conf/hugegraph-studio.properties \
-    && sed -i "s/^graph.server.host.*$/graph.server.host=0.0.0.0/g" ./conf/hugegraph-studio.properties
+EXPOSE 8080
+VOLUME /hugegraph
 
-EXPOSE 8080 8088
-WORKDIR /root
-VOLUME /root
-
-ENTRYPOINT ["./hugegraph-server/bin/start-hugegraph.sh"]
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+CMD ["./bin/start-hugegraph.sh", "-d false -j $JAVA_OPTS -g zgc"]

README.md

@@ -12,25 +12,44 @@
 [![Codecov](https://codecov.io/gh/hugegraph/hugegraph/branch/master/graph/badge.svg)](https://codecov.io/gh/hugegraph/hugegraph)
 [![GitHub Releases Downloads](https://img.shields.io/github/downloads/hugegraph/hugegraph/total.svg)](https://github.com/hugegraph/hugegraph/releases)
 
-[HugeGraph](https://hugegraph.apache.org/) is a fast-speed and highly-scalable [graph database](https://en.wikipedia.org/wiki/Graph_database). Billions of vertices and edges can be easily stored into and queried from HugeGraph due to its excellent OLTP ability. As compliance to [Apache TinkerPop 3](https://tinkerpop.apache.org/) framework, various complicated graph queries can be accomplished through [Gremlin](https://tinkerpop.apache.org/gremlin.html)(a powerful graph traversal language).
+[HugeGraph](https://hugegraph.apache.org/) is a fast-speed and highly-scalable [graph database](https://en.wikipedia.org/wiki/Graph_database). 
+Billions of vertices and edges can be easily stored into and queried from HugeGraph due to its excellent OLTP ability. As compliance to [Apache TinkerPop 3](https://tinkerpop.apache.org/) framework, various complicated graph queries can be accomplished through [Gremlin](https://tinkerpop.apache.org/gremlin.html)(a powerful graph traversal language).
 
 ## Features
 
-- Compliance to [Apache TinkerPop 3](https://tinkerpop.apache.org/), supporting [Gremlin](https://tinkerpop.apache.org/gremlin.html)
+- Compliance to [Apache TinkerPop 3](https://tinkerpop.apache.org/), support [Gremlin](https://tinkerpop.apache.org/gremlin.html) & [Cypher](https://en.wikipedia.org/wiki/Cypher) language
 - Schema Metadata Management, including VertexLabel, EdgeLabel, PropertyKey and IndexLabel
 - Multi-type Indexes, supporting exact query, range query and complex conditions combination query
-- Plug-in Backend Store Driver Framework, supporting RocksDB, Cassandra, ScyllaDB, HBase and MySQL now and easy to add other backend store driver if needed
-- Integration with Hadoop/Spark
+- Plug-in Backend Store Driver Framework, support `RocksDB`, `Cassandra`, `HBase`, `ScyllaDB`, and `MySQL/Postgre` now and easy to add other backend store driver if needed
+- Integration with `Flink/Spark/HDFS`, and friendly to connect other big data platforms
 
-## Getting Started
+## Quick Start
 
-The project [homepage](https://hugegraph.apache.org/docs/) contains more information on HugeGraph and provides links to **documentation**, getting-started guides and release downloads.
+### 1. Docker Way
 
-And here are links of other repositories:
-1. [hugegraph-toolchain](https://github.com/apache/incubator-hugegraph-toolchain) (include loader/dashboard/tool/client)
-2. [hugegraph-computer](https://github.com/apache/incubator-hugegraph-computer) (graph computing system)
-3. [hugegraph-commons](https://github.com/apache/incubator-hugegraph-commons) (include common & rpc module)
-4. [hugegraph-website](https://github.com/apache/incubator-hugegraph-doc) (include doc & website code)
+We can use `docker run -itd --name=graph -p 8080:8080 hugegraph/hugegraph` to quickly start an inner 
+HugeGraph server with `RocksDB` in background.
+
+Optional: use `docker exec -it graph bash` to enter the container to do some operations.
+
+### 2. Download Way
+
+Visit [Download Page](https://hugegraph.apache.org/docs/download/download/) and refer the [doc](https://hugegraph.apache.org/docs/quickstart/hugegraph-server/#33-source-code-compilation) 
+to download the latest release package and start the server.
+
+### 3. Source Building Way
+
+Visit [Source Building Page](https://hugegraph.apache.org/docs/quickstart/hugegraph-server/#33-source-code-compilation) and follow the 
+steps to build the source code and start the server.
+
+The project [doc page](https://hugegraph.apache.org/docs/) contains more information on HugeGraph
+and provides detailed documentation for users. (Structure / Usage / API / Configs...)
+
+And here are links of other **HugeGraph** component/repositories:
+1. [hugegraph-toolchain](https://github.com/apache/incubator-hugegraph-toolchain) (graph **loader/dashboard/tool/client**)
+2. [hugegraph-computer](https://github.com/apache/incubator-hugegraph-computer) (matched **graph computing** system)
+3. [hugegraph-commons](https://github.com/apache/incubator-hugegraph-commons) (**common & rpc** module)
+4. [hugegraph-website](https://github.com/apache/incubator-hugegraph-doc) (**doc & website** code)
 
 ## Contributing
 

hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh

@@ -15,20 +15,20 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 #
-abs_path() {
+function abs_path() {
     SOURCE="${BASH_SOURCE[0]}"
-    while [ -h "$SOURCE" ]; do
+    while [[ -h "$SOURCE" ]]; do
         DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)"
         SOURCE="$(readlink "$SOURCE")"
         [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
     done
-    echo "$(cd -P "$(dirname "$SOURCE")" && pwd)"
+    cd -P "$(dirname "$SOURCE")" && pwd
 }
 
 if [[ $# -lt 3 ]]; then
     echo "USAGE: $0 GREMLIN_SERVER_CONF REST_SERVER_CONF OPEN_SECURITY_CHECK"
     echo " e.g.: $0 conf/gremlin-server.yaml conf/rest-server.properties true"
-    exit 1;
+    exit 1
 fi
 
 BIN=$(abs_path)
@@ -67,8 +67,8 @@ MIN_MEM=$((1 * 512))
 MIN_JAVA_VERSION=8
 
 # download binary file
-if [[ ! -e "${CONF}/hugegraph-server.keystore"  ]]; then
-  download "${CONF}" "https://github.com/apache/hugegraph-doc/raw/binary-1.0/dist/server/hugegraph-server.keystore"
+if [[ ! -e "${CONF}/hugegraph-server.keystore" ]]; then
+    download "${CONF}" "https://github.com/apache/hugegraph-doc/raw/binary-1.0/dist/server/hugegraph-server.keystore"
 fi
 
 # Add the slf4j-log4j12 binding
@@ -77,8 +77,8 @@ CP=$(find -L $LIB -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':')
 CP="$CP":$(find -L $LIB -name 'hugegraph*.jar' | sort | tr '\n' ':')
 # Add the remaining jars in lib.
 CP="$CP":$(find -L $LIB -name '*.jar' \
-                \! -name 'hugegraph*' \
-                \! -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':')
+    \! -name 'hugegraph*' \
+    \! -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':')
 # Add the jars in ext (at any subdirectory depth)
 CP="$CP":$(find -L $EXT -name '*.jar' | sort | tr '\n' ':')
 # Add the jars in plugins (at any subdirectory depth)
@@ -90,7 +90,7 @@ CP="$CP":$(find -L $PLUGINS -name '*.jar' | sort | tr '\n' ':')
 export CLASSPATH="${CLASSPATH:-}:$CP"
 
 # Change to $BIN's parent
-cd "${TOP}" || exit 1;
+cd "${TOP}" || exit 1
 
 # Find java & enable server option
 if [ "$JAVA_HOME" = "" ]; then
@@ -102,7 +102,7 @@ fi
 JAVA_VERSION=$($JAVA -version 2>&1 | head -1 | cut -d'"' -f2 | sed 's/^1\.//' | cut -d'.' -f1)
 if [[ $? -ne 0 || $JAVA_VERSION -lt $MIN_JAVA_VERSION ]]; then
     echo "Make sure the JDK is installed and the version >= $MIN_JAVA_VERSION, current is $JAVA_VERSION" \
-         >> ${OUTPUT}
+         >> "${OUTPUT}"
     exit 1
 fi
 
@@ -110,7 +110,7 @@ fi
 if [ "$JAVA_OPTIONS" = "" ]; then
     XMX=$(calc_xmx $MIN_MEM $MAX_MEM)
     if [ $? -ne 0 ]; then
-        echo "Failed to start HugeGraphServer, requires at least ${MIN_MEM}MB free memory" >> ${OUTPUT}
+        echo "Failed to start HugeGraphServer, requires at least ${MIN_MEM}MB free memory" >> "${OUTPUT}"
         exit 1
     fi
     JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION}"
@@ -128,14 +128,22 @@ fi
 
 # Using G1GC as the default garbage collector (Recommended for large memory machines)
 case "$GC_OPTION" in
-    g1)
+    g1|G1|g1gc)
         echo "Using G1GC as the default garbage collector"
         JAVA_OPTIONS="${JAVA_OPTIONS} -XX:+UseG1GC -XX:+ParallelRefProcEnabled \
-                      -XX:InitiatingHeapOccupancyPercent=50 -XX:G1RSetUpdatingPauseTimePercent=5"
+                                      -XX:InitiatingHeapOccupancyPercent=50 \
+                                      -XX:G1RSetUpdatingPauseTimePercent=5"
+        ;;
+    zgc|ZGC)
+        echo "Using ZGC as the default garbage collector (Only support Java 11+)"
+        JAVA_OPTIONS="${JAVA_OPTIONS} -XX:+UseZGC -XX:+UnlockExperimentalVMOptions \
+                                      -XX:ConcGCThreads=2 -XX:ParallelGCThreads=6 \
+                                      -XX:ZCollectionInterval=120 -XX:ZAllocationSpikeTolerance=5 \
+                                      -XX:+UnlockDiagnosticVMOptions -XX:-ZProactive"
         ;;
     "") ;;
     *)
-        echo "Unrecognized gc option: '$GC_OPTION', only support 'g1' now" >> ${OUTPUT}
+        echo "Unrecognized gc option: '$GC_OPTION', only support 'G1/ZGC' now" >> ${OUTPUT}
         exit 1
 esac
 
@@ -145,6 +153,6 @@ if [[ ${OPEN_SECURITY_CHECK} == "true" ]]; then
 fi
 
 # Turn on security check
-exec ${JAVA} -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \
-     -cp ${CLASSPATH}: org.apache.hugegraph.dist.HugeGraphServer ${GREMLIN_SERVER_CONF} ${REST_SERVER_CONF} \
-     >> ${OUTPUT} 2>&1
+exec ${JAVA} -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} -cp ${CLASSPATH}: \
+    org.apache.hugegraph.dist.HugeGraphServer ${GREMLIN_SERVER_CONF} ${REST_SERVER_CONF} \
+    >> ${OUTPUT} 2>&1

hugegraph-dist/src/assembly/static/bin/init-store.sh

@@ -15,25 +15,25 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 #
-abs_path() {
+function abs_path() {
     SOURCE="${BASH_SOURCE[0]}"
     while [[ -h "$SOURCE" ]]; do
-        DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+        DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)"
         SOURCE="$(readlink "$SOURCE")"
-        [[ ${SOURCE} != /* ]] && SOURCE="$DIR/$SOURCE"
+        [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
     done
-    echo "$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+    cd -P "$(dirname "$SOURCE")" && pwd
 }
 
-BIN=`abs_path`
-TOP="$(cd ${BIN}/../ && pwd)"
+BIN=$(abs_path)
+TOP="$(cd "${BIN}"/../ && pwd)"
 CONF="$TOP/conf"
 LIB="$TOP/lib"
 PLUGINS="$TOP/plugins"
 
-. ${BIN}/util.sh
+. "${BIN}"/util.sh
 
-ensure_path_writable ${PLUGINS}
+ensure_path_writable "${PLUGINS}"
 
 if [[ -n "$JAVA_HOME" ]]; then
     JAVA="$JAVA_HOME"/bin/java
@@ -43,11 +43,11 @@ else
     EXT="$LIB:$PLUGINS"
 fi
 
-cd ${TOP}
+cd "${TOP}" || exit
 
 DEFAULT_JAVA_OPTIONS=""
-JAVA_VERSION=$($JAVA -version 2>&1 | awk 'NR==1{gsub(/"/,""); print $3}' \
-              | awk -F'_' '{print $1}')
+JAVA_VERSION=$($JAVA -version 2>&1 | awk 'NR==1{gsub(/"/,""); print $3}' | awk -F'_' '{print $1}')
+# TODO: better not string number compare, use `bc` like github.com/koalaman/shellcheck/wiki/SC2072
 if [[ $? -eq 0 && $JAVA_VERSION >  "1.9" ]]; then
       DEFAULT_JAVA_OPTIONS="--add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED"
 fi

hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh

@@ -17,41 +17,45 @@
 #
 OPEN_MONITOR="false"
 OPEN_SECURITY_CHECK="true"
-VERBOSE=""
+DAEMON="true"
+#VERBOSE=""
 GC_OPTION=""
 USER_OPTION=""
 SERVER_STARTUP_TIMEOUT_S=30
 
-while getopts "g:m:s:j:t:v" arg; do
+while getopts "d:g:m:s:j:t:v" arg; do
     case ${arg} in
+        d) DAEMON="$OPTARG" ;;
         g) GC_OPTION="$OPTARG" ;;
         m) OPEN_MONITOR="$OPTARG" ;;
         s) OPEN_SECURITY_CHECK="$OPTARG" ;;
         j) USER_OPTION="$OPTARG" ;;
         t) SERVER_STARTUP_TIMEOUT_S="$OPTARG" ;;
+        # TODO: should remove it in future (check the usage carefully)
         v) VERBOSE="verbose" ;;
-        ?) echo "USAGE: $0 [-g g1] [-m true|false] [-s true|false] [-j java_options] [-t timeout] [-v]" && exit 1 ;;
+        ?) echo "USAGE: $0 [-d true|false] [-g g1] [-m true|false] [-s true|false] [-j java_options]
+                [-t timeout]" && exit 1 ;;
     esac
 done
 
 if [[ "$OPEN_MONITOR" != "true" && "$OPEN_MONITOR" != "false" ]]; then
-    echo "USAGE: $0 [-g g1] [-m true|false] [-s true|false] [-j xxx] [-v]"
+    echo "USAGE: $0 [-d true|false] [-g g1] [-m true|false] [-s true|false] [-j java_options]"
     exit 1
 fi
 
 if [[ "$OPEN_SECURITY_CHECK" != "true" && "$OPEN_SECURITY_CHECK" != "false" ]]; then
-    echo "USAGE: $0 [-g g1] [-m true|false] [-s true|false] [-j xxx] [-v]"
+    echo "USAGE: $0 [-d true|false] [-g g1] [-m true|false] [-s true|false] [-j java_options]"
     exit 1
 fi
 
 function abs_path() {
     SOURCE="${BASH_SOURCE[0]}"
-    while [ -h "$SOURCE" ]; do
-        DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+    while [[ -h "$SOURCE" ]]; do
+        DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)"
         SOURCE="$(readlink "$SOURCE")"
         [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
     done
-    echo "$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+    cd -P "$(dirname "$SOURCE")" && pwd
 }
 
 BIN=$(abs_path)
@@ -75,10 +79,15 @@ if [ ! -d "$LOGS" ]; then
     mkdir -p "$LOGS"
 fi
 
-echo "Starting HugeGraphServer..."
-
-${BIN}/hugegraph-server.sh ${CONF}/gremlin-server.yaml ${CONF}/rest-server.properties \
-${OPEN_SECURITY_CHECK} ${USER_OPTION} ${GC_OPTION} >>${LOGS}/hugegraph-server.log 2>&1 &
+if [[ $DAEMON == "true" ]]; then
+    echo "Starting HugeGraphServer in daemon mode..."
+    "${BIN}"/hugegraph-server.sh "${CONF}"/gremlin-server.yaml "${CONF}"/rest-server.properties \
+    "${OPEN_SECURITY_CHECK}" "${USER_OPTION}" "${GC_OPTION}" >>"${LOGS}"/hugegraph-server.log 2>&1 &
+else
+    echo "Starting HugeGraphServer in foreground mode..."
+    "${BIN}"/hugegraph-server.sh "${CONF}"/gremlin-server.yaml "${CONF}"/rest-server.properties \
+    "${OPEN_SECURITY_CHECK}" "${USER_OPTION}" "${GC_OPTION}" >>"${LOGS}"/hugegraph-server.log 2>&1
+fi
 
 PID="$!"
 # Write pid to file
@@ -86,15 +95,16 @@ echo "$PID" > "$PID_FILE"
 
 trap 'kill $PID; exit' SIGHUP SIGINT SIGQUIT SIGTERM
 
-wait_for_startup ${PID} 'HugeGraphServer' "$REST_SERVER_URL/graphs" ${SERVER_STARTUP_TIMEOUT_S} || {
+wait_for_startup ${PID} 'HugeGraphServer' "$REST_SERVER_URL/graphs" "${SERVER_STARTUP_TIMEOUT_S}" || {
     echo "See $LOGS/hugegraph-server.log for HugeGraphServer log output." >&2
-    exit 1
+    if [[ $DAEMON == "true" ]]; then
+        exit 1
+    fi
 }
 disown
 
 if [ "$OPEN_MONITOR" == "true" ]; then
-    "$BIN"/start-monitor.sh
-    if [ $? -ne 0 ]; then
+    if ! "$BIN"/start-monitor.sh; then
         echo "Failed to open monitor, please start it manually"
     fi
     echo "An HugeGraphServer monitor task has been append to crontab"

hugegraph-dist/src/assembly/travis/run-api-test.sh

@@ -21,15 +21,15 @@ BACKEND=$1
 REPORT_DIR=$2
 REPORT_FILE=$REPORT_DIR/jacoco-api-test-for-raft.xml
 
-TRAVIS_DIR=`dirname $0`
-VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
+TRAVIS_DIR=$(dirname $0)
+VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
 SERVER_DIR=apache-hugegraph-incubating-$VERSION
 CONF=$SERVER_DIR/conf/graphs/hugegraph.properties
 REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties
 GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml
 JACOCO_PORT=36320
 
-mvn package -DskipTests
+mvn package -DskipTests -ntp
 
 
 # add mysql dependency