feat: support docker use the auth when starting (#2403)

- allow user to set env for docker to set auth mode - download keystore when package - fix a curl error (also use curl first in `function` download) --------- Co-authored-by: imbajin <jin@apache.org>
2024-01-15 20:50:24 +08:00 · 2024-01-15 20:50:24 +08:00 · de5904a600
parent 57cd0e8dd7
commit de5904a600
14 changed files with 227 additions and 56 deletions
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@ -96,7 +96,6 @@ header: # `header` section is configurations for source codes license header.
    - '**/util/StringEncoding.java'
    - 'hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherOpProcessor.java'
    - 'hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherPlugin.java'
-    - 'hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-storage.sh'
  comment: on-failure # on what condition license-eye will comment on the pull request, `on-failure`, `always`, `never`.

  # license-location-threshold specifies the index threshold where the license header can be located,
--- a/1
+++ b/1
@ -214,6 +214,5 @@ hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeScriptT
 hugegraph-core/src/main/java/org/apache/hugegraph/type/Nameable.java from https://github.com/JanusGraph/janusgraph
 hugegraph-core/src/main/java/org/apache/hugegraph/type/define/Cardinality.java from https://github.com/JanusGraph/janusgraph
 hugegraph-core/src/main/java/org/apache/hugegraph/util/StringEncoding.java from https://github.com/JanusGraph/janusgraph
-hugegraph-dist/src/assembly/static/bin/wait-storage.sh from https://github.com/JanusGraph/janusgraph
 hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherOpProcessor.java from https://github.com/opencypher/cypher-for-gremlin
 hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherPlugin.java from https://github.com/opencypher/cypher-for-gremlin
--- a/README.md
+++ b/README.md
@ -30,12 +30,24 @@ Billions of vertices and edges can be easily stored into and queried from HugeGr

 ## Quick Start

-### 1. Download Way
+### 1. Docker Way (Convenient for Test)
+
+We can use `docker run -itd --name=graph -p 8080:8080 hugegraph/hugegraph` to quickly start an inner 
+HugeGraph server with `RocksDB` (in backgrounds) for **test/dev**.
+You can visit [doc page](https://hugegraph.apache.org/docs/quickstart/hugegraph-server/#3-deploy) or the [README](hugegraph-server/hugegraph-dist/docker/READEME.md) for more details.
+
+> Note:
+>  
+> 1. The docker image of hugegraph is a convenience release, but not **official distribution** artifacts. You can find more details from [ASF Release Distribution Policy](https://infra.apache.org/release-distribution.html#dockerhub).
+> 
+> 2. Recommand to use `release tag`(like `1.2.0`) for the stable version. Use `latest` tag to experience the newest functions in development.
+
+### 2. Download Way

 Visit [Download Page](https://hugegraph.apache.org/docs/download/download/) and refer the [doc](https://hugegraph.apache.org/docs/quickstart/hugegraph-server/#32-download-the-binary-tar-tarball) 
 to download the latest release package and start the server.

-### 2. Source Building Way
+### 3. Source Building Way

 Visit [Source Building Page](https://hugegraph.apache.org/docs/quickstart/hugegraph-server/#33-source-code-compilation) and follow the 
 steps to build the source code and start the server.
@ -49,17 +61,7 @@ And here are links of other **HugeGraph** component/repositories:
 3. [hugegraph-commons](https://github.com/apache/incubator-hugegraph-commons) (**common & rpc** libs)
 4. [hugegraph-website](https://github.com/apache/incubator-hugegraph-doc) (**doc & website** code)

-### 3. Docker Way (Convenient for Test)

-We can use `docker run -itd --name=graph -p 8080:8080 hugegraph/hugegraph` to quickly start an inner 
-HugeGraph server with `RocksDB` (in backgrounds) for **test/dev**.
-You can visit [doc page](https://hugegraph.apache.org/docs/quickstart/hugegraph-server/#3-deploy) or the [README](hugegraph-server/hugegraph-dist/docker/READEME.md) for more details.
-
-> Note:
->  
-> 1. The docker image of hugegraph is a convenience release, but not **official distribution** artifacts. You can find more details from [ASF Release Distribution Policy](https://infra.apache.org/release-distribution.html#dockerhub).
-> 
-> 2. Recommand to use `release tag`(like `1.0.0`) for the stable version. Use `latest` tag to experience the newest functions in development.

 ## License

--- a/hugegraph-server/hugegraph-dist/docker/README.md
+++ b/hugegraph-server/hugegraph-dist/docker/README.md
@ -4,7 +4,7 @@
 > 
 > 1. The docker image of hugegraph is a convenience release, not official distribution artifacts from ASF. You can find more details from [ASF Release Distribution Policy](https://infra.apache.org/release-distribution.html#dockerhub).
 > 
-> 2. Recommand to use `release tag`(like `1.0.0`) for the stable version. Use `latest` tag to experience the newest functions in development.
+> 2. Recommand to use `release tag`(like `1.2.0`) for the stable version. Use `latest` tag to experience the newest functions in development.

 ## 1. Deploy

@ -12,7 +12,7 @@ We can use docker to quickly start an inner HugeGraph server with RocksDB in bac

 1. Using docker run

-    Use `docker run -itd --name=graph -p 18080:8080 hugegraph/hugegraph` to start hugegraph server.
+    Use `docker run -itd --name=graph -p 8080:8080 hugegraph/hugegraph` to start hugegraph server.

 2. Using docker compose

@ -35,7 +35,7 @@ If you want to customize the pre-loaded data, please mount the the groovy script

 1. Using docker run

-    Use `docker run -itd --name=graph -p 18080:8080 -e PRELOAD=true -v /path/to/yourScript:/hugegraph/scripts/example.groovy hugegraph/hugegraph`
+    Use `docker run -itd --name=graph -p 8080:8080 -e PRELOAD=true -v /path/to/yourScript:/hugegraph/scripts/example.groovy hugegraph/hugegraph`
    to start hugegraph server.

 2. Using docker compose 
@ -57,4 +57,27 @@ If you want to customize the pre-loaded data, please mount the the groovy script

 3. Using start-hugegraph.sh

-    If you deploy HugeGraph server without docker, you can also pass arguments using `-p`, like this: `bin/start-hugegraph.sh -p true`.
+    If you deploy HugeGraph server without docker, you can also pass arguments using `-p`, like this: `bin/start-hugegraph.sh -p true`.
+
+## 3. Enable Authentication
+
+1. Using docker run
+
+    Use `docker run -itd --name=graph -p 8080:8080 -e AUTH=true -e PASSWORD=123456 hugegraph/hugegraph` to enable the authentication and set the password with `-e AUTH=true -e PASSWORD=123456`.
+
+2. Using docker compose
+
+    Similarly, we can set the envionment variables in the docker-compose.yaml:
+
+    ```yaml
+    version: '3'
+    services:
+      server:
+        image: hugegraph/hugegraph
+        container_name: graph
+        ports:
+          - 8080:8080
+        environment:
+          - AUTH=true
+          - PASSWORD=123456
+    ```
--- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh
+++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh
@ -16,11 +16,24 @@
 # under the License.
 #

-
+# wait for storage like cassandra
 ./bin/wait-storage.sh

-./bin/init-store.sh
+# set auth if needed 
+if [[ $AUTH == "true" ]]; then
+    # set password if use do not provide
+    if [ -z "$PASSWORD" ]; then
+        echo "you have not set the password, we will use the default password"
+        PASSWORD="hugegraph"
+    fi
+    echo "init hugegraph with auth"
+    ./bin/enable-auth.sh
+    echo "$PASSWORD" | ./bin/init-store.sh
+else
+    ./bin/init-store.sh
+fi

+# start hugegraph
 ./bin/start-hugegraph.sh -j "$JAVA_OPTS" -g zgc

 tail -f /dev/null
--- a/hugegraph-server/hugegraph-dist/docker/example/docker-compose-cassandra.yml
+++ b/hugegraph-server/hugegraph-dist/docker/example/docker-compose-cassandra.yml
@ -22,7 +22,7 @@ services:
    image: hugegraph/hugegraph
    container_name: cas-graph
    ports:
-      - 18080:8080
+      - 8080:8080
    environment:
      hugegraph.backend: cassandra
      hugegraph.serializer: cassandra
--- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/docker-entrypoint.sh
+++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/docker-entrypoint.sh
@ -16,9 +16,20 @@
 # under the License.
 #

+curl --version >/dev/null 2>&1 ||
+  {
+    echo 'ERROR: Please install `curl` first if you need `hugegraph-server.keystore`'
+    exit
+  }

-./bin/wait-storage.sh
+# TODO: perhaps it's necessary verify the checksum before reusing the existing keystore
+if [[ ! -f hugegraph-server.keystore ]]; then
+  curl -s -S -L -o hugegraph-server.keystore \
+    https://github.com/apache/hugegraph-doc/raw/binary-1.0/dist/server/hugegraph-server.keystore ||
+    {
+      echo 'ERROR: Download `hugegraph-server.keystore` from GitHub failed, please check your network connection'
+      exit
+    }
+fi

-./bin/init-store.sh
-
-./bin/start-hugegraph.sh -d false -j "$JAVA_OPTS" -g zgc
+echo 'INFO: Successfully download `hugegraph-server.keystore`'
--- a/hugegraph-server/hugegraph-dist/download_swagger_ui.sh
+++ b/hugegraph-server/hugegraph-dist/download_swagger_ui.sh
--- a/hugegraph-server/hugegraph-dist/pom.xml
+++ b/hugegraph-server/hugegraph-dist/pom.xml
@ -187,7 +187,7 @@
                                    <exec executable="${shell-executable}"
                                          dir="${project.basedir}"
                                          failonerror="false">
-                                        <arg line="./dist.sh"/>
+                                        <arg line="./download_swagger_ui.sh"/>
                                    </exec>
                                </target>
                            </configuration>
@ -216,6 +216,45 @@
                                </target>
                            </configuration>
                        </execution>
+                        <execution>
+                            <id>download-keystore</id>
+                            <phase>prepare-package</phase>
+                            <goals>
+                                <goal>run</goal>
+                            </goals>
+                            <configuration>
+                                <target>
+                                    <exec executable="${shell-executable}"
+                                          dir="${project.basedir}"
+                                          failonerror="false">
+                                        <arg line="./download_keystore.sh"/>
+                                    </exec>
+                                </target>
+                            </configuration>
+                        </execution>
+                        <execution>
+                            <id>cp-keystore</id>
+                            <phase>package</phase>
+                            <goals>
+                                <goal>run</goal>
+                            </goals>
+                            <configuration>
+                                <target>
+                                    <exec executable="cp"
+                                          dir="${project.basedir}"
+                                          failonerror="false">
+                                        <arg value="hugegraph-server.keystore"/>
+                                        <arg value="../${final.name}/conf/hugegraph-server.keystore"/>
+                                    </exec>
+                                    <exec executable="rm"
+                                          dir="${project.basedir}"
+                                          failonerror="false">
+                                        <arg value="-rf"/>
+                                        <arg value="hugegraph-server.keystore"/>
+                                    </exec>
+                                </target>
+                            </configuration>
+                        </execution>
                    </executions>
                </plugin>
            </plugins>
--- a/hugegraph-server/hugegraph-dist/release-docs/LICENSE
+++ b/hugegraph-server/hugegraph-dist/release-docs/LICENSE
@ -220,7 +220,6 @@ The text of each license is the standard Apache 2.0 license.
 hugegraph-core/src/main/java/org/apache/hugegraph/type/Nameable.java from https://github.com/JanusGraph/janusgraph
 hugegraph-core/src/main/java/org/apache/hugegraph/type/define/Cardinality.java from https://github.com/JanusGraph/janusgraph
 hugegraph-core/src/main/java/org/apache/hugegraph/util/StringEncoding.java from https://github.com/JanusGraph/janusgraph
-hugegraph-dist/src/assembly/static/bin/wait-storage.sh from https://github.com/JanusGraph/janusgraph
 hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeScriptTraversal.java from https://github.com/apache/tinkerpop
 hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/ProcessBasicSuite.java from https://github.com/apache/tinkerpop
 hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/StructureBasicSuite.java from https://github.com/apache/tinkerpop
--- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/enable-auth.sh
+++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/enable-auth.sh
@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+function abs_path() {
+    SOURCE="${BASH_SOURCE[0]}"
+    while [[ -h "$SOURCE" ]]; do
+        DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)"
+        SOURCE="$(readlink "$SOURCE")"
+        [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
+    done
+    cd -P "$(dirname "$SOURCE")" && pwd
+}
+
+BIN=$(abs_path)
+TOP="$(cd "${BIN}"/../ && pwd)"
+CONF="$TOP/conf"
+
+GREMLIN_SERVER_CONF="gremlin-server.yaml"
+REST_SERVER_CONF="rest-server.properties"
+GRAPH_CONF="hugegraph.properties"
+
+# make a backup
+BAK_CONF="$TOP/conf-bak"
+mkdir -p "$BAK_CONF"
+cp "${CONF}/${GREMLIN_SERVER_CONF}" "${BAK_CONF}/${GREMLIN_SERVER_CONF}.bak"
+cp "${CONF}/${REST_SERVER_CONF}" "${BAK_CONF}/${REST_SERVER_CONF}.bak"
+cp "${CONF}/graphs/${GRAPH_CONF}" "${BAK_CONF}/${GRAPH_CONF}.bak"
+
+
+sed -i -e '$a\authentication: {' \
+       -e '$a\  authenticator: org.apache.hugegraph.auth.StandardAuthenticator,' \
+       -e '$a\  authenticationHandler: org.apache.hugegraph.auth.WsAndHttpBasicAuthHandler,' \
+       -e '$a\  config: {tokens: conf/rest-server.properties}' \
+       -e '$a\}' ${CONF}/${GREMLIN_SERVER_CONF}
+
+sed -i -e '$a\auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator' \
+       -e '$a\auth.graph_store=hugegraph' ${CONF}/${REST_SERVER_CONF}
+
+sed -i 's/gremlin.graph=org.apache.hugegraph.HugeFactory/gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy/g' ${CONF}/graphs/${GRAPH_CONF}
--- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/gremlin-console.sh
+++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/gremlin-console.sh
@ -109,7 +109,7 @@ if [ -z "${HADOOP_GREMLIN_LIBS:-}" ]; then
 fi

 if [ -z "${JAVA_OPTIONS:-}" ]; then
-    JAVA_OPTIONS="-Dtinkerpop.ext=$EXT -Dlog4j.configurationFile=conf/log4j2.xml -Dgremlin.log4j.level=$GREMLIN_LOG_LEVEL -javaagent:$LIB/jamm-0.3.0.jar"
+    JAVA_OPTIONS="-Dtinkerpop.ext=$EXT -Dlog4j.configurationFile=conf/log4j2.xml -Dgremlin.log4j.level=$GREMLIN_LOG_LEVEL -javaagent:$LIB/jamm-0.3.2.jar"
 fi

 if [ "$PROFILING_ENABLED" = true ]; then
--- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh
+++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh
@ -17,10 +17,10 @@
 #
 function command_available() {
    local cmd=$1
-    if [ "$(command -v "$cmd" >/dev/null 2>&1)" ]; then
-        return 1
-    else
+    if [[ -x "$(command -v "$cmd")" ]]; then
        return 0
+    else
+        return 1
    fi
 }

@ -131,6 +131,7 @@ function wait_for_startup() {
    local stop_s=$((now_s + timeout_s))

    local status
+    local error_file_name="startup_error.txt"

    echo -n "Connecting to $server_name ($server_url)"
    while [ "$now_s" -le $stop_s ]; do
@ -141,16 +142,22 @@ function wait_for_startup() {
            return 1
        fi

-        status=$(curl -I -s -k -w "%{http_code}" -o /dev/null "$server_url")
+        status=$(curl -I -sS -k -w "%{http_code}" -o /dev/null "$server_url" 2> "$error_file_name")
        if [[ $status -eq 200 || $status -eq 401 ]]; then
            echo "OK"
            echo "Started [pid $pid]"
+            if [ -e "$error_file_name" ]; then
+                rm "$error_file_name"
+            fi
            return 0
        fi
        sleep 2
        now_s=$(date '+%s')
    done

+    echo ""
+    cat "$error_file_name"
+    rm "$error_file_name"
    echo "The operation timed out(${timeout_s}s) when attempting to connect to $server_url" >&2
    return 1
 }
@ -267,15 +274,20 @@ function get_ip() {

 function download() {
    local path=$1
-    local link_url=$2
-
-    if command_available "wget"; then
+    local download_url=$2
+    if command_available "curl"; then
+        if [ ! -d "$path" ]; then
+            mkdir -p "$path" || {
+                echo "Failed to create directory: $path"
+                exit 1
+            }
+        fi
+        curl -L "${download_url}" -o "${path}/$(basename "${download_url}")"
+    elif command_available "wget"; then
        wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt=""
-        wget "${link_url}" -P "${path}" $progress_opt
-    elif command_available "curl"; then
-        curl "${link_url}" -o "${path}"/"${link_url}"
+        wget "${download_url}" -P "${path}" $progress_opt
    else
-        echo "Required wget or curl but they are unavailable"
+        echo "Required curl or wget but they are unavailable"
        exit 1
    fi
 }
--- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-storage.sh
+++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-storage.sh
@ -1,18 +1,19 @@
 #!/bin/bash
 #
-# Copyright 2023 JanusGraph Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
 #

 function abs_path() {
@ -33,24 +34,43 @@ DETECT_STORAGE="$TOP/scripts/detect-storage.groovy"

 . "$BIN"/util.sh

+
+function key_exists {
+    local key=$1
+    local file_name=$2
+    grep -q -E "^\s*${key}\s*=\.*" ${file_name}
+}
+
+function update_key {
+    local key=$1
+    local val=$2
+    local file_name=$3
+    sed -ri "s#^(\s*${key}\s*=).*#\\1${val}#" ${file_name}
+}
+
+function add_key {
+    local key=$1
+    local val=$2
+    local file_name=$3
+    echo "${key}=${val}" >> ${file_name}
+}
+
 # apply config from env
 while IFS=' ' read -r envvar_key envvar_val; do
-    if [[ "${envvar_key}" =~ hugegraph\. ]] && [[ ! -z ${envvar_val} ]]; then
+    if [[ "${envvar_key}" =~ hugegraph\. ]] && [[ -n ${envvar_val} ]]; then
        envvar_key=${envvar_key#"hugegraph."}
-        if grep -q -E "^\s*${envvar_key}\s*=\.*" ${GRAPH_CONF}; then
-            sed -ri "s#^(\s*${envvar_key}\s*=).*#\\1${envvar_val}#" ${GRAPH_CONF}
+        if key_exists ${envvar_key} ${GRAPH_CONF}; then
+            update_key ${envvar_key} ${envvar_val} ${GRAPH_CONF}
        else
-            echo "${envvar_key}=${envvar_val}" >> ${GRAPH_CONF}
+            add_key ${envvar_key} ${envvar_val} ${GRAPH_CONF}
        fi
-    else
-        continue
    fi
 done < <(env | sort -r | awk -F= '{ st = index($0, "="); print $1 " " substr($0, st+1) }')

 # wait for storage
 if env | grep '^hugegraph\.' > /dev/null; then
-    if ! [ -z "${WAIT_STORAGE_TIMEOUT_S:-}" ]; then
+    if [ -n "${WAIT_STORAGE_TIMEOUT_S:-}" ]; then
        timeout "${WAIT_STORAGE_TIMEOUT_S}s" bash -c \
-        "until bin/gremlin-console.sh -- -e $DETECT_STORAGE > /dev/null 2>&1; do echo \"waiting for storage...\"; sleep 5; done"
+        "until bin/gremlin-console.sh -- -e $DETECT_STORAGE > /dev/null 2>&1; do echo \"Hugegraph server are waiting for storage backend...\"; sleep 5; done"
    fi
 fi