fix base image (#693)

* fix base image * update exec wrapper
2023-09-18 19:29:11 +08:00 · 2023-09-18 19:29:11 +08:00 · 25123cda7c
parent b65372390f
commit 25123cda7c
11 changed files with 104 additions and 89 deletions
--- a/README.md
+++ b/README.md
@ -1,6 +1,6 @@
 # Welcome to ByConity

-<p align="center"> 
+<p align="center">
 <img width="717" alt="ByConity Arch 2023" src="https://github.com/ByConity/ByConity/assets/23332032/c266aa89-c1b8-4a35-a47d-ee5718a9443a">

 Byconity, an advanced database management system, is a derivative of ClickHouse DBMS, building upon the robust codebase from ClickHouse v21.8. However, Byconity's development path has since diverged, thanks in part to insights gained from Snowflake's architecture.
@ -12,7 +12,7 @@ We deeply appreciate the profound contributions from the ClickHouse team, with w

 **Query Large Scale Data with Speed and Precision**
 When dealing with large-scale data, performance is crucial. Byconity shines in this aspect by providing powerful querying capabilities that excel in large-scale environments. With Byconity, you can extract valuable insights from vast amounts of data quickly and accurately.
-    
+
 **Break Down Data Silos with Byconity**
 Data silos pose significant challenges in data management. With different systems and processes often resulting in isolated islands of data, it hampers data analysis and insights. Byconity addresses this issue by seamlessly ingesting both batch-loaded data and streaming data, thus enabling your systems to break down silos for smoother data flow.

@ -25,20 +25,22 @@ Byconity is designed with a cloud-native approach, optimized to take full advant
 - **Avoid Data Silos** : By handling both batch and streaming data, Byconity ensures all your data can be integrated, promoting better insights.
 - **Cloud-Native Design** : Byconity is built with a cloud-native approach, allowing it to efficiently leverage the advantages of the cloud and work seamlessly on both Kubernetes and physical clusters.
 - **Open Source**: Being an open-source project, Byconity encourages community collaboration. You can contribute, improve, and tailor the platform according to your needs.
-    
+
+## Build and Run ByConity
+
+The easiest way to build ByConity is built in [docker dev-env](https://github.com/ByConity/ByConity/tree/master/docker/debian/dev-env). If you build on your local machine, ByConity executable file depends on Foundation DB library `libfdb_c.so`. So in order to run it, we need to install the FoundationDB client package. This [link](https://apple.github.io/foundationdb/getting-started-linux.html) tells how to install. We can download client package from FoundationDB GitHub release pages, for example [here][foundationdb-client-library].
+
+In case you want to build ByConity in metal machine, follow this [guide](https://github.com/ByConity/ByConity/tree/master/doc/build_in_metal_machine.md)
+
+[foundationdb-client-library]: https://github.com/apple/foundationdb/releases/tag/7.1.3
+
+Using [Docker Compose](./docker/docker-compose/README.md) would be convenient for running a ByConity cluster.
+
 ## Useful Link
-    
+
 - [Official Website](https://byconity.github.io/): has a quick high-level overview of ByConity on the home page.
 - [Documentation](https://byconity.github.io/docs/introduction/main-principle-concepts): introduce basic usage guide and tech deep dive.
 - [Getting started with Kubernetes](https://byconity.github.io/docs/deployment/deploy-k8s): demonstrates how to deploy a ByConity cluster in your Kubernetes clusters.
 - [Getting started with physical machines](https://byconity.github.io/docs/deployment/package-deployment): demonstrateds how to deploy ByConity in your physical clusters.
 - [Contribution Guideline](https://github.com/ByConity/ByConity/blob/master/CONTRIBUTING.md): Welcome you to join ByConity developer group and list some tips for fresh joiners to be quickly hands-on.
 - **Contact Us** : you can easily find us in [Discord server](https://discord.gg/V4BvTWGEQJ), [Youtube Channel](https://www.youtube.com/@ByConity/featured) and [Twitter](https://twitter.com/ByConity)
-
-## Build ByConity
-
-The easiest way to build ByConity is built in [docker](https://github.com/ByConity/ByConity/tree/master/docker/builder). ByConity executable file depend on Foundation DB library `libfdb_c.so`. So in order to run it, we need to install the FoundationDB client package. This [link](https://apple.github.io/foundationdb/getting-started-linux.html) tells how to install. We can download client package from FoundationDB GitHub release pages, for example [here][foundationdb-client-library].
-
-In case you want to build ByConity in metal machine, follow this [guide](https://github.com/ByConity/ByConity/tree/master/doc/build_in_metal_machine.md)
-
-[foundationdb-client-library]: https://github.com/apple/foundationdb/releases/tag/7.1.3
--- a/docker/debian/base/Dockerfile
+++ b/docker/debian/base/Dockerfile
@ -44,7 +44,8 @@ LABEL org.opencontainers.image.source="https://github.com/ByConity/ByConity"

 RUN sed -i 's/deb.debian.org/mirrors.ustc.edu.cn/g' /etc/apt/sources.list
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-    binutils gettext-base strace bpftrace sysstat gdb curl runit \
+    runit binutils gettext-base strace bpftrace sysstat gdb curl vim nano net-tools linux-perf \
+    dstat procps telnet \
    && rm -rf /var/lib/apt/lists/*

 COPY --from=downloader /downloads /downloads
@ -61,7 +62,6 @@ ARG APP_ROOT=/opt/byconity
 ENV PATH="${PATH}:${APP_ROOT}/bin" \
    LD_LIBRARY_PATH="${APP_ROOT}/lib:${LD_LIBRARY_PATH}" \
    APP_ROOT="${APP_ROOT}"
-WORKDIR ${APP_ROOT}

 WORKDIR /
 ENTRYPOINT ["/tini", "--"]
--- a/docker/debian/base/build
+++ b/docker/debian/base/build
@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -e
+
+#ccache -s # uncomment to display CCache statistics
+cd /root/ByConity
+mkdir -p /root/ByConity/build_docker
+mkdir -p /root/ByConity/build_install
+git config --global --add safe.directory /root/ByConity
+
+git_hash=`git rev-parse HEAD`
+VERSION_DATE=`git show -s --format=%cs $git_hash`
+
+sed -i \
+    -e "s/set (VERSION_NAME [^) ]*/set (VERSION_NAME "ByConity"/g;" \
+    -e "s/set (VERSION_SCM [^) ]*/set (VERSION_SCM $VERSION_SCM/g;" \
+    cmake/version.cmake
+
+sed -i \
+    -e "s/SET(VERSION_GITHASH [^) ]*/SET(VERSION_GITHASH $git_hash/g;" \
+    -e "s/SET(VERSION_DATE [^) ]*/SET(VERSION_DATE $VERSION_DATE/g;" \
+    cmake/autogenerated_versions.txt
+
+cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${CMAKE_FLAGS} -DCMAKE_INSTALL_PREFIX=build_install -S . -B build_docker
+NUM_JOBS=$(( ($(nproc || grep -c ^processor /proc/cpuinfo) + 1) / 2 ))
+
+ninja -C build_docker -j $NUM_JOBS install
--- a/docker/debian/dev-env/Makefile
+++ b/docker/debian/dev-env/Makefile
@ -1,28 +1,21 @@
-UID := $(shell id -u)
-GID := $(shell id -g)
-BYCONITY_SOURCE := $(shell pwd)/../../..
-NAME := ${UID}-dev-env
+BYCONITY_SOURCE := $(realpath ../../..)
+CONTAINER := $(shell id -u)-dev-env

 run:
-	mkdir -p ~/.ccache
-	docker run -it \
+	mkdir -p $(HOME)/.ccache
+	docker run -it --rm \
 		--privileged \
 		--cap-add SYS_PTRACE \
-		--user=${UID}:${GID} \
 		--network host \
 		-e CCACHE_DIR=/root/.ccache \
-		-v ~/.m2:/root/.m2 \
-		-v ${BYCONITY_SOURCE}:/root/ByConity \
-		-v ~/.ccache:/root/.ccache \
-		--name ${NAME} \
+		-e http_proxy=${http_proxy} \
+		-e https_proxy=${https_proxy} \
+		-v $(HOME)/.m2:/root/.m2 \
+		-v $(HOME)/ck:/root/ck \
+		-v $(HOME)/.ccache:/root/.ccache \
+		--name $(CONTAINER) \
 		--workdir /root/ByConity \
-		-d byconity/dev-env
+		byconity/dev-env bash

 pull:
 	docker pull byconity/dev-env
-
-exec:
-	docker exec -it ${NAME} zsh
-
-stop:
-	docker stop ${NAME}
--- a/docker/debian/release/Dockerfile
+++ b/docker/debian/release/Dockerfile
@ -8,5 +8,18 @@ ENV PATH="${PATH}:${APP_ROOT}/bin" \
    LD_LIBRARY_PATH="${APP_ROOT}/lib:${LD_LIBRARY_PATH}" \
    APP_ROOT="${APP_ROOT}"

+# JRE
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+    default-jre \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV JAVA_HOME=/usr/lib/jvm/default-java
+RUN echo "/usr/lib/jvm/default-java/lib" >> /etc/ld.so.conf.d/libjvm.conf
+RUN echo "/usr/lib/jvm/default-java/lib/server" >> /etc/ld.so.conf.d/libjvm.conf
+RUN ldconfig
+ENV CLASSPATH=${CLASSPATH}:${APP_ROOT}/bin/
+
+# config runit, see base image
 RUN mkdir -p /etc/service/byconity
-COPY run /etc/service/
+COPY run /etc/service/byconity/run
+RUN chmod 755 /etc/service/byconity/run
--- a/docker/debian/release/Makefile
+++ b/docker/debian/release/Makefile
@ -0,0 +1,24 @@
+BYCONITY_SOURCE=$(realpath ../../..)
+BUILD_TYPE="Release"
+CMAKE_FLAGS="-DENABLE_JAVA_EXTENSIONS=ON"
+VERSION_SCM="0.2.1"
+
+pull:
+	docker pull byconity/debian-builder
+	docker pull byconity/debian-runner
+
+compile:
+	docker run --rm \
+	--privileged --cap-add SYS_PTRACE \
+	-v ~/.m2:/root/.m2 \
+	-v ${BYCONITY_SOURCE}:/root/ByConity \
+	-e BUILD_TYPE=${BUILD_TYPE} -e CMAKE_FLAGS=${CMAKE_FLAGS} -e VERSION_SCM=${VERSION_SCM} \
+	-it byconity/debian-builder bash /build.sh
+
+build:
+	cp -r ${BYCONITY_SOURCE}/build_install .
+	DOCKER_BUILDKIT=1 docker build -t byconity/byconity .
+
+release:
+	docker tag byconity/byconity byconity/byconity:$(tag)
+	docker push byconity/byconity:$(tag)
--- a/docker/executable_wrapper/Dockerfile
+++ b/docker/executable_wrapper/Dockerfile
@ -1,16 +0,0 @@
-FROM debian:11.6-slim
-
-RUN groupadd -r clickhouse --gid=101 \
-&& useradd -r -g clickhouse --uid=101 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse
-
-RUN mkdir -p /root/app/usr/bin \
-    && mkdir -p /root/app/logs && mkdir /root/app/data \
-    mkdir -p /root/app/config
-
-COPY --chown=clickhouse:clickhouse run.sh /root/app
-COPY --chown=clickhouse:clickhouse clickhouse /root/app/usr/bin
-COPY --chown=clickhouse:clickhouse foundationdb-clients_7.1.27-1_amd64.deb /root/app 
-RUN dpkg -i /root/app/foundationdb-clients_7.1.27-1_amd64.deb
-RUN chmod +x /root/app/run.sh
-
-ENTRYPOINT ["/root/app/run.sh"]
--- a/docker/executable_wrapper/Makefile
+++ b/docker/executable_wrapper/Makefile
@ -1,16 +1,8 @@
-tag?=0.1.3
-image:
-	test -f ../../build_docker/programs/foundationdb-clients_7.1.27-1_amd64.deb || curl -L -o ../../build_docker/programs/foundationdb-clients_7.1.27-1_amd64.deb https://github.com/apple/foundationdb/releases/download/7.1.27/foundationdb-clients_7.1.27-1_amd64.deb
-	cp ./app/run.sh ../../build_docker/programs/
-	docker build --network=host -t byconity/byconity-server ../../build_docker/programs -f Dockerfile
 image_pull:
-	docker pull byconity/byconity-server:stable
-image_push:
-	docker tag byconity/byconity-server:latest byconity/byconity-server:$(tag)
-	docker tag byconity/byconity-server:latest byconity/byconity-server:stable
-	docker push byconity/byconity-server:$(tag)
-	docker push byconity/byconity-server:stable
+	docker pull byconity/byconity:stable
+
 run_test:
-	docker run --rm byconity/byconity-server:$(tag) client --help
+	docker run --rm byconity/byconity:stable clickhouse client --help
+
 debug_image:
 	docker exec -it byconity-tso /bin/bash
--- a/docker/executable_wrapper/README.md
+++ b/docker/executable_wrapper/README.md
@ -6,23 +6,6 @@ This directory has the docker file, Makefile, shell file and configration files

 If you want to deploy ByConity into physical serves with this docker wrapper, you can leverage the run.sh file to start ByConity services with docker command. Please check out the instructions from [here](https://byconity.github.io/docs/deployment/docker-wrapper).

-## How to build
-
-To update docker image for executable wrapper, following below steps:
-
-Step 1: build executable binary for ByConity
-
- From root folder of git repo, rm folder `build_docker` if it exists
- cd to `docker/builder`
- Build binary with `make build` 
-
-Step 2:
-
- cd to `docker/executable_wrapper`
- edit the Makefile to update `tag` if necessary
- execute `make image`
- execute `make image_push`
-
 ## How to use
 - Execute following commands in order to bring up corresponding component:
 ```
--- a/docker/executable_wrapper/app/run.sh
+++ b/docker/executable_wrapper/app/run.sh
@ -1,3 +0,0 @@
-#!/bin/bash
-
-exec /root/app/usr/bin/clickhouse $@ 
--- a/docker/executable_wrapper/run.sh
+++ b/docker/executable_wrapper/run.sh
@ -1,4 +1,5 @@
 #!/bin/bash
+BYCONITY_IMAGE=byconity/byconity:stable

 function run_tso() {
    docker run -d --restart=on-failure \
@ -6,7 +7,7 @@ function run_tso() {
    --mount type=bind,source="$(pwd)"/logs,target=/root/app/logs \
    --expose 18689 \
    --network host \
-    --name byconity-tso byconity/byconity-server:stable tso-server --config-file /root/app/config/tso.xml
+    --name byconity-tso ${BYCONITY_IMAGE} clickhouse tso-server --config-file /root/app/config/tso.xml
 }

 function run_rm() {
@ -15,7 +16,7 @@ function run_rm() {
    --mount type=bind,source="$(pwd)"/logs,target=/root/app/logs \
    --expose 18702 \
    --network host \
-    --name byconity-rm byconity/byconity-server:stable resource-manager --config-file /root/app/config/rm.xml
+    --name byconity-rm ${BYCONITY_IMAGE} clickhouse resource-manager --config-file /root/app/config/rm.xml
 }

 function run_server() {
@ -29,7 +30,7 @@ function run_server() {
    --expose 18687 \
    --expose 18688 \
    --network host \
-    --name byconity-server byconity/byconity-server:stable server -C --config-file /root/app/config/server.xml 
+    --name byconity-server ${BYCONITY_IMAGE} clickhouse server -C --config-file /root/app/config/server.xml
 }

 function run_read_worker() {
@ -43,7 +44,7 @@ function run_read_worker() {
    --expose 18693 \
    --expose 18694 \
    --network host \
-    --name byconity-read-worker byconity/byconity-server:stable server -C --config-file /root/app/config/worker.xml 
+    --name byconity-read-worker ${BYCONITY_IMAGE} clickhouse server -C --config-file /root/app/config/worker.xml
 }

 function run_write_worker() {
@ -57,7 +58,7 @@ function run_write_worker() {
    --expose 18699 \
    --expose 18700 \
    --network host \
-    --name byconity-write-worker byconity/byconity-server:stable server -C --config-file /root/app/config/worker-write.xml 
+    --name byconity-write-worker ${BYCONITY_IMAGE} clickhouse server -C --config-file /root/app/config/worker-write.xml
 }

 function run_dm() {
@ -67,19 +68,19 @@ function run_dm() {
    --mount type=bind,source="$(pwd)"/data,target=/root/app/data \
    --expose 18965 \
    --network host \
-    --name byconity-dm byconity/byconity-server:stable daemon-manager --config-file /root/app/config/dm.xml 
+    --name byconity-dm ${BYCONITY_IMAGE} clickhouse daemon-manager --config-file /root/app/config/dm.xml
 }

 function run_cli() {
    docker run -it\
    --network host \
-    --name byconity-cli byconity/byconity-server:stable client --host 127.0.0.1 --port 18684
+    --name byconity-cli ${BYCONITY_IMAGE} clickhouse client --host 127.0.0.1 --port 18684
 }

 function run_cli2() {
    docker run -it\
    --network host \
-    --rm byconity/byconity-server:stable client --host $1 --port 18684
+    --rm ${BYCONITY_IMAGE} clickhouse client --host $1 --port 18684
 }

 function stop_byconity() {