jianmu
/
openobserve
mirror of https://github.com/openobserve/openobserve
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: API Integration Tests workflow (#2324) 

- Simplify the API integration tests by removing steps not needed (Node,
k8s, building docker, etc).
- Testing the UI, linting, formatting, etc should be done in a separate
workflow file
- Simplify Rust cache workflow.
  - By default it caches /target
- Removed Dockerfile and k8s deployment from CI.
  - You can just run `OpenObserve` in the background
- Replace `ubicloud-standard-16` with `ubuntu-latest` which is *FREE*.
  - Using 16 cores, 64GB of ram is an overkill for running tests.
- Don't run the CI if changes to `.md` files are detected.

Notes:
- Before caching building `OpenObserve` takes around 5-7 mins, after
cache it goes down to 1-2 mins.
This commit is contained in:
Juan Calderon-Perez 2023-12-29 01:39:09 -05:00 committed by GitHub
parent a4bae1a584
commit 66968e8265
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 49 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
.github/workflows/api-testing.yml vendored
View File

@ -1,7 +1,16 @@
name: Run integration tests against latest code
name: API Integration Test
on:
push:
branches:
- "main"
paths-ignore:
- "**.md"
pull_request:
branches:
- "*"
paths-ignore:
- "**.md"
env:
COLUMNS: 150
@ -11,109 +20,62 @@ env:
jobs:
build:
runs-on:
labels: ubicloud-standard-16
strategy:
matrix:
python-version: ["3.11"]
runs-on: ubuntu-latest
steps:
- name: Remove unused tools
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- name: Create k8s Kind Cluster
uses: helm/kind-action@v1.8.0
with:
cluster_name: openobserve-ci-test
- name: Clone the current repo
- name: Checkout git repo
uses: actions/checkout@v4
- name: Print out the repo structure
run: ls -al
- name: Setup Rust Toolchain
uses: dtolnay/rust-toolchain@master
with:
toolchain: nightly-2023-12-24
components: rustfmt,clippy,llvm-tools
- uses: Swatinem/rust-cache@v2
- name: Setup Rust Cache
uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
cache-directories:
./target
- name: Install Protoc
uses: arduino/setup-protoc@v2
with:
version: "21.12"
- uses: actions/setup-node@v4
with:
node-version: 18
- name: Build binary
# run: cargo build --release --features mimalloc --target x86_64-unknown-linux-musl
- name: Build OpenObserve debug binary
run: cargo build --features mimalloc
# - run: mkdir -p bin ; cp target/x86_64-unknown-linux-musl/release/openobserve bin/openobserve
- run: mkdir -p bin ; cp target/debug/openobserve bin/openobserve
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver-opts: network=host
- name: Test locations
run: ls -al
- name: Build and push to local registry
uses: docker/build-push-action@v5
with:
context: "."
file: ./tests/api-testing/ci/Dockerfile.openobserve
# push: true
outputs: type=docker,dest=/tmp/openobserve.tar
# tags: localhost:5000/openobserve/openobserve:latest-amd64
tags: openobserve:latest-amd64
cache-from: type=gha
cache-to: type=gha,mode=max
# - name: Inspect
# run: |
# docker buildx imagetools inspect localhost:5000/openobserve/openobserve:latest-amd64
- name: Load built image into kind
# run: kind load docker-image localhost:5000/openobserve/openobserve:latest-amd64 --name openobserve-ci-test
run: kind load image-archive /tmp/openobserve.tar --name openobserve-ci-test
- name: Deploy openobserve
# run: kubectl create ns openobserve && kubectl apply -f https://raw.githubusercontent.com/zinclabs/openobserve/main/deploy/k8s/statefulset.yaml
run: kubectl create ns openobserve && kubectl apply -f ./tests/api-testing/ci/statefulset.yaml
- name: Set up Python ${{ matrix.python-version }}
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Wait for 10s, for the deployment to complete.
run: sleep 10s
shell: bash
- name: Check the services - 1
run: kubectl get services --all-namespaces
- name: Check the services - 2
run: kubectl get pods --all-namespaces
- name: Check the logs
run: kubectl logs openobserve-0 -n openobserve
- name: Port forward the 5080
run: kubectl -n openobserve port-forward svc/openobserve 5080:5080 &
# You can test your matrix by printing the current Python version
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Enable caching and define a working directory
python-version: "3.11.6"
- name: Setup rye
uses: eifinger/setup-rye@v1
with:
enable-cache: true
working-directory: 'tests/api-testing/'
version: '0.16.0'
- name: Start OpenObserve
run: target/debug/openobserve &
- name: Wait for start
run: sleep 15
- name: Pin cpython
run: rye pin cpython@3.11.6
working-directory: tests/api-testing/
- name: Rye sync
run: rye sync
working-directory: tests/api-testing/
- name: Run tests
run: cd tests/api-testing/ && rye pin cpython@3.11.6 && rye sync && rye run pytest --force-sugar
# Try to test the openapi generator as well
# - name: openapi-generator test
# uses: openapi-generators/openapitools-generator-action@v1
# with:
# generator: python
# openapi-url: http://localhost:5080/api-doc/openapi.json
# generator-tag: v7.0.1
# command-args: '--package-name py-openobserve o ./py-openobserve --additional-properties=useSingleRequestParameter=true,packageVersion=2.0.0'
run: rye run pytest --force-sugar
working-directory: tests/api-testing/

2
tests/api-testing/.python-version
View File

@ -1 +1 @@
3.11.6
3.11.7

8
tests/api-testing/ci/Dockerfile.openobserve
View File

@ -1,8 +0,0 @@
# syntax=docker/dockerfile:1
FROM public.ecr.aws/debian/debian:bookworm-slim as runtime
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates
RUN update-ca-certificates
COPY ./bin/openobserve /
RUN ["/openobserve", "init-dir", "-p", "/data/"]
CMD ["/openobserve"]

81
tests/api-testing/ci/statefulset.yaml
View File

@ -1,81 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: openobserve
namespace: openobserve
spec:
clusterIP: None
selector:
app: openobserve
ports:
- name: http
port: 5080
targetPort: 5080
---
# create statefulset
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: openobserve
namespace: openobserve
labels:
name: openobserve
spec:
serviceName: openobserve
replicas: 1
selector:
matchLabels:
name: openobserve
app: openobserve
template:
metadata:
labels:
name: openobserve
app: openobserve
spec:
securityContext:
fsGroup: 2000
runAsUser: 10000
runAsGroup: 3000
runAsNonRoot: true
# terminationGracePeriodSeconds: 0
containers:
- name: openobserve
# image: localhost:5000/openobserve/openobserve:latest-amd64
image: openobserve:latest-amd64
env:
- name: ZO_ROOT_USER_EMAIL
value: root@example.com
- name: ZO_ROOT_USER_PASSWORD
value: Complexpass#123
- name: ZO_DATA_DIR
value: /data
# command: ["/bin/bash", "-c", "while true; do sleep 1; done"]
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 4096m
memory: 2048Mi
requests:
cpu: 256m
memory: 50Mi
ports:
- containerPort: 5080
name: http
volumeMounts:
- name: data
mountPath: /data
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
# storageClassName: default
# NOTE: You can increase the storage size
resources:
requests:
storage: 10Gi

2
tests/api-testing/pyproject.toml
View File

@ -10,7 +10,7 @@ dependencies = [
"ruff>=0.1.8",
]
readme = "README.md"
requires-python = ">= 3.8"
requires-python = ">= 3.11"
[build-system]
requires = ["hatchling"]