From 13f79de3cd384771ac886d31a7f5737fc2470f11 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Date: Thu, 22 Jun 2023 09:29:06 +0200 Subject: [PATCH] chore: debug test (#29) * chore: debug test * test: help investigate Signed-off-by: Joan Fontanals Martinez * test: add timeout Signed-off-by: Joan Fontanals Martinez * test: run integration tests split Signed-off-by: Joan Fontanals Martinez --------- Signed-off-by: Joan Fontanals Martinez --- .github/workflows/ci.yml | 16 +++++++++++----- scripts/get-all-test-paths.sh | 9 +++------ setup.py | 3 +++ tests/integration/test_hnswlib_vectordb_serve.py | 6 ++++++ .../integration/test_inmemory_vectordb_serve.py | 6 ++++++ 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 517d75c..865bf06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: echo "::set-output name=matrix::$(bash scripts/get-all-test-paths.sh unit 1)" - id: set-matrix-integration run: | - echo "::set-output name=matrix::$(bash scripts/get-all-test-paths.sh integration_local 1)" + echo "::set-output name=matrix::$(bash scripts/get-all-test-paths.sh integration 1)" outputs: matrix-unit: ${{ steps.set-matrix-unit.outputs.matrix }} matrix-integration: ${{ steps.set-matrix-integration.outputs.matrix }} @@ -88,14 +88,17 @@ jobs: python -m pip install --upgrade pip python -m pip install wheel pip install pytest + pip install pytest-repeat + pip install pytest-timeout + pip install flaky pip install . #pip install -U docarray[hnswlib]>=0.33.0 - pip install git+https://github.com/jina-ai/jina.git@fix-handle-list-float + pip install git+https://github.com/jina-ai/jina.git@help-investigate pip install git+https://github.com/docarray/docarray.git@main - name: Test id: test run: | - pytest -v -s ${{ matrix.test-path }} + pytest -v -s --force-flaky --min-passes 1 --max-runs 5 ${{ matrix.test-path }} timeout-minutes: 30 integration-tests: @@ -119,14 +122,17 @@ jobs: python -m pip install --upgrade pip python -m pip install wheel pip install pytest + pip install pytest-repeat + pip install pytest-timeout + pip install flaky pip install . #pip install -U docarray[hnswlib]>=0.33.0 - pip install git+https://github.com/jina-ai/jina.git@fix-handle-list-float + pip install git+https://github.com/jina-ai/jina.git@help-investigate pip install git+https://github.com/docarray/docarray.git@main - name: Test id: test run: | - pytest -v -s ${{ matrix.test-path }} + pytest -v -s --force-flaky --min-passes 1 --max-runs 5 ${{ matrix.test-path }} timeout-minutes: 30 # just for blocking the merge until all parallel integration-tests are successful diff --git a/scripts/get-all-test-paths.sh b/scripts/get-all-test-paths.sh index 5ce9378..e3bbe35 100644 --- a/scripts/get-all-test-paths.sh +++ b/scripts/get-all-test-paths.sh @@ -7,16 +7,13 @@ DEFAULT_BATCH_SIZE=5 BATCH_SIZE="${2:-$DEFAULT_BATCH_SIZE}" declare -a unit_tests=($(find tests/unit -name "test_*.py")) -declare -a integration_tests_local=($(find tests/integration/local -name "test_*.py")) -declare -a integration_tests_jcloud=($(find tests/integration/jcloud -name "test_*.py")) +declare -a integration_tests=($(find tests/integration -name "test_*.py")) declare -a all_tests=("${unit_tests[@]}" "${integration_tests[@]}") if [ "$TEST_SUITE" == "unit" ]; then dest="$(echo "${unit_tests[@]}" | xargs -n$BATCH_SIZE)" -elif [[ "$TEST_SUITE" == "integration_local" ]]; then - dest="$(echo "${integration_tests_local[@]}" | xargs -n$BATCH_SIZE)" -elif [[ "$TEST_SUITE" == "integration_jcloud" ]]; then - dest="$(echo "${integration_tests_jcloud[@]}" | xargs -n$BATCH_SIZE)" +elif [[ "$TEST_SUITE" == "integration" ]]; then + dest="$(echo "${integration_tests[@]}" | xargs -n$BATCH_SIZE)" else dest="$(echo "${all_tests[@]}" | xargs -n$BATCH_SIZE)" fi diff --git a/setup.py b/setup.py index 4ce82ea..3da8d7c 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,9 @@ setup( 'test': [ 'pytest', 'pytest-asyncio', + 'pytest-repeat', + 'flaky', + 'pytest-timeout' ], }, install_requires=requirements, diff --git a/tests/integration/test_hnswlib_vectordb_serve.py b/tests/integration/test_hnswlib_vectordb_serve.py index a4c84e6..1728b0b 100644 --- a/tests/integration/test_hnswlib_vectordb_serve.py +++ b/tests/integration/test_hnswlib_vectordb_serve.py @@ -1,3 +1,4 @@ +import multiprocessing import pytest import random import time @@ -22,6 +23,7 @@ def docs_to_index(): for _ in range(2000)]) +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -42,6 +44,7 @@ def test_hnswlib_vectordb_batch(docs_to_index, replicas, shards, protocol, tmpdi assert res.scores[0] < 0.001 # some precision issues, should be 0.0 +@pytest.mark.timeout(270) @pytest.mark.parametrize('limit', [1, 10, 2000, 2500]) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @@ -61,6 +64,7 @@ def test_hnswlib_vectordb_single_query(docs_to_index, limit, replicas, shards, p assert resp.scores[0] < 0.001 # some precision issues, should be 0.0 +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -90,6 +94,7 @@ def test_hnswlib_vectordb_delete(docs_to_index, replicas, shards, protocol, tmpd assert resp.text != resp.matches[0].text +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -118,6 +123,7 @@ def test_hnswlib_vectordb_udpate_text(docs_to_index, replicas, shards, protocol, assert resp.matches[0].text == resp.text + '_changed' +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) diff --git a/tests/integration/test_inmemory_vectordb_serve.py b/tests/integration/test_inmemory_vectordb_serve.py index ac42ffc..a150c3f 100644 --- a/tests/integration/test_inmemory_vectordb_serve.py +++ b/tests/integration/test_inmemory_vectordb_serve.py @@ -1,3 +1,4 @@ +import multiprocessing import pytest import random import string @@ -22,6 +23,7 @@ def docs_to_index(): for _ in range(2000)]) +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -42,6 +44,7 @@ def test_inmemory_vectordb_batch(docs_to_index, replicas, shards, protocol, tmpd assert res.scores[0] > 0.99 # some precision issues, should be 1.0 +@pytest.mark.timeout(270) @pytest.mark.parametrize('limit', [1, 10, 2000, 2500]) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @@ -61,6 +64,7 @@ def test_inmemory_vectordb_single_query(docs_to_index, limit, replicas, shards, assert resp.scores[0] > 0.99 # some precision issues, should be 1.0 +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -90,6 +94,7 @@ def test_inmemory_vectordb_delete(docs_to_index, replicas, shards, protocol, tmp assert resp.text != resp.matches[0].text +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -118,6 +123,7 @@ def test_inmemory_vectordb_udpate_text(docs_to_index, replicas, shards, protocol assert resp.matches[0].text == resp.text + '_changed' +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket'])