mirror of https://github.com/apache/cassandra
10236 lines
501 KiB
Plaintext
10236 lines
501 KiB
Plaintext
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
version: 2
|
|
jobs:
|
|
j11_jvm_upgrade_dtests_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_dtests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j17_without_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j17_without_vnodes)
|
|
no_output_timeout: 15m
|
|
command: "echo \"cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j17_without_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j17_without_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_jvm_dtests_latest_vnode:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine distributed Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16 -Djvm_dtests.latest=true' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_cqlshlib_cython_tests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Run cqlshlib Unit Tests
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
export cython="yes"
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra/
|
|
ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true
|
|
./pylib/cassandra-cqlsh-tests.sh $(pwd)
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/pylib
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_utests_fqltool_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_dtests_latest_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Run repeated Python DTests
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
|
|
echo "Repeated dtest name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated dtest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
|
|
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_DTESTS} $count times"
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
|
|
tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
|
|
|
|
stop_on_failure_arg=""
|
|
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
|
|
stop_on_failure_arg="-x"
|
|
fi
|
|
|
|
vnodes_args=""
|
|
if true; then
|
|
vnodes_args="--use-vnodes --num-tokens=16"
|
|
fi
|
|
|
|
upgrade_arg=""
|
|
if false; then
|
|
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
|
|
fi
|
|
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_cqlshlib_cython_tests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Run cqlshlib Unit Tests
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
export cython="yes"
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra/
|
|
ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true
|
|
./pylib/cassandra-cqlsh-tests.sh $(pwd)
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/pylib
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_utests_compression:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist-compression)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_cqlsh_dtests_py38_vnode:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j17_with_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j17_with_vnodes)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt"
|
|
cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.8
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j17_with_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j17_with_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_dtests_vnode_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Run repeated Python DTests
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
|
|
echo "Repeated dtest name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated dtest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
|
|
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_DTESTS} $count times"
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
|
|
tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
|
|
|
|
stop_on_failure_arg=""
|
|
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
|
|
stop_on_failure_arg="-x"
|
|
fi
|
|
|
|
vnodes_args=""
|
|
if true; then
|
|
vnodes_args="--use-vnodes --num-tokens=16"
|
|
fi
|
|
|
|
upgrade_arg=""
|
|
if false; then
|
|
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
|
|
fi
|
|
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_dtests_vnode_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Run repeated Python DTests
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
|
|
echo "Repeated dtest name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated dtest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
|
|
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_DTESTS} $count times"
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
|
|
tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
|
|
|
|
stop_on_failure_arg=""
|
|
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
|
|
stop_on_failure_arg="-x"
|
|
fi
|
|
|
|
vnodes_args=""
|
|
if true; then
|
|
vnodes_args="--use-vnodes --num-tokens=16"
|
|
fi
|
|
|
|
upgrade_arg=""
|
|
if false; then
|
|
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
|
|
fi
|
|
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_cqlsh_dtests_py311_latest:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j17_dtests_latest)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_latest)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_latest_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_latest_raw /tmp/all_dtest_tests_j17_dtests_latest\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_latest_raw > /tmp/all_dtest_tests_j17_dtests_latest || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_latest > /tmp/split_dtest_tests_j17_dtests_latest.txt\ncat /tmp/split_dtest_tests_j17_dtests_latest.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_latest_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_latest_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j17_dtests_latest)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j17_dtests_latest_final.txt"
|
|
cat /tmp/split_dtest_tests_j17_dtests_latest_final.txt
|
|
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.11
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_latest_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_latest.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j17_dtests_latest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j17_dtests_latest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_utests_system_keyspace_directory:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist-system-keyspace-directory)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_utests_stress_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_unit_tests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_cqlsh_dtests_py38_latest:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_dtests_latest)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_latest)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_latest_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_latest_raw /tmp/all_dtest_tests_j11_dtests_latest\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_latest_raw > /tmp/all_dtest_tests_j11_dtests_latest || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_latest > /tmp/split_dtest_tests_j11_dtests_latest.txt\ncat /tmp/split_dtest_tests_j11_dtests_latest.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_latest_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_latest_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_dtests_latest)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j11_dtests_latest_final.txt"
|
|
cat /tmp/split_dtest_tests_j11_dtests_latest_final.txt
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.8
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_latest_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_latest.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_dtests_latest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_dtests_latest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_utests_system_keyspace_directory_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_cqlsh_dtests_py311:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_without_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_without_vnodes)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
|
|
cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
|
|
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.11
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_without_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_without_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_dtests_large_vnode_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Run repeated Python DTests
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
|
|
echo "Repeated dtest name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated dtest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
|
|
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
|
|
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
|
|
|
|
stop_on_failure_arg=""
|
|
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
|
|
stop_on_failure_arg="-x"
|
|
fi
|
|
|
|
vnodes_args=""
|
|
if true; then
|
|
vnodes_args="--use-vnodes --num-tokens=16"
|
|
fi
|
|
|
|
upgrade_arg=""
|
|
if false; then
|
|
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
|
|
fi
|
|
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_dtests_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Run repeated Python DTests
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
|
|
echo "Repeated dtest name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated dtest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
|
|
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_DTESTS} $count times"
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
|
|
tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
|
|
|
|
stop_on_failure_arg=""
|
|
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
|
|
stop_on_failure_arg="-x"
|
|
fi
|
|
|
|
vnodes_args=""
|
|
if false; then
|
|
vnodes_args="--use-vnodes --num-tokens=16"
|
|
fi
|
|
|
|
upgrade_arg=""
|
|
if false; then
|
|
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
|
|
fi
|
|
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_unit_tests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_utests_latest_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-latest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-latest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_cqlsh_dtests_py311:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j17_without_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j17_without_vnodes)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt"
|
|
cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt
|
|
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.11
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j17_without_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j17_without_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_cqlsh_dtests_py38:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_without_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_without_vnodes)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
|
|
cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.8
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_without_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_without_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_utests_stress_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_utests_compression_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_utests_oa:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist-oa)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist-oa -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_repeated_ant_test:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run repeated JUnit test
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
|
|
echo "Repeated utest class name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated utest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
|
|
echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
|
|
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
|
|
target=${REPEATED_ANT_TEST_TARGET}
|
|
class_path=${REPEATED_ANT_TEST_CLASS}
|
|
class_name="${class_path##*.}"
|
|
|
|
# Prepare the -Dtest.name argument.
|
|
# It can be the fully qualified class name or the short class name, depending on the target.
|
|
if [[ $target == "test" || \
|
|
$target == "test-cdc" || \
|
|
$target == "test-compression" || \
|
|
$target == "test-latest" || \
|
|
$target == "test-oa" || \
|
|
$target == "test-system-keyspace-directory" || \
|
|
$target == "fqltool-test" || \
|
|
$target == "long-test" || \
|
|
$target == "stress-test" || \
|
|
$target == "test-simulator-dtest" ]]; then
|
|
name="-Dtest.name=$class_name"
|
|
else
|
|
name="-Dtest.name=$class_path"
|
|
fi
|
|
|
|
# Prepare the -Dtest.methods argument, which is optional
|
|
if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
|
|
methods=""
|
|
else
|
|
methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
|
|
fi
|
|
|
|
# Prepare the JVM dtests vnodes argument, which is optional
|
|
vnodes_args=""
|
|
if ${REPEATED_ANT_TEST_VNODES}; then
|
|
vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
|
|
fi
|
|
|
|
# Run the test target as many times as requested collecting the exit code,
|
|
# stopping the iteration only if stop_on_failure is set.
|
|
exit_code="$?"
|
|
for i in $(seq -w 1 $count); do
|
|
|
|
echo "Running test iteration $i of $count"
|
|
|
|
# run the test
|
|
status="passes"
|
|
if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
|
|
status="fails"
|
|
exit_code=1
|
|
fi
|
|
|
|
# move the stdout output file
|
|
dest=/tmp/results/repeated_utest/stdout/${status}/${i}
|
|
mkdir -p $dest
|
|
mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
|
|
|
|
# move the XML output files
|
|
source=build/test/output
|
|
dest=/tmp/results/repeated_utest/output/${status}/${i}
|
|
mkdir -p $dest
|
|
if [[ -d $source && -n "$(ls $source)" ]]; then
|
|
mv $source/* $dest/
|
|
fi
|
|
|
|
# move the log files
|
|
source=build/test/logs
|
|
dest=/tmp/results/repeated_utest/logs/${status}/${i}
|
|
mkdir -p $dest
|
|
if [[ -d $source && -n "$(ls $source)" ]]; then
|
|
mv $source/* $dest/
|
|
fi
|
|
|
|
# maybe stop iterations on test failure
|
|
if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
|
|
break
|
|
fi
|
|
done
|
|
|
|
(exit ${exit_code})
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utest/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utest/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utest/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utest/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_dtests_large_vnode:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_large_with_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_large_with_vnodes_raw /tmp/all_dtest_tests_j11_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_large_with_vnodes_raw > /tmp/all_dtest_tests_j11_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_large_with_vnodes > /tmp/split_dtest_tests_j11_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_large_with_vnodes)
|
|
no_output_timeout: 15m
|
|
command: "echo \"cat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests --log-cli-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_large_with_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_large_with_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_dtests_large_vnode_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Run repeated Python DTests
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
|
|
echo "Repeated dtest name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated dtest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
|
|
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
|
|
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
|
|
|
|
stop_on_failure_arg=""
|
|
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
|
|
stop_on_failure_arg="-x"
|
|
fi
|
|
|
|
vnodes_args=""
|
|
if true; then
|
|
vnodes_args="--use-vnodes --num-tokens=16"
|
|
fi
|
|
|
|
upgrade_arg=""
|
|
if false; then
|
|
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
|
|
fi
|
|
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_dtests_large:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_large_without_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_large_without_vnodes_raw /tmp/all_dtest_tests_j11_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_large_without_vnodes_raw > /tmp/all_dtest_tests_j11_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_large_without_vnodes > /tmp/split_dtest_tests_j11_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_large_without_vnodes)
|
|
no_output_timeout: 15m
|
|
command: "echo \"cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --only-resource-intensive-tests --force-resource-intensive-tests --log-cli-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_large_without_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_large_without_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_utests_system_keyspace_directory_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_utests_oa:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist-oa)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist-oa -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_upgrade_dtests_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Run repeated Python DTests
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
|
|
echo "Repeated dtest name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated dtest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then
|
|
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_UPGRADE_DTESTS} $count times"
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
|
|
tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g")
|
|
|
|
stop_on_failure_arg=""
|
|
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
|
|
stop_on_failure_arg="-x"
|
|
fi
|
|
|
|
vnodes_args=""
|
|
if false; then
|
|
vnodes_args="--use-vnodes --num-tokens=16"
|
|
fi
|
|
|
|
upgrade_arg=""
|
|
if true; then
|
|
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
|
|
fi
|
|
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_jvm_dtests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine distributed Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_repeated_ant_test:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run repeated JUnit test
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
|
|
echo "Repeated utest class name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated utest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
|
|
echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
|
|
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
|
|
target=${REPEATED_ANT_TEST_TARGET}
|
|
class_path=${REPEATED_ANT_TEST_CLASS}
|
|
class_name="${class_path##*.}"
|
|
|
|
# Prepare the -Dtest.name argument.
|
|
# It can be the fully qualified class name or the short class name, depending on the target.
|
|
if [[ $target == "test" || \
|
|
$target == "test-cdc" || \
|
|
$target == "test-compression" || \
|
|
$target == "test-latest" || \
|
|
$target == "test-oa" || \
|
|
$target == "test-system-keyspace-directory" || \
|
|
$target == "fqltool-test" || \
|
|
$target == "long-test" || \
|
|
$target == "stress-test" || \
|
|
$target == "test-simulator-dtest" ]]; then
|
|
name="-Dtest.name=$class_name"
|
|
else
|
|
name="-Dtest.name=$class_path"
|
|
fi
|
|
|
|
# Prepare the -Dtest.methods argument, which is optional
|
|
if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
|
|
methods=""
|
|
else
|
|
methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
|
|
fi
|
|
|
|
# Prepare the JVM dtests vnodes argument, which is optional
|
|
vnodes_args=""
|
|
if ${REPEATED_ANT_TEST_VNODES}; then
|
|
vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
|
|
fi
|
|
|
|
# Run the test target as many times as requested collecting the exit code,
|
|
# stopping the iteration only if stop_on_failure is set.
|
|
exit_code="$?"
|
|
for i in $(seq -w 1 $count); do
|
|
|
|
echo "Running test iteration $i of $count"
|
|
|
|
# run the test
|
|
status="passes"
|
|
if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
|
|
status="fails"
|
|
exit_code=1
|
|
fi
|
|
|
|
# move the stdout output file
|
|
dest=/tmp/results/repeated_utest/stdout/${status}/${i}
|
|
mkdir -p $dest
|
|
mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
|
|
|
|
# move the XML output files
|
|
source=build/test/output
|
|
dest=/tmp/results/repeated_utest/output/${status}/${i}
|
|
mkdir -p $dest
|
|
if [[ -d $source && -n "$(ls $source)" ]]; then
|
|
mv $source/* $dest/
|
|
fi
|
|
|
|
# move the log files
|
|
source=build/test/logs
|
|
dest=/tmp/results/repeated_utest/logs/${status}/${i}
|
|
mkdir -p $dest
|
|
if [[ -d $source && -n "$(ls $source)" ]]; then
|
|
mv $source/* $dest/
|
|
fi
|
|
|
|
# maybe stop iterations on test failure
|
|
if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
|
|
break
|
|
fi
|
|
done
|
|
|
|
(exit ${exit_code})
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utest/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utest/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utest/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utest/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_build:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Clone Cassandra Repository (via git)
|
|
command: |
|
|
git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
|
|
- run:
|
|
name: Build Cassandra
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
cd ~/cassandra
|
|
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
|
|
for x in $(seq 1 3); do
|
|
${ANT_HOME}/bin/ant clean realclean jar
|
|
RETURN="$?"
|
|
if [ "${RETURN}" -eq "0" ]; then
|
|
break
|
|
fi
|
|
done
|
|
# Exit, if we didn't build successfully
|
|
if [ "${RETURN}" -ne "0" ]; then
|
|
echo "Build failed with exit code: ${RETURN}"
|
|
exit ${RETURN}
|
|
fi
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Run static code analysis
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
cd ~/cassandra
|
|
ant check
|
|
- persist_to_workspace:
|
|
root: /home/cassandra
|
|
paths:
|
|
- cassandra
|
|
- .m2
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_utests_cdc:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist-cdc)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_utests_cdc_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_jvm_dtests_latest_vnode:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine distributed Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16 -Djvm_dtests.latest=true' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_utests_fqltool:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Run Unit Tests (fqltool-test)
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
ant fqltool-test -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_jvm_dtests_latest_vnode_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_jvm_dtests_latest_vnode_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_dtest_jars_build:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Build Cassandra DTest jars
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
cd ~/cassandra
|
|
mkdir ~/dtest_jars
|
|
git remote add apache https://github.com/apache/cassandra.git
|
|
for branch in cassandra-4.0 cassandra-4.1 cassandra-5.0 trunk; do
|
|
# check out the correct cassandra version:
|
|
git remote set-branches --add apache '$branch'
|
|
git fetch --depth 1 apache $branch
|
|
git checkout $branch
|
|
git clean -fd
|
|
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
|
|
for x in $(seq 1 3); do
|
|
${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar -Dno-checkstyle=true -Drat.skip=true
|
|
RETURN="$?"
|
|
if [ "${RETURN}" -eq "0" ]; then
|
|
cp build/dtest*.jar ~/dtest_jars
|
|
break
|
|
fi
|
|
done
|
|
# Exit, if we didn't build successfully
|
|
if [ "${RETURN}" -ne "0" ]; then
|
|
echo "Build failed with exit code: ${RETURN}"
|
|
exit ${RETURN}
|
|
fi
|
|
done
|
|
# and build the dtest-jar for the branch under test
|
|
${ANT_HOME}/bin/ant realclean
|
|
git checkout origin/$CIRCLE_BRANCH
|
|
git clean -fd
|
|
for x in $(seq 1 3); do
|
|
${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar -Dno-checkstyle=true -Drat.skip=true
|
|
RETURN="$?"
|
|
if [ "${RETURN}" -eq "0" ]; then
|
|
cp build/dtest*.jar ~/dtest_jars
|
|
break
|
|
fi
|
|
done
|
|
# Exit, if we didn't build successfully
|
|
if [ "${RETURN}" -ne "0" ]; then
|
|
echo "Build failed with exit code: ${RETURN}"
|
|
exit ${RETURN}
|
|
fi
|
|
ls -l ~/dtest_jars
|
|
no_output_timeout: 15m
|
|
- persist_to_workspace:
|
|
root: /home/cassandra
|
|
paths:
|
|
- dtest_jars
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_utests_long:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Run Unit Tests (long-test)
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
ant long-test -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_unit_tests_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_simulator_dtests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Simulator Tests
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
ant test-simulator-dtest -Dno-build-test=true
|
|
no_output_timeout: 30m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_jvm_upgrade_dtests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine distributed Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_utests_stress:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Run Unit Tests (stress-test)
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
ant stress-test -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_utests_cdc_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_dtests_vnode:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j17_with_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j17_with_vnodes)
|
|
no_output_timeout: 15m
|
|
command: "echo \"cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j17_with_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j17_with_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_dtests_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Run repeated Python DTests
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
|
|
echo "Repeated dtest name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated dtest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
|
|
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_DTESTS} $count times"
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
|
|
tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
|
|
|
|
stop_on_failure_arg=""
|
|
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
|
|
stop_on_failure_arg="-x"
|
|
fi
|
|
|
|
vnodes_args=""
|
|
if false; then
|
|
vnodes_args="--use-vnodes --num-tokens=16"
|
|
fi
|
|
|
|
upgrade_arg=""
|
|
if false; then
|
|
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
|
|
fi
|
|
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_utests_compression:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist-compression)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_utests_latest:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist-latest)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist-latest -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_dtests_latest_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Run repeated Python DTests
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
|
|
echo "Repeated dtest name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated dtest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
|
|
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_DTESTS} $count times"
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
|
|
tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
|
|
|
|
stop_on_failure_arg=""
|
|
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
|
|
stop_on_failure_arg="-x"
|
|
fi
|
|
|
|
vnodes_args=""
|
|
if true; then
|
|
vnodes_args="--use-vnodes --num-tokens=16"
|
|
fi
|
|
|
|
upgrade_arg=""
|
|
if false; then
|
|
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
|
|
fi
|
|
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_utests_latest_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-latest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-latest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_upgrade_dtests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_upgradetests_without_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j11_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j11_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_upgradetests_without_vnodes > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_upgradetests_without_vnodes)
|
|
no_output_timeout: 15m
|
|
command: "echo \"cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-cli-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_upgradetests_without_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_upgradetests_without_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_dtests_large_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Run repeated Python DTests
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
|
|
echo "Repeated dtest name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated dtest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
|
|
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
|
|
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
|
|
|
|
stop_on_failure_arg=""
|
|
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
|
|
stop_on_failure_arg="-x"
|
|
fi
|
|
|
|
vnodes_args=""
|
|
if false; then
|
|
vnodes_args="--use-vnodes --num-tokens=16"
|
|
fi
|
|
|
|
upgrade_arg=""
|
|
if false; then
|
|
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
|
|
fi
|
|
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_utests_compression_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_utests_fqltool:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Run Unit Tests (fqltool-test)
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
ant fqltool-test -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_dtests_large_vnode:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j17_large_with_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_large_with_vnodes_raw /tmp/all_dtest_tests_j17_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_large_with_vnodes_raw > /tmp/all_dtest_tests_j17_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_large_with_vnodes > /tmp/split_dtest_tests_j17_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j17_large_with_vnodes)
|
|
no_output_timeout: 15m
|
|
command: "echo \"cat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests --log-cli-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j17_large_with_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j17_large_with_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_cqlsh_dtests_py38_vnode:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_with_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_with_vnodes)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
|
|
cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.8
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_with_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_with_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_simulator_dtests_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_jvm_dtests_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_cqlsh_dtests_py311_latest:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_dtests_latest)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_latest)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_latest_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_latest_raw /tmp/all_dtest_tests_j11_dtests_latest\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_latest_raw > /tmp/all_dtest_tests_j11_dtests_latest || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_latest > /tmp/split_dtest_tests_j11_dtests_latest.txt\ncat /tmp/split_dtest_tests_j11_dtests_latest.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_latest_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_latest_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_dtests_latest)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j11_dtests_latest_final.txt"
|
|
cat /tmp/split_dtest_tests_j11_dtests_latest_final.txt
|
|
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.11
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_latest_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_latest.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_dtests_latest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_dtests_latest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_cqlshlib_tests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Run cqlshlib Unit Tests
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra/
|
|
ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true
|
|
./pylib/cassandra-cqlsh-tests.sh $(pwd)
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/pylib
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_cqlsh_dtests_py311_vnode:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_with_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_with_vnodes)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
|
|
cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
|
|
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.11
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_with_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_with_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_utests_system_keyspace_directory:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist-system-keyspace-directory)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_cqlsh_dtests_py38_latest:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j17_dtests_latest)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_latest)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_latest_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_latest_raw /tmp/all_dtest_tests_j17_dtests_latest\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_latest_raw > /tmp/all_dtest_tests_j17_dtests_latest || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_latest > /tmp/split_dtest_tests_j17_dtests_latest.txt\ncat /tmp/split_dtest_tests_j17_dtests_latest.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_latest_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_latest_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j17_dtests_latest)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j17_dtests_latest_final.txt"
|
|
cat /tmp/split_dtest_tests_j17_dtests_latest_final.txt
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.8
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_latest_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_latest.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j17_dtests_latest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j17_dtests_latest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_cqlsh_dtests_py311_vnode:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j17_with_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_with_vnodes_raw /tmp/all_dtest_tests_j17_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_with_vnodes_raw > /tmp/all_dtest_tests_j17_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_with_vnodes > /tmp/split_dtest_tests_j17_with_vnodes.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_with_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j17_with_vnodes)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt"
|
|
cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt
|
|
|
|
source ~/env3.11/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.11
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_with_vnodes_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j17_with_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j17_with_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_jvm_dtests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine distributed Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_dtests_latest:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j17_dtests_latest)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_dtests_latest)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_dtests_latest_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_dtests_latest_raw /tmp/all_dtest_tests_j17_dtests_latest\nelse\n grep -e '' /tmp/all_dtest_tests_j17_dtests_latest_raw > /tmp/all_dtest_tests_j17_dtests_latest || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_dtests_latest > /tmp/split_dtest_tests_j17_dtests_latest.txt\ncat /tmp/split_dtest_tests_j17_dtests_latest.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_dtests_latest_final.txt\ncat /tmp/split_dtest_tests_j17_dtests_latest_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j17_dtests_latest)
|
|
no_output_timeout: 15m
|
|
command: "echo \"cat /tmp/split_dtest_tests_j17_dtests_latest_final.txt\"\ncat /tmp/split_dtest_tests_j17_dtests_latest_final.txt\n\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_dtests_latest_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --log-cli-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_dtests_latest.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j17_dtests_latest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j17_dtests_latest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_utests_latest:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist-latest)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist-latest -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_utests_stress:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Run Unit Tests (stress-test)
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
ant stress-test -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_build:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Clone Cassandra Repository (via git)
|
|
command: |
|
|
git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
|
|
- run:
|
|
name: Build Cassandra
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
cd ~/cassandra
|
|
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
|
|
for x in $(seq 1 3); do
|
|
${ANT_HOME}/bin/ant clean realclean jar
|
|
RETURN="$?"
|
|
if [ "${RETURN}" -eq "0" ]; then
|
|
break
|
|
fi
|
|
done
|
|
# Exit, if we didn't build successfully
|
|
if [ "${RETURN}" -ne "0" ]; then
|
|
echo "Build failed with exit code: ${RETURN}"
|
|
exit ${RETURN}
|
|
fi
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Run static code analysis
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
cd ~/cassandra
|
|
ant check
|
|
- persist_to_workspace:
|
|
root: /home/cassandra
|
|
paths:
|
|
- cassandra
|
|
- .m2
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_utests_oa_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-oa\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-oa $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_dtests_latest:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_dtests_latest)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_latest)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_latest_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_latest_raw /tmp/all_dtest_tests_j11_dtests_latest\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_latest_raw > /tmp/all_dtest_tests_j11_dtests_latest || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_latest > /tmp/split_dtest_tests_j11_dtests_latest.txt\ncat /tmp/split_dtest_tests_j11_dtests_latest.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_latest_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_latest_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_dtests_latest)
|
|
no_output_timeout: 15m
|
|
command: "echo \"cat /tmp/split_dtest_tests_j11_dtests_latest_final.txt\"\ncat /tmp/split_dtest_tests_j11_dtests_latest_final.txt\n\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_latest_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests --log-cli-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_latest.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_dtests_latest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_dtests_latest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_cqlshlib_tests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Run cqlshlib Unit Tests
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra/
|
|
ant jar -Dno-checkstyle=true -Drat.skip=true -Dant.gen-doc.skip=true -Djavadoc.skip=true
|
|
./pylib/cassandra-cqlsh-tests.sh $(pwd)
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/pylib
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j11_dtests:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_without_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_without_vnodes)
|
|
no_output_timeout: 15m
|
|
command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_without_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_without_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_cqlsh_dtests_py38:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j17_without_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_without_vnodes_raw /tmp/all_dtest_tests_j17_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_without_vnodes_raw > /tmp/all_dtest_tests_j17_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_without_vnodes > /tmp/split_dtest_tests_j17_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_without_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j17_without_vnodes)
|
|
no_output_timeout: 15m
|
|
command: |
|
|
echo "cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt"
|
|
cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
|
|
export CQLSH_PYTHON=/usr/bin/python3.8
|
|
fi
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_without_vnodes_final.txt`
|
|
if [ ! -z "$SPLIT_TESTS" ]; then
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j17_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
|
else
|
|
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
|
|
(exit 1)
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j17_without_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j17_without_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_utests_cdc:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Determine unit Tests to Run
|
|
command: |
|
|
# reminder: this code (along with all the steps) is independently executed on every circle container
|
|
# so the goal here is to get the circleci script to return the tests *this* container will run
|
|
# which we do via the `circleci` cli tool.
|
|
|
|
rm -fr ~/cassandra-dtest/upgrade_tests
|
|
echo "***java tests***"
|
|
|
|
# get all of our unit test filenames
|
|
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
|
|
|
# split up the unit tests into groups based on the number of containers we have
|
|
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
|
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
|
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Run Unit Tests (testclasslist-cdc)
|
|
command: |
|
|
set -x
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
|
if [ -z "$test_timeout" ]; then
|
|
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
|
fi
|
|
ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_utests_fqltool_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_dtests_large:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j17_large_without_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j17_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j17_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j17_large_without_vnodes_raw /tmp/all_dtest_tests_j17_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j17_large_without_vnodes_raw > /tmp/all_dtest_tests_j17_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j17_large_without_vnodes > /tmp/split_dtest_tests_j17_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j17_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j17_large_without_vnodes)
|
|
no_output_timeout: 15m
|
|
command: "echo \"cat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j17_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --only-resource-intensive-tests --force-resource-intensive-tests --log-cli-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j17_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j17_large_without_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j17_large_without_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_unit_tests_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_utests_long:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 1
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Run Unit Tests (long-test)
|
|
command: |
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
time mv ~/cassandra /tmp
|
|
cd /tmp/cassandra
|
|
if [ -d ~/dtest_jars ]; then
|
|
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
|
fi
|
|
ant long-test -Dno-build-test=true
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/cassandra/build/test/output/
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/cassandra/build/test/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_utests_oa_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-oa\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-oa $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_dtests_vnode:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Determine Tests to Run (j11_with_vnodes)
|
|
no_output_timeout: 5m
|
|
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
|
- run:
|
|
name: Run dtests (j11_with_vnodes)
|
|
no_output_timeout: 15m
|
|
command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest_j11_with_vnodes
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_j11_with_vnodes_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
j17_jvm_dtests_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_dtests_large_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Clone Cassandra dtest Repository (via git)
|
|
command: |
|
|
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
|
- run:
|
|
name: Configure virtualenv and python Dependencies
|
|
command: |
|
|
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
|
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
|
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
|
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
|
|
pip3 uninstall -y cqlsh
|
|
pip3 freeze
|
|
- run:
|
|
name: Run repeated Python DTests
|
|
no_output_timeout: 15m
|
|
command: |
|
|
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
|
|
echo "Repeated dtest name hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
|
|
echo "Repeated dtest count hasn't been defined, exiting without running any test"
|
|
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
|
|
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
|
|
else
|
|
|
|
# Calculate the number of test iterations to be run by the current parallel runner.
|
|
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
|
|
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
|
|
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
|
|
count=$((count+1))
|
|
fi
|
|
|
|
if (($count <= 0)); then
|
|
echo "No tests to run in this runner"
|
|
else
|
|
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
|
|
|
|
source ~/env3.8/bin/activate
|
|
export PATH=$JAVA_HOME/bin:$PATH
|
|
|
|
java -version
|
|
cd ~/cassandra-dtest
|
|
mkdir -p /tmp/dtest
|
|
|
|
echo "env: $(env)"
|
|
echo "** done env"
|
|
mkdir -p /tmp/results/dtests
|
|
|
|
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
|
|
|
|
stop_on_failure_arg=""
|
|
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
|
|
stop_on_failure_arg="-x"
|
|
fi
|
|
|
|
vnodes_args=""
|
|
if false; then
|
|
vnodes_args="--use-vnodes --num-tokens=16"
|
|
fi
|
|
|
|
upgrade_arg=""
|
|
if false; then
|
|
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
|
|
fi
|
|
|
|
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
|
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
|
|
fi
|
|
fi
|
|
- store_test_results:
|
|
path: /tmp/results
|
|
- store_artifacts:
|
|
path: /tmp/dtest
|
|
destination: dtest
|
|
- store_artifacts:
|
|
path: ~/cassandra-dtest/logs
|
|
destination: dtest_logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j17_utests_long_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-17-openjdk-amd64
|
|
j11_utests_long_repeat:
|
|
docker:
|
|
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
|
resource_class: medium
|
|
working_directory: ~/
|
|
shell: /bin/bash -eo pipefail -l
|
|
parallelism: 4
|
|
steps:
|
|
- attach_workspace:
|
|
at: /home/cassandra
|
|
- run:
|
|
name: Log Environment Information
|
|
command: |
|
|
echo '*** id ***'
|
|
id
|
|
echo '*** cat /proc/cpuinfo ***'
|
|
cat /proc/cpuinfo
|
|
echo '*** free -m ***'
|
|
free -m
|
|
echo '*** df -m ***'
|
|
df -m
|
|
echo '*** ifconfig -a ***'
|
|
ifconfig -a
|
|
echo '*** uname -a ***'
|
|
uname -a
|
|
echo '*** mount ***'
|
|
mount
|
|
echo '*** env ***'
|
|
env
|
|
echo '*** java ***'
|
|
which java
|
|
java -version
|
|
- run:
|
|
name: Repeatedly run new or modifed JUnit tests
|
|
no_output_timeout: 15m
|
|
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-latest\" ]]; then\n testtag=\"latest\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-latest\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=\"build/test/logs\"\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" && -n \"$(ls ${source}/${testtag}*)\" ]]; then\n mv $source/${testtag}*/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
|
- store_test_results:
|
|
path: /tmp/results/repeated_utests/output
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/stdout
|
|
destination: stdout
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/output
|
|
destination: junitxml
|
|
- store_artifacts:
|
|
path: /tmp/results/repeated_utests/logs
|
|
destination: logs
|
|
environment:
|
|
- ANT_HOME: /usr/share/ant
|
|
- LANG: en_US.UTF-8
|
|
- KEEP_TEST_DIR: true
|
|
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
|
- PYTHONIOENCODING: utf-8
|
|
- PYTHONUNBUFFERED: true
|
|
- CASS_DRIVER_NO_EXTENSIONS: true
|
|
- CASS_DRIVER_NO_CYTHON: true
|
|
- CASSANDRA_SKIP_SYNC: true
|
|
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
|
- DTEST_BRANCH: trunk
|
|
- CCM_MAX_HEAP_SIZE: 1024M
|
|
- CCM_HEAP_NEWSIZE: 256M
|
|
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
|
- REPEATED_UTESTS: null
|
|
- REPEATED_UTESTS_COUNT: 500
|
|
- REPEATED_UTESTS_FQLTOOL: null
|
|
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
|
- REPEATED_UTESTS_LONG: null
|
|
- REPEATED_UTESTS_LONG_COUNT: 100
|
|
- REPEATED_UTESTS_STRESS: null
|
|
- REPEATED_UTESTS_STRESS_COUNT: 500
|
|
- REPEATED_SIMULATOR_DTESTS: null
|
|
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_DTESTS: null
|
|
- REPEATED_JVM_DTESTS_COUNT: 500
|
|
- REPEATED_JVM_UPGRADE_DTESTS: null
|
|
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
|
- REPEATED_DTESTS: null
|
|
- REPEATED_DTESTS_COUNT: 500
|
|
- REPEATED_LARGE_DTESTS: null
|
|
- REPEATED_LARGE_DTESTS_COUNT: 100
|
|
- REPEATED_UPGRADE_DTESTS: null
|
|
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
|
- REPEATED_ANT_TEST_TARGET: testsome
|
|
- REPEATED_ANT_TEST_CLASS: null
|
|
- REPEATED_ANT_TEST_METHODS: null
|
|
- REPEATED_ANT_TEST_VNODES: false
|
|
- REPEATED_ANT_TEST_COUNT: 500
|
|
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
|
- CASSANDRA_USE_JDK11: true
|
|
workflows:
|
|
version: 2
|
|
java11_separate_tests:
|
|
jobs:
|
|
- start_j11_build:
|
|
type: approval
|
|
- j11_build:
|
|
requires:
|
|
- start_j11_build
|
|
- start_j11_unit_tests:
|
|
type: approval
|
|
- j11_unit_tests:
|
|
requires:
|
|
- start_j11_unit_tests
|
|
- j11_build
|
|
- start_j11_jvm_dtests:
|
|
type: approval
|
|
- j11_jvm_dtests:
|
|
requires:
|
|
- start_j11_jvm_dtests
|
|
- j11_build
|
|
- start_j11_jvm_dtests_latest_vnode:
|
|
type: approval
|
|
- j11_jvm_dtests_latest_vnode:
|
|
requires:
|
|
- start_j11_jvm_dtests_latest_vnode
|
|
- j11_build
|
|
- start_j17_jvm_dtests:
|
|
type: approval
|
|
- j17_jvm_dtests:
|
|
requires:
|
|
- start_j17_jvm_dtests
|
|
- j11_build
|
|
- start_j17_jvm_dtests_latest_vnode:
|
|
type: approval
|
|
- j17_jvm_dtests_latest_vnode:
|
|
requires:
|
|
- start_j17_jvm_dtests_latest_vnode
|
|
- j11_build
|
|
- start_j11_simulator_dtests:
|
|
type: approval
|
|
- j11_simulator_dtests:
|
|
requires:
|
|
- start_j11_simulator_dtests
|
|
- j11_build
|
|
- start_j11_cqlshlib_tests:
|
|
type: approval
|
|
- j11_cqlshlib_tests:
|
|
requires:
|
|
- start_j11_cqlshlib_tests
|
|
- j11_build
|
|
- start_j11_cqlshlib_cython_tests:
|
|
type: approval
|
|
- j11_cqlshlib_cython_tests:
|
|
requires:
|
|
- start_j11_cqlshlib_cython_tests
|
|
- j11_build
|
|
- start_j17_cqlshlib_tests:
|
|
type: approval
|
|
- j17_cqlshlib_tests:
|
|
requires:
|
|
- start_j17_cqlshlib_tests
|
|
- j11_build
|
|
- start_j17_cqlshlib_cython_tests:
|
|
type: approval
|
|
- j17_cqlshlib_cython_tests:
|
|
requires:
|
|
- start_j17_cqlshlib_cython_tests
|
|
- j11_build
|
|
- start_j17_unit_tests:
|
|
type: approval
|
|
- j17_unit_tests:
|
|
requires:
|
|
- start_j17_unit_tests
|
|
- j11_build
|
|
- start_j11_utests_oa:
|
|
type: approval
|
|
- j11_utests_oa:
|
|
requires:
|
|
- start_j11_utests_oa
|
|
- j11_build
|
|
- start_j17_utests_oa:
|
|
type: approval
|
|
- j17_utests_oa:
|
|
requires:
|
|
- start_j17_utests_oa
|
|
- j11_build
|
|
- start_j11_utests_long:
|
|
type: approval
|
|
- j11_utests_long:
|
|
requires:
|
|
- start_j11_utests_long
|
|
- j11_build
|
|
- start_j17_utests_long:
|
|
type: approval
|
|
- j17_utests_long:
|
|
requires:
|
|
- start_j17_utests_long
|
|
- j11_build
|
|
- start_j11_utests_cdc:
|
|
type: approval
|
|
- j11_utests_cdc:
|
|
requires:
|
|
- start_j11_utests_cdc
|
|
- j11_build
|
|
- start_j17_utests_cdc:
|
|
type: approval
|
|
- j17_utests_cdc:
|
|
requires:
|
|
- start_j17_utests_cdc
|
|
- j11_build
|
|
- start_j11_utests_compression:
|
|
type: approval
|
|
- j11_utests_compression:
|
|
requires:
|
|
- start_j11_utests_compression
|
|
- j11_build
|
|
- start_j17_utests_compression:
|
|
type: approval
|
|
- j17_utests_compression:
|
|
requires:
|
|
- start_j17_utests_compression
|
|
- j11_build
|
|
- start_j11_utests_latest:
|
|
type: approval
|
|
- j11_utests_latest:
|
|
requires:
|
|
- start_j11_utests_latest
|
|
- j11_build
|
|
- start_j17_utests_latest:
|
|
type: approval
|
|
- j17_utests_latest:
|
|
requires:
|
|
- start_j17_utests_latest
|
|
- j11_build
|
|
- start_j11_utests_stress:
|
|
type: approval
|
|
- j11_utests_stress:
|
|
requires:
|
|
- start_j11_utests_stress
|
|
- j11_build
|
|
- start_j17_utests_stress:
|
|
type: approval
|
|
- j17_utests_stress:
|
|
requires:
|
|
- start_j17_utests_stress
|
|
- j11_build
|
|
- start_j11_utests_fqltool:
|
|
type: approval
|
|
- j11_utests_fqltool:
|
|
requires:
|
|
- start_j11_utests_fqltool
|
|
- j11_build
|
|
- start_j17_utests_fqltool:
|
|
type: approval
|
|
- j17_utests_fqltool:
|
|
requires:
|
|
- start_j17_utests_fqltool
|
|
- j11_build
|
|
- start_j11_utests_system_keyspace_directory:
|
|
type: approval
|
|
- j11_utests_system_keyspace_directory:
|
|
requires:
|
|
- start_j11_utests_system_keyspace_directory
|
|
- j11_build
|
|
- start_j17_utests_system_keyspace_directory:
|
|
type: approval
|
|
- j17_utests_system_keyspace_directory:
|
|
requires:
|
|
- start_j17_utests_system_keyspace_directory
|
|
- j11_build
|
|
- start_j11_dtest_jars_build:
|
|
type: approval
|
|
- j11_dtest_jars_build:
|
|
requires:
|
|
- j11_build
|
|
- start_j11_dtest_jars_build
|
|
- start_jvm_upgrade_dtests:
|
|
type: approval
|
|
- j11_jvm_upgrade_dtests:
|
|
requires:
|
|
- start_jvm_upgrade_dtests
|
|
- j11_dtest_jars_build
|
|
- start_j11_dtests:
|
|
type: approval
|
|
- j11_dtests:
|
|
requires:
|
|
- start_j11_dtests
|
|
- j11_build
|
|
- start_j11_dtests_vnode:
|
|
type: approval
|
|
- j11_dtests_vnode:
|
|
requires:
|
|
- start_j11_dtests_vnode
|
|
- j11_build
|
|
- start_j11_dtests_latest:
|
|
type: approval
|
|
- j11_dtests_latest:
|
|
requires:
|
|
- start_j11_dtests_latest
|
|
- j11_build
|
|
- start_j17_dtests:
|
|
type: approval
|
|
- j17_dtests:
|
|
requires:
|
|
- start_j17_dtests
|
|
- j11_build
|
|
- start_j17_dtests_vnode:
|
|
type: approval
|
|
- j17_dtests_vnode:
|
|
requires:
|
|
- start_j17_dtests_vnode
|
|
- j11_build
|
|
- start_j17_dtests_latest:
|
|
type: approval
|
|
- j17_dtests_latest:
|
|
requires:
|
|
- start_j17_dtests_latest
|
|
- j11_build
|
|
- start_j11_dtests_large:
|
|
type: approval
|
|
- j11_dtests_large:
|
|
requires:
|
|
- start_j11_dtests_large
|
|
- j11_build
|
|
- start_j11_dtests_large_vnode:
|
|
type: approval
|
|
- j11_dtests_large_vnode:
|
|
requires:
|
|
- start_j11_dtests_large_vnode
|
|
- j11_build
|
|
- start_j17_dtests_large:
|
|
type: approval
|
|
- j17_dtests_large:
|
|
requires:
|
|
- start_j17_dtests_large
|
|
- j11_build
|
|
- start_j17_dtests_large_vnode:
|
|
type: approval
|
|
- j17_dtests_large_vnode:
|
|
requires:
|
|
- start_j17_dtests_large_vnode
|
|
- j11_build
|
|
- start_j11_cqlsh_tests:
|
|
type: approval
|
|
- j11_cqlsh_dtests_py38:
|
|
requires:
|
|
- start_j11_cqlsh_tests
|
|
- j11_build
|
|
- j11_cqlsh_dtests_py311:
|
|
requires:
|
|
- start_j11_cqlsh_tests
|
|
- j11_build
|
|
- j11_cqlsh_dtests_py38_vnode:
|
|
requires:
|
|
- start_j11_cqlsh_tests
|
|
- j11_build
|
|
- j11_cqlsh_dtests_py311_vnode:
|
|
requires:
|
|
- start_j11_cqlsh_tests
|
|
- j11_build
|
|
- start_j11_cqlsh_tests_latest:
|
|
type: approval
|
|
- j11_cqlsh_dtests_py38_latest:
|
|
requires:
|
|
- start_j11_cqlsh_tests_latest
|
|
- j11_build
|
|
- j11_cqlsh_dtests_py311_latest:
|
|
requires:
|
|
- start_j11_cqlsh_tests_latest
|
|
- j11_build
|
|
- start_j17_cqlsh_tests:
|
|
type: approval
|
|
- j17_cqlsh_dtests_py38:
|
|
requires:
|
|
- start_j17_cqlsh_tests
|
|
- j11_build
|
|
- j17_cqlsh_dtests_py311:
|
|
requires:
|
|
- start_j17_cqlsh_tests
|
|
- j11_build
|
|
- j17_cqlsh_dtests_py38_vnode:
|
|
requires:
|
|
- start_j17_cqlsh_tests
|
|
- j11_build
|
|
- j17_cqlsh_dtests_py311_vnode:
|
|
requires:
|
|
- start_j17_cqlsh_tests
|
|
- j11_build
|
|
- start_j17_cqlsh_tests_latest:
|
|
type: approval
|
|
- j17_cqlsh_dtests_py38_latest:
|
|
requires:
|
|
- start_j17_cqlsh_tests_latest
|
|
- j11_build
|
|
- j17_cqlsh_dtests_py311_latest:
|
|
requires:
|
|
- start_j17_cqlsh_tests_latest
|
|
- j11_build
|
|
- start_j11_upgrade_dtests:
|
|
type: approval
|
|
- j11_upgrade_dtests:
|
|
requires:
|
|
- start_j11_upgrade_dtests
|
|
- j11_build
|
|
java11_pre-commit_tests:
|
|
jobs:
|
|
- start_pre-commit_tests:
|
|
type: approval
|
|
- j11_build:
|
|
requires:
|
|
- start_pre-commit_tests
|
|
- j11_unit_tests:
|
|
requires:
|
|
- j11_build
|
|
- j11_utests_oa:
|
|
requires:
|
|
- j11_build
|
|
- j11_utests_latest:
|
|
requires:
|
|
- j11_build
|
|
- j11_simulator_dtests:
|
|
requires:
|
|
- j11_build
|
|
- j11_jvm_dtests:
|
|
requires:
|
|
- j11_build
|
|
- j11_jvm_dtests_latest_vnode:
|
|
requires:
|
|
- j11_build
|
|
- j17_jvm_dtests:
|
|
requires:
|
|
- j11_build
|
|
- j17_jvm_dtests_latest_vnode:
|
|
requires:
|
|
- j11_build
|
|
- j11_cqlshlib_tests:
|
|
requires:
|
|
- j11_build
|
|
- j11_cqlshlib_cython_tests:
|
|
requires:
|
|
- j11_build
|
|
- j17_cqlshlib_tests:
|
|
requires:
|
|
- j11_build
|
|
- j17_cqlshlib_cython_tests:
|
|
requires:
|
|
- j11_build
|
|
- j17_unit_tests:
|
|
requires:
|
|
- j11_build
|
|
- j17_utests_oa:
|
|
requires:
|
|
- j11_build
|
|
- j17_utests_latest:
|
|
requires:
|
|
- j11_build
|
|
- start_utests_long:
|
|
type: approval
|
|
- j11_utests_long:
|
|
requires:
|
|
- start_utests_long
|
|
- j11_build
|
|
- j17_utests_long:
|
|
requires:
|
|
- start_utests_long
|
|
- j11_build
|
|
- start_utests_cdc:
|
|
type: approval
|
|
- j11_utests_cdc:
|
|
requires:
|
|
- start_utests_cdc
|
|
- j11_build
|
|
- j17_utests_cdc:
|
|
requires:
|
|
- start_utests_cdc
|
|
- j11_build
|
|
- start_utests_compression:
|
|
type: approval
|
|
- j11_utests_compression:
|
|
requires:
|
|
- start_utests_compression
|
|
- j11_build
|
|
- j17_utests_compression:
|
|
requires:
|
|
- start_utests_compression
|
|
- j11_build
|
|
- start_utests_stress:
|
|
type: approval
|
|
- j11_utests_stress:
|
|
requires:
|
|
- start_utests_stress
|
|
- j11_build
|
|
- j17_utests_stress:
|
|
requires:
|
|
- start_utests_stress
|
|
- j11_build
|
|
- start_utests_fqltool:
|
|
type: approval
|
|
- j11_utests_fqltool:
|
|
requires:
|
|
- start_utests_fqltool
|
|
- j11_build
|
|
- j17_utests_fqltool:
|
|
requires:
|
|
- start_utests_fqltool
|
|
- j11_build
|
|
- start_utests_system_keyspace_directory:
|
|
type: approval
|
|
- j11_utests_system_keyspace_directory:
|
|
requires:
|
|
- j11_build
|
|
- j17_utests_system_keyspace_directory:
|
|
requires:
|
|
- start_utests_system_keyspace_directory
|
|
- j11_build
|
|
- start_jvm_upgrade_dtests:
|
|
type: approval
|
|
- j11_dtest_jars_build:
|
|
requires:
|
|
- j11_build
|
|
- start_jvm_upgrade_dtests
|
|
- j11_jvm_upgrade_dtests:
|
|
requires:
|
|
- j11_dtest_jars_build
|
|
- j11_dtests:
|
|
requires:
|
|
- j11_build
|
|
- j11_dtests_vnode:
|
|
requires:
|
|
- j11_build
|
|
- j11_dtests_latest:
|
|
requires:
|
|
- j11_build
|
|
- j17_dtests:
|
|
requires:
|
|
- j11_build
|
|
- j17_dtests_vnode:
|
|
requires:
|
|
- j11_build
|
|
- j17_dtests_latest:
|
|
requires:
|
|
- j11_build
|
|
- start_j11_dtests_large:
|
|
type: approval
|
|
- j11_dtests_large:
|
|
requires:
|
|
- start_j11_dtests_large
|
|
- j11_build
|
|
- j11_dtests_large_vnode:
|
|
requires:
|
|
- start_j11_dtests_large
|
|
- j11_build
|
|
- start_j17_dtests_large:
|
|
type: approval
|
|
- j17_dtests_large:
|
|
requires:
|
|
- start_j17_dtests_large
|
|
- j11_build
|
|
- j17_dtests_large_vnode:
|
|
requires:
|
|
- start_j17_dtests_large
|
|
- j11_build
|
|
- j11_cqlsh_dtests_py38:
|
|
requires:
|
|
- j11_build
|
|
- j11_cqlsh_dtests_py311:
|
|
requires:
|
|
- j11_build
|
|
- j11_cqlsh_dtests_py38_vnode:
|
|
requires:
|
|
- j11_build
|
|
- j11_cqlsh_dtests_py311_vnode:
|
|
requires:
|
|
- j11_build
|
|
- start_j11_cqlsh_dtests_latest:
|
|
type: approval
|
|
- j11_cqlsh_dtests_py38_latest:
|
|
requires:
|
|
- start_j11_cqlsh_dtests_latest
|
|
- j11_build
|
|
- j11_cqlsh_dtests_py311_latest:
|
|
requires:
|
|
- start_j11_cqlsh_dtests_latest
|
|
- j11_build
|
|
- j17_cqlsh_dtests_py38:
|
|
requires:
|
|
- j11_build
|
|
- j17_cqlsh_dtests_py311:
|
|
requires:
|
|
- j11_build
|
|
- j17_cqlsh_dtests_py38_vnode:
|
|
requires:
|
|
- j11_build
|
|
- j17_cqlsh_dtests_py311_vnode:
|
|
requires:
|
|
- j11_build
|
|
- start_j17_cqlsh-dtests-latest:
|
|
type: approval
|
|
- j17_cqlsh_dtests_py38_latest:
|
|
requires:
|
|
- start_j17_cqlsh-dtests-latest
|
|
- j11_build
|
|
- j17_cqlsh_dtests_py311_latest:
|
|
requires:
|
|
- start_j17_cqlsh-dtests-latest
|
|
- j11_build
|
|
- start_j11_upgrade_dtests:
|
|
type: approval
|
|
- j11_upgrade_dtests:
|
|
requires:
|
|
- j11_build
|
|
- start_j11_upgrade_dtests
|
|
java17_separate_tests:
|
|
jobs:
|
|
- start_j17_build:
|
|
type: approval
|
|
- j17_build:
|
|
requires:
|
|
- start_j17_build
|
|
- start_j17_unit_tests:
|
|
type: approval
|
|
- j17_unit_tests:
|
|
requires:
|
|
- start_j17_unit_tests
|
|
- j17_build
|
|
- start_j17_jvm_dtests:
|
|
type: approval
|
|
- j17_jvm_dtests:
|
|
requires:
|
|
- start_j17_jvm_dtests
|
|
- j17_build
|
|
- start_j17_jvm_dtests_latest_vnode:
|
|
type: approval
|
|
- j17_jvm_dtests_latest_vnode:
|
|
requires:
|
|
- start_j17_jvm_dtests_latest_vnode
|
|
- j17_build
|
|
- start_j17_cqlshlib_tests:
|
|
type: approval
|
|
- j17_cqlshlib_tests:
|
|
requires:
|
|
- start_j17_cqlshlib_tests
|
|
- j17_build
|
|
- start_j17_cqlshlib_cython_tests:
|
|
type: approval
|
|
- j17_cqlshlib_cython_tests:
|
|
requires:
|
|
- start_j17_cqlshlib_cython_tests
|
|
- j17_build
|
|
- start_j17_dtests:
|
|
type: approval
|
|
- j17_dtests:
|
|
requires:
|
|
- start_j17_dtests
|
|
- j17_build
|
|
- start_j17_dtests_vnode:
|
|
type: approval
|
|
- j17_dtests_vnode:
|
|
requires:
|
|
- start_j17_dtests_vnode
|
|
- j17_build
|
|
- start_j17_dtests_latest:
|
|
type: approval
|
|
- j17_dtests_latest:
|
|
requires:
|
|
- start_j17_dtests_latest
|
|
- j17_build
|
|
- start_j17_dtests_large:
|
|
type: approval
|
|
- j17_dtests_large:
|
|
requires:
|
|
- start_j17_dtests_large
|
|
- j17_build
|
|
- start_j17_dtests_large_vnode:
|
|
type: approval
|
|
- j17_dtests_large_vnode:
|
|
requires:
|
|
- start_j17_dtests_large_vnode
|
|
- j17_build
|
|
- start_j17_cqlsh_tests:
|
|
type: approval
|
|
- j17_cqlsh_dtests_py38:
|
|
requires:
|
|
- start_j17_cqlsh_tests
|
|
- j17_build
|
|
- j17_cqlsh_dtests_py311:
|
|
requires:
|
|
- start_j17_cqlsh_tests
|
|
- j17_build
|
|
- j17_cqlsh_dtests_py38_vnode:
|
|
requires:
|
|
- start_j17_cqlsh_tests
|
|
- j17_build
|
|
- j17_cqlsh_dtests_py311_vnode:
|
|
requires:
|
|
- start_j17_cqlsh_tests
|
|
- j17_build
|
|
- start_j17_cqlsh-dtests-latest:
|
|
type: approval
|
|
- j17_cqlsh_dtests_py38_latest:
|
|
requires:
|
|
- start_j17_cqlsh-dtests-latest
|
|
- j17_build
|
|
- j17_cqlsh_dtests_py311_latest:
|
|
requires:
|
|
- start_j17_cqlsh-dtests-latest
|
|
- j17_build
|
|
- start_j17_utests_oa:
|
|
type: approval
|
|
- j17_utests_oa:
|
|
requires:
|
|
- start_j17_utests_oa
|
|
- j17_build
|
|
- start_j17_utests_long:
|
|
type: approval
|
|
- j17_utests_long:
|
|
requires:
|
|
- start_j17_utests_long
|
|
- j17_build
|
|
- start_j17_utests_cdc:
|
|
type: approval
|
|
- j17_utests_cdc:
|
|
requires:
|
|
- start_j17_utests_cdc
|
|
- j17_build
|
|
- start_j17_utests_compression:
|
|
type: approval
|
|
- j17_utests_compression:
|
|
requires:
|
|
- start_j17_utests_compression
|
|
- j17_build
|
|
- start_j17_utests_latest:
|
|
type: approval
|
|
- j17_utests_latest:
|
|
requires:
|
|
- start_j17_utests_latest
|
|
- j17_build
|
|
- start_j17_utests_stress:
|
|
type: approval
|
|
- j17_utests_stress:
|
|
requires:
|
|
- start_j17_utests_stress
|
|
- j17_build
|
|
- start_j17_utests_fqltool:
|
|
type: approval
|
|
- j17_utests_fqltool:
|
|
requires:
|
|
- start_j17_utests_fqltool
|
|
- j17_build
|
|
- start_j17_utests_system_keyspace_directory:
|
|
type: approval
|
|
- j17_utests_system_keyspace_directory:
|
|
requires:
|
|
- start_j17_utests_system_keyspace_directory
|
|
- j17_build
|
|
java17_pre-commit_tests:
|
|
jobs:
|
|
- start_pre-commit_tests:
|
|
type: approval
|
|
- j17_build:
|
|
requires:
|
|
- start_pre-commit_tests
|
|
- j17_unit_tests:
|
|
requires:
|
|
- j17_build
|
|
- j17_utests_oa:
|
|
requires:
|
|
- j17_build
|
|
- j17_utests_latest:
|
|
requires:
|
|
- j17_build
|
|
- j17_jvm_dtests:
|
|
requires:
|
|
- j17_build
|
|
- j17_jvm_dtests_latest_vnode:
|
|
requires:
|
|
- j17_build
|
|
- j17_cqlshlib_tests:
|
|
requires:
|
|
- j17_build
|
|
- j17_cqlshlib_cython_tests:
|
|
requires:
|
|
- j17_build
|
|
- j17_dtests:
|
|
requires:
|
|
- j17_build
|
|
- j17_dtests_vnode:
|
|
requires:
|
|
- j17_build
|
|
- j17_dtests_latest:
|
|
requires:
|
|
- j17_build
|
|
- start_j17_dtests_large:
|
|
type: approval
|
|
- j17_dtests_large:
|
|
requires:
|
|
- start_j17_dtests_large
|
|
- j17_build
|
|
- j17_dtests_large_vnode:
|
|
requires:
|
|
- start_j17_dtests_large
|
|
- j17_build
|
|
- j17_cqlsh_dtests_py38:
|
|
requires:
|
|
- j17_build
|
|
- j17_cqlsh_dtests_py311:
|
|
requires:
|
|
- j17_build
|
|
- j17_cqlsh_dtests_py38_vnode:
|
|
requires:
|
|
- j17_build
|
|
- j17_cqlsh_dtests_py311_vnode:
|
|
requires:
|
|
- j17_build
|
|
- start_j17_cqlsh-dtests-latest:
|
|
type: approval
|
|
- j17_cqlsh_dtests_py38_latest:
|
|
requires:
|
|
- start_j17_cqlsh-dtests-latest
|
|
- j17_build
|
|
- j17_cqlsh_dtests_py311_latest:
|
|
requires:
|
|
- start_j17_cqlsh-dtests-latest
|
|
- j17_build
|
|
- start_utests_long:
|
|
type: approval
|
|
- j17_utests_long:
|
|
requires:
|
|
- start_utests_long
|
|
- j17_build
|
|
- start_utests_cdc:
|
|
type: approval
|
|
- j17_utests_cdc:
|
|
requires:
|
|
- start_utests_cdc
|
|
- j17_build
|
|
- start_utests_compression:
|
|
type: approval
|
|
- j17_utests_compression:
|
|
requires:
|
|
- start_utests_compression
|
|
- j17_build
|
|
- start_utests_stress:
|
|
type: approval
|
|
- j17_utests_stress:
|
|
requires:
|
|
- start_utests_stress
|
|
- j17_build
|
|
- start_utests_fqltool:
|
|
type: approval
|
|
- j17_utests_fqltool:
|
|
requires:
|
|
- start_utests_fqltool
|
|
- j17_build
|
|
- start_utests_system_keyspace_directory:
|
|
type: approval
|
|
- j17_utests_system_keyspace_directory:
|
|
requires:
|
|
- start_utests_system_keyspace_directory
|
|
- j17_build
|