Merge client files

docker/test/stateful_with_coverage/Dockerfile

@@ -1,11 +1,14 @@
 # docker build -t yandex/clickhouse-stateful-test .
 FROM yandex/clickhouse-stateless-test
 
+RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/sources.list
+
 RUN apt-get update -y \
     && env DEBIAN_FRONTEND=noninteractive \
         apt-get install --yes --no-install-recommends \
         python-requests \
-        llvm-8
+        llvm-8 \
+        llvm-9
 
 COPY s3downloader /s3downloader
 COPY run.sh /run.sh

docker/test/stateful_with_coverage/run.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -x
-
 kill_clickhouse () {
     while kill -0 `pgrep -u clickhouse`;
     do
@@ -15,6 +13,23 @@ start_clickhouse () {
     LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
 }
 
+wait_llvm_profdata () {
+    while kill -0 `pgrep llvm-profdata-9`;
+    do
+        echo "Waiting for profdata " `pgrep llvm-profdata-9` "still alive"
+        sleep 3
+    done
+}
+
+merge_client_files_in_background () {
+    client_files=`ls /client_*profraw 2>/dev/null`
+    if [ ! -z "$client_files" ]
+    then
+        llvm-profdata-9 merge -sparse $client_files -o merged_client_`date +%s`.profraw
+        rm $client_files
+    fi
+}
+
 chmod 777 /
 
 dpkg -i package_folder/clickhouse-common-static_*.deb; \
@@ -37,6 +52,7 @@ ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/con
     ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/; \
     ln -s /usr/lib/llvm-8/bin/llvm-symbolizer /usr/bin/llvm-symbolizer
 
+
 service zookeeper start
 
 sleep 5
@@ -49,22 +65,34 @@ sleep 5
 
 chmod 777 -R /var/lib/clickhouse
 
-LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "SHOW DATABASES"
-LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "CREATE DATABASE datasets"
-LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "CREATE DATABASE test"
+while /bin/true; do
+    merge_client_files_in_background
+    sleep 2
+done &
+
+LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW DATABASES"
+LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "CREATE DATABASE datasets"
+LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "CREATE DATABASE test"
 
 kill_clickhouse
 start_clickhouse
 
 sleep 10
 
-LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "SHOW TABLES FROM datasets"
-LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "SHOW TABLES FROM test"
-LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
-LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
-LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "SHOW TABLES FROM test"
-LLVM_PROFILE_FILE='client.profraw' clickhouse-test --shard --zookeeper --no-stateless $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
+LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW TABLES FROM datasets"
+LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW TABLES FROM test"
+LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
+LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
+LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-client --query "SHOW TABLES FROM test"
+LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-test --shard --zookeeper --no-stateless $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
 
 kill_clickhouse
 
+wait_llvm_profdata
+
+sleep 3
+
+wait_llvm_profdata # 100% merged all parts
+
+
 cp /*.profraw /profraw ||:

docker/test/stateless_with_coverage/Dockerfile

@@ -1,6 +1,8 @@
 # docker build -t yandex/clickhouse-stateless-with-coverage-test .
 FROM yandex/clickhouse-deb-builder
 
+RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/sources.list
+
 RUN apt-get update -y \
     && env DEBIAN_FRONTEND=noninteractive \
         apt-get install --yes --no-install-recommends \
@@ -23,7 +25,8 @@ RUN apt-get update -y \
             moreutils \
             brotli \
             gdb \
-            lsof
+            lsof \
+            llvm-9
 
 
 ENV TZ=Europe/Moscow

docker/test/stateless_with_coverage/run.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -x
-
 kill_clickhouse () {
     while kill -0 `pgrep -u clickhouse`;
     do
@@ -11,10 +9,27 @@ kill_clickhouse () {
     done
 }
 
+wait_llvm_profdata () {
+    while kill -0 `pgrep llvm-profdata-9`;
+    do
+        echo "Waiting for profdata " `pgrep llvm-profdata-9` "still alive"
+        sleep 3
+    done
+}
+
 start_clickhouse () {
     LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
 }
 
+merge_client_files_in_background () {
+    client_files=`ls /client_*profraw 2>/dev/null`
+    if [ ! -z "$client_files" ]
+    then
+        llvm-profdata-9 merge -sparse $client_files -o merged_client_`date +%s`.profraw
+        rm $client_files
+    fi
+}
+
 chmod 777 /
 
 dpkg -i package_folder/clickhouse-common-static_*.deb; \
@@ -46,8 +61,19 @@ start_clickhouse
 
 sleep 10
 
-LLVM_PROFILE_FILE='client.profraw' clickhouse-test --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
+while /bin/true; do
+    merge_client_files_in_background
+    sleep 2
+done &
+
+LLVM_PROFILE_FILE='client_%h_%p_%m.profraw' clickhouse-test --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
 
 kill_clickhouse
 
+wait_llvm_profdata
+
+sleep 3
+
+wait_llvm_profdata # 100% merged all parts
+
 cp /*.profraw /profraw ||: