mirror of https://github.com/ByConity/ByConity
performance comparison
This commit is contained in:
Alexander Kuzmenkov
parent
e72a23065e
commit
5aadd628f1
|
|
@ -93,53 +93,52 @@ function run_tests
|
|||
# Just check that the script runs at all
|
||||
"$script_dir/perf.py" --help > /dev/null
|
||||
|
||||
# When testing commits from master, use the older test files. This allows the
|
||||
# tests to pass even when we add new functions and tests for them, that are
|
||||
# not supported in the old revision.
|
||||
# When testing a PR, use the test files from the PR so that we can test their
|
||||
# changes.
|
||||
test_prefix=$([ "$PR_TO_TEST" == "0" ] && echo left || echo right)/performance
|
||||
# Find the directory with test files.
|
||||
if [ -v CHPC_TEST_PATH ]
|
||||
then
|
||||
# Use the explicitly set path to directory with test files.
|
||||
test_prefix="$CHPC_TEST_PATH"
|
||||
elif [ "$PR_TO_TEST" = "0" ]
|
||||
then
|
||||
# When testing commits from master, use the older test files. This
|
||||
# allows the tests to pass even when we add new functions and tests for
|
||||
# them, that are not supported in the old revision.
|
||||
test_prefix=left/performance
|
||||
elif [ "$PR_TO_TEST" != "" ] && [ "$PR_TO_TEST" != "0" ]
|
||||
then
|
||||
# For PRs, use newer test files so we can test these changes.
|
||||
test_prefix=right/performance
|
||||
|
||||
# If some tests were changed in the PR, we may want to run only these
|
||||
# ones. The list of changed tests in changed-test.txt is prepared in
|
||||
# entrypoint.sh from git diffs, because it has the cloned repo. Used
|
||||
# to use rsync for that but it was really ugly and not always correct
|
||||
# (e.g. when the reference SHA is really old and has some other
|
||||
# differences to the tested SHA, besides the one introduced by the PR).
|
||||
changed_test_files=$(sed "s/tests\/performance/${test_prefix//\//\\/}/" changed-tests.txt)
|
||||
fi
|
||||
|
||||
# Determine which tests to run.
|
||||
if [ -v CHPC_TEST_GREP ]
|
||||
then
|
||||
# Run only explicitly specified tests, if any.
|
||||
test_files=$(ls "$test_prefix" | grep "$CHPC_TEST_GREP" | xargs -I{} -n1 readlink -f "$test_prefix/{}")
|
||||
elif [ "$changed_test_files" != "" ]
|
||||
then
|
||||
# Use test files that changed in the PR.
|
||||
test_files="$changed_test_files"
|
||||
else
|
||||
# The default -- run all tests found in the test dir.
|
||||
test_files=$(ls "$test_prefix"/*.xml)
|
||||
fi
|
||||
|
||||
# Delete old report files.
|
||||
for x in {test-times,skipped-tests,wall-clock-times,report-thresholds,client-times}.tsv
|
||||
do
|
||||
rm -v "$x" ||:
|
||||
touch "$x"
|
||||
done
|
||||
|
||||
|
||||
# FIXME a quick crutch to bring the run time down for the unstable tests --
|
||||
# if some performance tests xmls were changed in a PR, run only these ones.
|
||||
if [ "$PR_TO_TEST" != "0" ]
|
||||
then
|
||||
# changed-test.txt prepared in entrypoint.sh from git diffs, because it
|
||||
# has the cloned repo. Used to use rsync for that but it was really ugly
|
||||
# and not always correct (e.g. when the reference SHA is really old and
|
||||
# has some other differences to the tested SHA, besides the one introduced
|
||||
# by the PR).
|
||||
test_files_override=$(sed "s/tests\/performance/${test_prefix//\//\\/}/" changed-tests.txt)
|
||||
if [ "$test_files_override" != "" ]
|
||||
then
|
||||
test_files=$test_files_override
|
||||
fi
|
||||
fi
|
||||
|
||||
# Run only explicitly specified tests, if any
|
||||
if [ -v CHPC_TEST_GREP ]
|
||||
then
|
||||
test_files=$(ls "$test_prefix" | grep "$CHPC_TEST_GREP" | xargs -I{} -n1 readlink -f "$test_prefix/{}")
|
||||
fi
|
||||
|
||||
if [ "$test_files" == "" ]
|
||||
then
|
||||
# FIXME remove some broken long tests
|
||||
for test_name in {IPv4,IPv6,modulo,parse_engine_file,number_formatting_formats,select_format,arithmetic,cryptographic_hashes,logical_functions_{medium,small}}
|
||||
do
|
||||
printf "%s\tMarked as broken (see compare.sh)\n" "$test_name">> skipped-tests.tsv
|
||||
rm "$test_prefix/$test_name.xml" ||:
|
||||
done
|
||||
test_files=$(ls "$test_prefix"/*.xml)
|
||||
fi
|
||||
|
||||
# Run the tests.
|
||||
test_name="<none>"
|
||||
for test in $test_files
|
||||
|
|
@ -554,11 +553,10 @@ case "$stage" in
|
|||
echo Servers stopped.
|
||||
;&
|
||||
"analyze_queries")
|
||||
# FIXME grep for set_index fails -- argument list too long.
|
||||
time analyze_queries ||:
|
||||
;&
|
||||
"report")
|
||||
time report
|
||||
time report ||:
|
||||
|
||||
time "$script_dir/report.py" --report=all-queries > all-queries.html 2> >(tee -a report/errors.log 1>&2) ||:
|
||||
time "$script_dir/report.py" > report.html
|
||||
|
|
|
|||
|
|
@ -54,7 +54,27 @@ docker/test/performance-comparison/perf.py --host=localhost --port=9000 --runs=1
|
|||
```
|
||||
|
||||
#### Run all tests on some custom configuration
|
||||
Technically possible, but inconvenient -- requires some scripting and setting up the directory structure. See `manual-run.sh` for inspiration. `compare.sh` has some stages which you can skip, specified by the `stage` environment variable.
|
||||
Start two servers manually on ports `9001` (old) and `9002` (new). Change to a
|
||||
new directory to be used as workspace for tests, and try something like this:
|
||||
```
|
||||
$ PATH=$PATH:~/ch4/build-gcc9-rel/programs \
|
||||
CHPC_TEST_PATH=~/ch3/ch/tests/performance \
|
||||
CHPC_TEST_GREP=visit_param \
|
||||
stage=run_tests \
|
||||
~/ch3/ch/docker/test/performance-comparison/compare.sh
|
||||
```
|
||||
* `PATH` must contain `clickhouse-local` and `clickhouse-client`.
|
||||
* `CHPC_TEST_PATH` -- path to performance test cases, e.g. `tests/performance`.
|
||||
* `CHPC_TEST_GREP` -- a filter for which tests to run, as a grep pattern.
|
||||
* `stage` -- from which execution stage to start. To run the tests, use
|
||||
`run_tests` stage.
|
||||
|
||||
The tests will run, and the `report.html` will be generated in the current
|
||||
directory.
|
||||
|
||||
More complex setup is possible, but inconvenient and requires some scripting.
|
||||
See `manual-run.sh` for inspiration.
|
||||
|
||||
|
||||
#### Statistical considerations
|
||||
Generating randomization distribution for medians is tricky. Suppose we have N
|
||||
|
|
|
|||
|
|
@ -143,13 +143,42 @@ def printSimpleTable(caption, columns, rows):
|
|||
print(tableRow(row))
|
||||
print(tableEnd())
|
||||
|
||||
def print_tested_commits():
|
||||
global report_errors
|
||||
try:
|
||||
printSimpleTable('Tested commits', ['Old', 'New'],
|
||||
[['<pre>{}</pre>'.format(x) for x in
|
||||
[open('left-commit.txt').read(),
|
||||
open('right-commit.txt').read()]]])
|
||||
except:
|
||||
# Don't fail if no commit info -- maybe it's a manual run.
|
||||
report_errors.append(
|
||||
traceback.format_exception_only(
|
||||
*sys.exc_info()[:2])[-1])
|
||||
pass
|
||||
|
||||
def print_report_errors():
|
||||
global report_errors
|
||||
# Add the errors reported by various steps of comparison script
|
||||
try:
|
||||
report_errors += [l.strip() for l in open('report/errors.log')]
|
||||
except:
|
||||
report_errors.append(
|
||||
traceback.format_exception_only(
|
||||
*sys.exc_info()[:2])[-1])
|
||||
pass
|
||||
|
||||
if len(report_errors):
|
||||
print(tableStart('Errors while building the report'))
|
||||
print(tableHeader(['Error']))
|
||||
for x in report_errors:
|
||||
print(tableRow([x]))
|
||||
print(tableEnd())
|
||||
|
||||
if args.report == 'main':
|
||||
print(header_template.format())
|
||||
|
||||
printSimpleTable('Tested commits', ['Old', 'New'],
|
||||
[['<pre>{}</pre>'.format(x) for x in
|
||||
[open('left-commit.txt').read(),
|
||||
open('right-commit.txt').read()]]])
|
||||
print_tested_commits()
|
||||
|
||||
def print_changes():
|
||||
rows = tsvRows('report/changed-perf.tsv')
|
||||
|
|
@ -288,15 +317,7 @@ if args.report == 'main':
|
|||
|
||||
print_test_times()
|
||||
|
||||
# Add the errors reported by various steps of comparison script
|
||||
report_errors += [l.strip() for l in open('report/errors.log')]
|
||||
if len(report_errors):
|
||||
print(tableStart('Errors while building the report'))
|
||||
print(tableHeader(['Error']))
|
||||
for x in report_errors:
|
||||
print(tableRow([x]))
|
||||
print(tableEnd())
|
||||
|
||||
print_report_errors()
|
||||
|
||||
print("""
|
||||
<p class="links">
|
||||
|
|
@ -350,10 +371,7 @@ elif args.report == 'all-queries':
|
|||
|
||||
print(header_template.format())
|
||||
|
||||
printSimpleTable('Tested commits', ['Old', 'New'],
|
||||
[['<pre>{}</pre>'.format(x) for x in
|
||||
[open('left-commit.txt').read(),
|
||||
open('right-commit.txt').read()]]])
|
||||
print_tested_commits()
|
||||
|
||||
def print_all_queries():
|
||||
rows = tsvRows('report/all-queries.tsv')
|
||||
|
|
@ -405,6 +423,8 @@ elif args.report == 'all-queries':
|
|||
|
||||
print_all_queries()
|
||||
|
||||
print_report_errors()
|
||||
|
||||
print("""
|
||||
<p class="links">
|
||||
<a href="output.7z">Test output</a>
|
||||
|
|
|
|||
Loading…
Reference in New Issue