Add some tests for timeout behavior with disconnected clusters
This commit is contained in:
A.J. Beamon
Markus Pilman
parent
a3d53c57b1
commit
de863d170d
|
|
@ -82,6 +82,10 @@ if(NOT WIN32)
|
|||
|
||||
set(UNIT_TEST_VERSION_510_SRCS test/unit/unit_tests_version_510.cpp)
|
||||
set(TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS test/unit/trace_partial_file_suffix_test.cpp)
|
||||
set(DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS
|
||||
test/unit/disconnected_timeout_tests.cpp
|
||||
test/unit/fdb_api.cpp
|
||||
test/unit/fdb_api.hpp)
|
||||
|
||||
if(OPEN_FOR_IDE)
|
||||
add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h)
|
||||
|
|
@ -92,6 +96,7 @@ if(NOT WIN32)
|
|||
add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS})
|
||||
add_library(fdb_c_unit_tests_version_510 OBJECT ${UNIT_TEST_VERSION_510_SRCS})
|
||||
add_library(trace_partial_file_suffix_test OBJECT ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
|
||||
add_library(disconnected_timeout_unit_tests OBJECT ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})
|
||||
else()
|
||||
add_executable(fdb_c_performance_test test/performance_test.c test/test.h)
|
||||
add_executable(fdb_c_ryw_benchmark test/ryw_benchmark.c test/test.h)
|
||||
|
|
@ -101,6 +106,7 @@ if(NOT WIN32)
|
|||
add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS})
|
||||
add_executable(fdb_c_unit_tests_version_510 ${UNIT_TEST_VERSION_510_SRCS})
|
||||
add_executable(trace_partial_file_suffix_test ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
|
||||
add_executable(disconnected_timeout_unit_tests ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})
|
||||
strip_debug_symbols(fdb_c_performance_test)
|
||||
strip_debug_symbols(fdb_c_ryw_benchmark)
|
||||
strip_debug_symbols(fdb_c_txn_size_test)
|
||||
|
|
@ -112,13 +118,16 @@ if(NOT WIN32)
|
|||
add_dependencies(fdb_c_setup_tests doctest)
|
||||
add_dependencies(fdb_c_unit_tests doctest)
|
||||
add_dependencies(fdb_c_unit_tests_version_510 doctest)
|
||||
add_dependencies(disconnected_timeout_unit_tests doctest)
|
||||
target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
|
||||
target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
|
||||
target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR})
|
||||
target_include_directories(disconnected_timeout_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
|
||||
target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads)
|
||||
target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads)
|
||||
target_link_libraries(fdb_c_unit_tests_version_510 PRIVATE fdb_c Threads::Threads)
|
||||
target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads)
|
||||
target_link_libraries(disconnected_timeout_unit_tests PRIVATE fdb_c Threads::Threads)
|
||||
|
||||
# do not set RPATH for mako
|
||||
set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
|
||||
|
|
@ -170,6 +179,13 @@ if(NOT WIN32)
|
|||
fdb
|
||||
${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so
|
||||
)
|
||||
add_unavailable_fdbclient_test(
|
||||
NAME disconnected_timeout_unit_tests
|
||||
COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>
|
||||
@CLUSTER_FILE@
|
||||
fdb
|
||||
${CMAKE_CURRENT_BINARY_DIR}/libfdb_c.so
|
||||
)
|
||||
endif()
|
||||
|
||||
set(c_workloads_srcs
|
||||
|
|
|
|||
|
|
@ -0,0 +1,292 @@
|
|||
/*
|
||||
* disconnected_timeout_tests.cpp
|
||||
*
|
||||
* This source file is part of the FoundationDB open source project
|
||||
*
|
||||
* Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Unit tests that test the timeouts for a disconnected cluster
|
||||
|
||||
#define FDB_API_VERSION 710
|
||||
#include <foundationdb/fdb_c.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <iostream>
|
||||
#include <string.h>
|
||||
#include <thread>
|
||||
|
||||
#define DOCTEST_CONFIG_IMPLEMENT
|
||||
#include "doctest.h"
|
||||
#include "fdb_api.hpp"
|
||||
|
||||
void fdb_check(fdb_error_t e) {
|
||||
if (e) {
|
||||
std::cerr << fdb_get_error(e) << std::endl;
|
||||
std::abort();
|
||||
}
|
||||
}
|
||||
|
||||
FDBDatabase* fdb_open_database(const char* clusterFile) {
|
||||
FDBDatabase* db;
|
||||
fdb_check(fdb_create_database(clusterFile, &db));
|
||||
return db;
|
||||
}
|
||||
|
||||
static FDBDatabase* db = nullptr;
|
||||
static FDBDatabase* timeoutDb = nullptr;
|
||||
|
||||
// Blocks until the given future is ready, returning an error code if there was
|
||||
// an issue.
|
||||
fdb_error_t wait_future(fdb::Future& f) {
|
||||
fdb_check(f.block_until_ready());
|
||||
return f.get_error();
|
||||
}
|
||||
|
||||
void validateTimeoutDuration(double expectedSeconds, std::chrono::time_point<std::chrono::steady_clock> start) {
|
||||
std::chrono::duration<double> duration = std::chrono::steady_clock::now() - start;
|
||||
double actualSeconds = duration.count();
|
||||
CHECK(actualSeconds >= expectedSeconds - 1e-6);
|
||||
CHECK(actualSeconds < expectedSeconds * 2);
|
||||
}
|
||||
|
||||
TEST_CASE("500ms_transaction_timeout") {
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
|
||||
fdb::Transaction tr(db);
|
||||
|
||||
int64_t timeout = 500;
|
||||
fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
|
||||
fdb::Int64Future grvFuture = tr.get_read_version();
|
||||
fdb_error_t err = wait_future(grvFuture);
|
||||
|
||||
CHECK(err == 1031);
|
||||
validateTimeoutDuration(timeout / 1000.0, start);
|
||||
}
|
||||
|
||||
TEST_CASE("500ms_transaction_timeout_after_op") {
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
|
||||
fdb::Transaction tr(db);
|
||||
fdb::Int64Future grvFuture = tr.get_read_version();
|
||||
|
||||
int64_t timeout = 500;
|
||||
fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
|
||||
fdb_error_t err = wait_future(grvFuture);
|
||||
|
||||
CHECK(err == 1031);
|
||||
validateTimeoutDuration(timeout / 1000.0, start);
|
||||
}
|
||||
|
||||
TEST_CASE("500ms_transaction_timeout_before_op_2000ms_after") {
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
|
||||
fdb::Transaction tr(db);
|
||||
|
||||
int64_t timeout = 500;
|
||||
fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
|
||||
fdb::Int64Future grvFuture = tr.get_read_version();
|
||||
|
||||
timeout = 2000;
|
||||
fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
|
||||
fdb_error_t err = wait_future(grvFuture);
|
||||
|
||||
CHECK(err == 1031);
|
||||
validateTimeoutDuration(timeout / 1000.0, start);
|
||||
}
|
||||
|
||||
TEST_CASE("2000ms_transaction_timeout_before_op_500ms_after") {
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
|
||||
fdb::Transaction tr(db);
|
||||
|
||||
int64_t timeout = 2000;
|
||||
fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
|
||||
fdb::Int64Future grvFuture = tr.get_read_version();
|
||||
|
||||
timeout = 500;
|
||||
fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
|
||||
fdb_error_t err = wait_future(grvFuture);
|
||||
|
||||
CHECK(err == 1031);
|
||||
validateTimeoutDuration(timeout / 1000.0, start);
|
||||
}
|
||||
|
||||
TEST_CASE("500ms_database_timeout") {
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
|
||||
int64_t timeout = 500;
|
||||
fdb_check(fdb_database_set_option(
|
||||
timeoutDb, FDB_DB_OPTION_TRANSACTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
|
||||
fdb::Transaction tr(timeoutDb);
|
||||
|
||||
fdb::Int64Future grvFuture = tr.get_read_version();
|
||||
fdb_error_t err = wait_future(grvFuture);
|
||||
|
||||
CHECK(err == 1031);
|
||||
validateTimeoutDuration(timeout / 1000.0, start);
|
||||
}
|
||||
|
||||
TEST_CASE("2000ms_database_timeout_500ms_transaction_timeout") {
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
|
||||
int64_t timeout = 2000;
|
||||
fdb_check(fdb_database_set_option(
|
||||
timeoutDb, FDB_DB_OPTION_TRANSACTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
|
||||
fdb::Transaction tr(timeoutDb);
|
||||
|
||||
timeout = 500;
|
||||
fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
|
||||
fdb::Int64Future grvFuture = tr.get_read_version();
|
||||
fdb_error_t err = wait_future(grvFuture);
|
||||
|
||||
CHECK(err == 1031);
|
||||
validateTimeoutDuration(timeout / 1000.0, start);
|
||||
}
|
||||
|
||||
TEST_CASE("500ms_database_timeout_2000ms_transaction_timeout_with_reset") {
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
|
||||
int64_t dbTimeout = 500;
|
||||
fdb_check(fdb_database_set_option(
|
||||
timeoutDb, FDB_DB_OPTION_TRANSACTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&dbTimeout), sizeof(dbTimeout)));
|
||||
|
||||
fdb::Transaction tr(timeoutDb);
|
||||
|
||||
int64_t trTimeout = 2000;
|
||||
fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&trTimeout), sizeof(trTimeout)));
|
||||
|
||||
tr.reset();
|
||||
|
||||
fdb::Int64Future grvFuture = tr.get_read_version();
|
||||
fdb_error_t err = wait_future(grvFuture);
|
||||
|
||||
CHECK(err == 1031);
|
||||
validateTimeoutDuration(dbTimeout / 1000.0, start);
|
||||
}
|
||||
|
||||
TEST_CASE("transaction_reset_cancels_without_timeout") {
|
||||
fdb::Transaction tr(db);
|
||||
fdb::Int64Future grvFuture = tr.get_read_version();
|
||||
tr.reset();
|
||||
|
||||
fdb_error_t err = wait_future(grvFuture);
|
||||
CHECK(err == 1025);
|
||||
}
|
||||
|
||||
TEST_CASE("transaction_reset_cancels_with_timeout") {
|
||||
fdb::Transaction tr(db);
|
||||
|
||||
int64_t timeout = 500;
|
||||
fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
|
||||
fdb::Int64Future grvFuture = tr.get_read_version();
|
||||
tr.reset();
|
||||
|
||||
fdb_error_t err = wait_future(grvFuture);
|
||||
CHECK(err == 1025);
|
||||
}
|
||||
|
||||
TEST_CASE("transaction_destruction_cancels_without_timeout") {
|
||||
FDBTransaction* tr;
|
||||
fdb_check(fdb_database_create_transaction(db, &tr));
|
||||
|
||||
FDBFuture* grvFuture = fdb_transaction_get_read_version(tr);
|
||||
fdb_transaction_destroy(tr);
|
||||
|
||||
fdb_check(fdb_future_block_until_ready(grvFuture));
|
||||
fdb_error_t err = fdb_future_get_error(grvFuture);
|
||||
CHECK(err == 1025);
|
||||
|
||||
fdb_future_destroy(grvFuture);
|
||||
}
|
||||
|
||||
TEST_CASE("transaction_destruction_cancels_with_timeout") {
|
||||
FDBTransaction* tr;
|
||||
fdb_check(fdb_database_create_transaction(db, &tr));
|
||||
|
||||
int64_t timeout = 500;
|
||||
fdb_check(fdb_transaction_set_option(
|
||||
tr, FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
|
||||
FDBFuture* grvFuture = fdb_transaction_get_read_version(tr);
|
||||
fdb_transaction_destroy(tr);
|
||||
|
||||
fdb_check(fdb_future_block_until_ready(grvFuture));
|
||||
fdb_error_t err = fdb_future_get_error(grvFuture);
|
||||
CHECK(err == 1025);
|
||||
|
||||
fdb_future_destroy(grvFuture);
|
||||
}
|
||||
|
||||
TEST_CASE("transaction_set_timeout_and_destroy_repeatedly") {
|
||||
for (int i = 0; i < 1000; ++i) {
|
||||
fdb::Transaction tr(db);
|
||||
int64_t timeout = 500;
|
||||
fdb_check(tr.set_option(FDB_TR_OPTION_TIMEOUT, reinterpret_cast<const uint8_t*>(&timeout), sizeof(timeout)));
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc < 2) {
|
||||
std::cout << "Disconnected timeout unit tests for the FoundationDB C API.\n"
|
||||
<< "Usage: disconnected_timeout_tests <unavailableClusterFile> [externalClient] [doctest args]"
|
||||
<< std::endl;
|
||||
return 1;
|
||||
}
|
||||
fdb_check(fdb_select_api_version(710));
|
||||
if (argc >= 3) {
|
||||
std::string externalClientLibrary = argv[2];
|
||||
if (externalClientLibrary.substr(0, 2) != "--") {
|
||||
fdb_check(fdb_network_set_option(
|
||||
FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT, reinterpret_cast<const uint8_t*>(""), 0));
|
||||
fdb_check(fdb_network_set_option(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_LIBRARY,
|
||||
reinterpret_cast<const uint8_t*>(externalClientLibrary.c_str()),
|
||||
externalClientLibrary.size()));
|
||||
}
|
||||
}
|
||||
|
||||
doctest::Context context;
|
||||
context.applyCommandLine(argc, argv);
|
||||
|
||||
fdb_check(fdb_setup_network());
|
||||
std::thread network_thread{ &fdb_run_network };
|
||||
|
||||
db = fdb_open_database(argv[1]);
|
||||
timeoutDb = fdb_open_database(argv[1]);
|
||||
|
||||
int res = context.run();
|
||||
fdb_database_destroy(db);
|
||||
fdb_database_destroy(timeoutDb);
|
||||
|
||||
if (context.shouldExit()) {
|
||||
fdb_check(fdb_stop_network());
|
||||
network_thread.join();
|
||||
return res;
|
||||
}
|
||||
fdb_check(fdb_stop_network());
|
||||
network_thread.join();
|
||||
|
||||
return res;
|
||||
}
|
||||
|
|
@ -443,6 +443,40 @@ function(add_fdbclient_test)
|
|||
set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1)
|
||||
endfunction()
|
||||
|
||||
# Creates a cluster file for a nonexistent cluster before running the specified command
|
||||
# (usually a ctest test)
|
||||
function(add_unavailable_fdbclient_test)
|
||||
set(options DISABLED ENABLED)
|
||||
set(oneValueArgs NAME TEST_TIMEOUT)
|
||||
set(multiValueArgs COMMAND)
|
||||
cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
|
||||
if(OPEN_FOR_IDE)
|
||||
return()
|
||||
endif()
|
||||
if(NOT T_ENABLED AND T_DISABLED)
|
||||
return()
|
||||
endif()
|
||||
if(NOT T_NAME)
|
||||
message(FATAL_ERROR "NAME is a required argument for add_unavailable_fdbclient_test")
|
||||
endif()
|
||||
if(NOT T_COMMAND)
|
||||
message(FATAL_ERROR "COMMAND is a required argument for add_unavailable_fdbclient_test")
|
||||
endif()
|
||||
message(STATUS "Adding unavailable client test ${T_NAME}")
|
||||
add_test(NAME "${T_NAME}"
|
||||
COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/fake_cluster.py
|
||||
--output-dir ${CMAKE_BINARY_DIR}
|
||||
--
|
||||
${T_COMMAND})
|
||||
if (T_TEST_TIMEOUT)
|
||||
set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT ${T_TEST_TIMEOUT})
|
||||
else()
|
||||
# default timeout
|
||||
set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60)
|
||||
endif()
|
||||
set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1)
|
||||
endfunction()
|
||||
|
||||
# Creates 3 distinct clusters before running the specified command.
|
||||
# This is useful for testing features that require multiple clusters (like the
|
||||
# multi-cluster FDB client)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from local_cluster import LocalCluster
|
||||
from argparse import ArgumentParser, RawDescriptionHelpFormatter
|
||||
from pathlib import Path
|
||||
|
||||
class ClusterFileGenerator:
|
||||
def __init__(self, output_dir: str):
|
||||
self.output_dir = Path(output_dir).resolve()
|
||||
assert self.output_dir.exists(), "{} does not exist".format(output_dir)
|
||||
assert self.output_dir.is_dir(), "{} is not a directory".format(output_dir)
|
||||
self.tmp_dir = self.output_dir.joinpath(
|
||||
'tmp',
|
||||
''.join(choice(LocalCluster.valid_letters_for_secret) for i in range(16)))
|
||||
tmp_dir.mkdir(parents=True)
|
||||
self.cluster_file_path = self.tmp_dir.joinpath('fdb.cluster')
|
||||
|
||||
def __enter__(self):
|
||||
with open(self.cluster_file_path, 'x') as f:
|
||||
f.write('foo:bar@1.1.1.1:5678')
|
||||
|
||||
return self
|
||||
|
||||
def __exit__(self, xc_type, exc_value, traceback):
|
||||
shutil.rmtree(self.tmp_dir)
|
||||
|
||||
def close(self):
|
||||
shutil.rmtree(self.tmp_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
|
||||
description="""
|
||||
This script generates a cluster file that can be used to run a test and that will
|
||||
be cleaned up when the test is over. The cluster file will not correspond to a real
|
||||
cluster.
|
||||
|
||||
Before the command is executed, the following arguments will be preprocessed:
|
||||
- All occurrences of @CLUSTER_FILE@ will be replaced with the path to the generated cluster file.
|
||||
|
||||
The environment variable FDB_CLUSTER_FILE is set to the generated cluster file for the command if
|
||||
it is not set already.
|
||||
""")
|
||||
parser.add_argument('--output-dir', '-o', metavar='OUTPUT_DIRECTORY', help='Directory where output files are written', required=True)
|
||||
parser.add_argument('cmd', metavar="COMMAND", nargs="+", help="The command to run")
|
||||
args = parser.parse_args()
|
||||
errcode = 1
|
||||
|
||||
with ClusterFileGenerator(args.output_dir) as generator:
|
||||
print("cluster-file: {}".format(generator.cluster_file_path))
|
||||
cmd_args = []
|
||||
for cmd in args.cmd:
|
||||
if cmd == '@CLUSTER_FILE@':
|
||||
cmd_args.append(str(generator.cluster_file_path))
|
||||
else:
|
||||
cmd_args.append(cmd)
|
||||
|
||||
env = dict(**os.environ)
|
||||
env['FDB_CLUSTER_FILE'] = env.get('FDB_CLUSTER_FILE', generator.cluster_file_path)
|
||||
errcode = subprocess.run(cmd_args, stdout=sys.stdout, stderr=sys.stderr, env=env).returncode
|
||||
|
||||
sys.exit(errcode)
|
||||
Loading…
Reference in New Issue