wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

mTLS test helpers (#7218) 

* Add TLS option to (local_cluster|tmp_cluster).py

* Add TLS-enabled C API test
This commit is contained in:
Junhyun Shim 2022-05-23 12:47:51 +02:00 committed by GitHub
parent c073f113a5
commit 02b2f97e99
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 205 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
bindings/c/CMakeLists.txt
View File

@ -301,6 +301,31 @@ endif()
@LOG_DIR@
)
add_fdbclient_test(
NAME fdb_c_api_tests_with_tls
DISABLE_LOG_DUMP
TLS_ENABLED
COMMAND ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py
--cluster-file
@CLUSTER_FILE@
--tester-binary
$<TARGET_FILE:fdb_c_api_tester>
--external-client-library
${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
--test-dir
${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
--tmp-dir
@TMP_DIR@
--log-dir
@LOG_DIR@
--tls-cert-file
@CLIENT_CERT_FILE@
--tls-key-file
@CLIENT_KEY_FILE@
--tls-ca-file
@SERVER_CA_FILE@
)
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT USE_SANITIZER)
add_test(NAME fdb_c_upgrade_single_threaded_630api
COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py

3
bindings/c/test/apitester/TesterOptions.h
View File

@ -52,6 +52,9 @@ public:
std::vector<std::pair<std::string, std::string>> knobs;
TestSpec testSpec;
std::string bgBasePath;
std::string tlsCertFile;
std::string tlsKeyFile;
std::string tlsCaFile;
};
} // namespace FdbApiTester

35
bindings/c/test/apitester/fdb_c_api_tester.cpp
View File

@ -54,7 +54,10 @@ enum TesterOptionId {
OPT_FDB_API_VERSION,
OPT_TRANSACTION_RETRY_LIMIT,
OPT_BLOB_GRANULE_LOCAL_FILE_PATH,
OPT_STATS_INTERVAL
OPT_STATS_INTERVAL,
OPT_TLS_CERT_FILE,
OPT_TLS_KEY_FILE,
OPT_TLS_CA_FILE,
};
CSimpleOpt::SOption TesterOptionDefs[] = //
@ -79,6 +82,9 @@ CSimpleOpt::SOption TesterOptionDefs[] = //
{ OPT_TRANSACTION_RETRY_LIMIT, "--transaction-retry-limit", SO_REQ_SEP },
{ OPT_BLOB_GRANULE_LOCAL_FILE_PATH, "--blob-granule-local-file-path", SO_REQ_SEP },
{ OPT_STATS_INTERVAL, "--stats-interval", SO_REQ_SEP },
{ OPT_TLS_CERT_FILE, "--tls-cert-file", SO_REQ_SEP },
{ OPT_TLS_KEY_FILE, "--tls-key-file", SO_REQ_SEP },
{ OPT_TLS_CA_FILE, "--tls-ca-file", SO_REQ_SEP },
SO_END_OF_OPTIONS };
void printProgramUsage(const char* execName) {
@ -122,6 +128,12 @@ void printProgramUsage(const char* execName) {
" Test file to run.\n"
" --stats-interval MILLISECONDS\n"
" Time interval in milliseconds for printing workload statistics (default: 0 - disabled).\n"
" --tls-cert-file FILE\n"
" Path to file containing client's TLS certificate chain\n"
" --tls-key-file FILE\n"
" Path to file containing client's TLS private key\n"
" --tls-ca-file FILE\n"
" Path to file containing TLS CA certificate\n"
" -h, --help Display this help and exit.\n",
FDB_API_VERSION);
}
@ -221,6 +233,15 @@ bool processArg(TesterOptions& options, const CSimpleOpt& args) {
case OPT_STATS_INTERVAL:
processIntOption(args.OptionText(), args.OptionArg(), 0, 60000, options.statsIntervalMs);
break;
case OPT_TLS_CERT_FILE:
options.tlsCertFile.assign(args.OptionArg());
break;
case OPT_TLS_KEY_FILE:
options.tlsKeyFile.assign(args.OptionArg());
break;
case OPT_TLS_CA_FILE:
options.tlsCaFile.assign(args.OptionArg());
break;
}
return true;
}
@ -299,6 +320,18 @@ void applyNetworkOptions(TesterOptions& options) {
fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_KNOB,
fmt::format("{}={}", knob.first.c_str(), knob.second.c_str())));
}
if (!options.tlsCertFile.empty()) {
fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_TLS_CERT_PATH, options.tlsCertFile));
}
if (!options.tlsKeyFile.empty()) {
fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_TLS_KEY_PATH, options.tlsKeyFile));
}
if (!options.tlsCaFile.empty()) {
fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_TLS_CA_PATH, options.tlsCaFile));
}
}
void randomizeOptions(TesterOptions& options) {

15
bindings/c/test/apitester/run_c_api_tests.py
View File

@ -84,6 +84,15 @@ def run_tester(args, test_file):
cmd += ["--blob-granule-local-file-path",
args.blob_granule_local_file_path]
if args.tls_ca_file is not None:
cmd += ["--tls-ca-file", args.tls_ca_file]
if args.tls_key_file is not None:
cmd += ["--tls-key-file", args.tls_key_file]
if args.tls_cert_file is not None:
cmd += ["--tls-cert-file", args.tls_cert_file]
get_logger().info('\nRunning tester \'%s\'...' % ' '.join(cmd))
proc = Popen(cmd, stdout=sys.stdout, stderr=sys.stderr)
timed_out = False
@ -149,6 +158,12 @@ def parse_args(argv):
help='The directory for storing temporary files (default: None)')
parser.add_argument('--blob-granule-local-file-path', type=str, default=None,
help='Enable blob granule tests if set, value is path to local blob granule files')
parser.add_argument('--tls-ca-file', type=str, default=None,
help='Path to client\'s TLS CA file: i.e. certificate of CA that signed the server certificate')
parser.add_argument('--tls-cert-file', type=str, default=None,
help='Path to client\'s TLS certificate file')
parser.add_argument('--tls-key-file', type=str, default=None,
help='Path to client\'s TLS private key file')
return parser.parse_args(argv)

6
cmake/AddFdbTest.cmake
View File

@ -404,8 +404,7 @@ endfunction()
# Creates a single cluster before running the specified command (usually a ctest test)
function(add_fdbclient_test)
set(options DISABLED ENABLED DISABLE_LOG_DUMP)
set(options DISABLED ENABLED API_TEST_BLOB_GRANULES_ENABLED)
set(options DISABLED ENABLED DISABLE_LOG_DUMP API_TEST_BLOB_GRANULES_ENABLED TLS_ENABLED)
set(oneValueArgs NAME PROCESS_NUMBER TEST_TIMEOUT WORKING_DIRECTORY)
set(multiValueArgs COMMAND)
cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
@ -435,6 +434,9 @@ function(add_fdbclient_test)
if(T_API_TEST_BLOB_GRANULES_ENABLED)
list(APPEND TMP_CLUSTER_CMD --blob-granules-enabled)
endif()
if(T_TLS_ENABLED)
list(APPEND TMP_CLUSTER_CMD --tls-enabled)
endif()
message(STATUS "Adding Client test ${T_NAME}")
add_test(NAME "${T_NAME}"
WORKING_DIRECTORY ${T_WORKING_DIRECTORY}

3
flow/MkCertCli.cpp
View File

@ -179,9 +179,6 @@ mkcert::CertChainRef ChainSpec::makeChain(Arena& arena) {
ofsCert.write(reinterpret_cast<char const*>(cert.begin()), cert.size());
auto key = chain[0].privateKeyPem;
ofsKey.write(reinterpret_cast<char const*>(key.begin()), key.size());
ofsCert.close();
ofsKey.close();
ofsCa.close();
return chain;
}

79
tests/TestRunner/local_cluster.py
View File

@ -39,6 +39,17 @@ def is_port_in_use(port):
valid_letters_for_secret = string.ascii_letters + string.digits
class TLSConfig:
# Passing a negative chain length generates expired leaf certificate
def __init__(
self,
server_chain_len: int = 3,
client_chain_len: int = 2,
verify_peers = "Check.Valid=1",
):
self.server_chain_len = server_chain_len
self.client_chain_len = client_chain_len
self.verify_peers = verify_peers
def random_secret_string(length):
return "".join(random.choice(valid_letters_for_secret) for _ in range(length))
@ -67,11 +78,12 @@ cluster-file = {etcdir}/fdb.cluster
## Default parameters for individual fdbserver processes
[fdbserver]
command = {fdbserver_bin}
public-address = {ip_address}:$ID
public-address = {ip_address}:$ID{optional_tls}
listen-address = public
datadir = {datadir}/$ID
logdir = {logdir}
{bg_knob_line}
{tls_config}
# logsize = 10MiB
# maxlogssize = 100MiB
# machine-id =
@ -98,12 +110,15 @@ logdir = {logdir}
port=None,
ip_address=None,
blob_granules_enabled: bool = False,
redundancy: str = "single"
redundancy: str = "single",
tls_config: TLSConfig = None,
mkcert_binary: str = "",
):
self.basedir = Path(basedir)
self.etc = self.basedir.joinpath("etc")
self.log = self.basedir.joinpath("log")
self.data = self.basedir.joinpath("data")
self.cert = self.basedir.joinpath("cert")
self.conf_file = self.etc.joinpath("foundationdb.conf")
self.cluster_file = self.etc.joinpath("fdb.cluster")
self.fdbserver_binary = Path(fdbserver_binary)
@ -137,11 +152,22 @@ logdir = {logdir}
self.use_legacy_conf_syntax = False
self.coordinators = set()
self.active_servers = set(self.server_ports.keys())
self.tls_config = tls_config
self.mkcert_binary = Path(mkcert_binary)
self.server_cert_file = self.cert.joinpath("server_cert.pem")
self.client_cert_file = self.cert.joinpath("client_cert.pem")
self.server_key_file = self.cert.joinpath("server_key.pem")
self.client_key_file = self.cert.joinpath("client_key.pem")
self.server_ca_file = self.cert.joinpath("server_ca.pem")
self.client_ca_file = self.cert.joinpath("client_ca.pem")
if create_config:
self.create_cluster_file()
self.save_config()
if self.tls_config is not None:
self.create_tls_cert()
def __next_port(self):
if self.first_port is None:
return get_free_port()
@ -166,6 +192,8 @@ logdir = {logdir}
logdir=self.log,
ip_address=self.ip_address,
bg_knob_line=bg_knob_line,
tls_config=self.tls_conf_string(),
optional_tls=":tls" if self.tls_config is not None else "",
)
)
# By default, the cluster only has one process
@ -190,11 +218,12 @@ logdir = {logdir}
def create_cluster_file(self):
with open(self.cluster_file, "x") as f:
f.write(
"{desc}:{secret}@{ip_addr}:{server_port}".format(
"{desc}:{secret}@{ip_addr}:{server_port}{optional_tls}".format(
desc=self.cluster_desc,
secret=self.cluster_secret,
ip_addr=self.ip_address,
server_port=self.server_ports[0],
optional_tls=":tls" if self.tls_config is not None else "",
)
)
self.coordinators = {0}
@ -248,6 +277,10 @@ logdir = {logdir}
def __fdbcli_exec(self, cmd, stdout, stderr, timeout):
args = [self.fdbcli_binary, "-C", self.cluster_file, "--exec", cmd]
if self.tls_config:
args += ["--tls-certificate-file", self.client_cert_file,
"--tls-key-file", self.client_key_file,
"--tls-ca-file", self.server_ca_file]
res = subprocess.run(args, env=self.process_env(), stderr=stderr, stdout=stdout, timeout=timeout)
assert res.returncode == 0, "fdbcli command {} failed with {}".format(cmd, res.returncode)
return res.stdout
@ -271,6 +304,46 @@ logdir = {logdir}
if self.blob_granules_enabled:
self.fdbcli_exec("blobrange start \\x00 \\xff")
# Generate and install test certificate chains and keys
def create_tls_cert(self):
assert self.tls_config is not None, "TLS not enabled"
assert self.mkcert_binary.exists() and self.mkcert_binary.is_file(), "{} does not exist".format(self.mkcert_binary)
self.cert.mkdir(exist_ok=True)
server_chain_len = abs(self.tls_config.server_chain_len)
client_chain_len = abs(self.tls_config.client_chain_len)
expire_server_cert = (self.tls_config.server_chain_len < 0)
expire_client_cert = (self.tls_config.client_chain_len < 0)
args = [
str(self.mkcert_binary),
"--server-chain-length", str(server_chain_len),
"--client-chain-length", str(client_chain_len),
"--server-cert-file", str(self.server_cert_file),
"--client-cert-file", str(self.client_cert_file),
"--server-key-file", str(self.server_key_file),
"--client-key-file", str(self.client_key_file),
"--server-ca-file", str(self.server_ca_file),
"--client-ca-file", str(self.client_ca_file),
"--print-args",
]
if expire_server_cert:
args.append("--expire-server-cert")
if expire_client_cert:
args.append("--expire-client-cert")
subprocess.run(args, check=True)
# Materialize server's TLS configuration section
def tls_conf_string(self):
if self.tls_config is None:
return ""
else:
conf_map = {
"tls-certificate-file": self.server_cert_file,
"tls-key-file": self.server_key_file,
"tls-ca-file": self.client_ca_file,
"tls-verify-peers": self.tls_config.verify_peers,
}
return "\n".join("{} = {}".format(k, v) for k, v in conf_map.items())
# Get cluster status using fdbcli
def get_status(self):
status_output = self.fdbcli_exec_and_get("status json")

59
tests/TestRunner/tmp_cluster.py
View File

@ -5,25 +5,27 @@ import os
import shutil
import subprocess
import sys
from local_cluster import LocalCluster, random_secret_string
from local_cluster import LocalCluster, TLSConfig, random_secret_string
from argparse import ArgumentParser, RawDescriptionHelpFormatter
from pathlib import Path
class TempCluster:
class TempCluster(LocalCluster):
def __init__(
self,
build_dir: str,
process_number: int = 1,
port: str = None,
blob_granules_enabled: bool = False,
tls_config: TLSConfig = None,
):
self.build_dir = Path(build_dir).resolve()
assert self.build_dir.exists(), "{} does not exist".format(build_dir)
assert self.build_dir.is_dir(), "{} is not a directory".format(build_dir)
tmp_dir = self.build_dir.joinpath("tmp", random_secret_string(16))
tmp_dir.mkdir(parents=True)
self.cluster = LocalCluster(
self.tmp_dir = tmp_dir
super().__init__(
tmp_dir,
self.build_dir.joinpath("bin", "fdbserver"),
self.build_dir.joinpath("bin", "fdbmonitor"),
@ -31,23 +33,21 @@ class TempCluster:
process_number,
port=port,
blob_granules_enabled=blob_granules_enabled,
tls_config=tls_config,
mkcert_binary=self.build_dir.joinpath("bin", "mkcert"),
)
self.log = self.cluster.log
self.etc = self.cluster.etc
self.data = self.cluster.data
self.tmp_dir = tmp_dir
def __enter__(self):
self.cluster.__enter__()
self.cluster.create_database()
super().__enter__()
super().create_database()
return self
def __exit__(self, xc_type, exc_value, traceback):
self.cluster.__exit__(xc_type, exc_value, traceback)
super().__exit__(xc_type, exc_value, traceback)
shutil.rmtree(self.tmp_dir)
def close(self):
self.cluster.__exit__(None, None, None)
super().__exit__(None, None, None)
shutil.rmtree(self.tmp_dir)
@ -94,12 +94,37 @@ if __name__ == "__main__":
parser.add_argument(
"--blob-granules-enabled", help="Enable blob granules", action="store_true"
)
parser.add_argument(
"--tls-enabled", help="Enable TLS (with test-only certificates)", action="store_true")
parser.add_argument(
"--server-cert-chain-len",
help="Length of server TLS certificate chain including root CA. Negative value deliberately generates expired leaf certificate for TLS testing. Only takes effect with --tls-enabled.",
type=int,
default=3,
)
parser.add_argument(
"--client-cert-chain-len",
help="Length of client TLS certificate chain including root CA. Negative value deliberately generates expired leaf certificate for TLS testing. Only takes effect with --tls-enabled.",
type=int,
default=2,
)
parser.add_argument(
"--tls-verify-peer",
help="Rules to verify client certificate chain. See https://apple.github.io/foundationdb/tls.html#peer-verification",
type=str,
default="Check.Valid=1",
)
args = parser.parse_args()
tls_config = None
if args.tls_enabled:
tls_config = TLSConfig(server_chain_len=args.server_cert_chain_len,
client_chain_len=args.client_cert_chain_len)
errcode = 1
with TempCluster(
args.build_dir,
args.process_number,
blob_granules_enabled=args.blob_granules_enabled,
tls_config=tls_config,
) as cluster:
print("log-dir: {}".format(cluster.log))
print("etc-dir: {}".format(cluster.etc))
@ -117,6 +142,18 @@ if __name__ == "__main__":
cmd_args.append(str(cluster.etc))
elif cmd == "@TMP_DIR@":
cmd_args.append(str(cluster.tmp_dir))
elif cmd == "@SERVER_CERT_FILE@":
cmd_args.append(str(cluster.server_cert_file))
elif cmd == "@SERVER_KEY_FILE@":
cmd_args.append(str(cluster.server_key_file))
elif cmd == "@SERVER_CA_FILE@":
cmd_args.append(str(cluster.server_ca_file))
elif cmd == "@CLIENT_CERT_FILE@":
cmd_args.append(str(cluster.client_cert_file))
elif cmd == "@CLIENT_KEY_FILE@":
cmd_args.append(str(cluster.client_key_file))
elif cmd == "@CLIENT_CA_FILE@":
cmd_args.append(str(cluster.client_ca_file))
elif cmd.startswith("@DATA_DIR@"):
cmd_args.append(str(cluster.data) + cmd[len("@DATA_DIR@") :])
else: