diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt index 104029f80d..4a49e16c84 100644 --- a/bindings/c/CMakeLists.txt +++ b/bindings/c/CMakeLists.txt @@ -301,6 +301,31 @@ endif() @LOG_DIR@ ) + add_fdbclient_test( + NAME fdb_c_api_tests_with_tls + DISABLE_LOG_DUMP + TLS_ENABLED + COMMAND ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py + --cluster-file + @CLUSTER_FILE@ + --tester-binary + $ + --external-client-library + ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so + --test-dir + ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests + --tmp-dir + @TMP_DIR@ + --log-dir + @LOG_DIR@ + --tls-cert-file + @CLIENT_CERT_FILE@ + --tls-key-file + @CLIENT_KEY_FILE@ + --tls-ca-file + @SERVER_CA_FILE@ + ) + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT USE_SANITIZER) add_test(NAME fdb_c_upgrade_single_threaded_630api COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py diff --git a/bindings/c/test/apitester/TesterOptions.h b/bindings/c/test/apitester/TesterOptions.h index 6eac40b03a..3ff57ec183 100644 --- a/bindings/c/test/apitester/TesterOptions.h +++ b/bindings/c/test/apitester/TesterOptions.h @@ -52,6 +52,9 @@ public: std::vector> knobs; TestSpec testSpec; std::string bgBasePath; + std::string tlsCertFile; + std::string tlsKeyFile; + std::string tlsCaFile; }; } // namespace FdbApiTester diff --git a/bindings/c/test/apitester/fdb_c_api_tester.cpp b/bindings/c/test/apitester/fdb_c_api_tester.cpp index 3fe0156a90..a08aa9e291 100644 --- a/bindings/c/test/apitester/fdb_c_api_tester.cpp +++ b/bindings/c/test/apitester/fdb_c_api_tester.cpp @@ -54,7 +54,10 @@ enum TesterOptionId { OPT_FDB_API_VERSION, OPT_TRANSACTION_RETRY_LIMIT, OPT_BLOB_GRANULE_LOCAL_FILE_PATH, - OPT_STATS_INTERVAL + OPT_STATS_INTERVAL, + OPT_TLS_CERT_FILE, + OPT_TLS_KEY_FILE, + OPT_TLS_CA_FILE, }; CSimpleOpt::SOption TesterOptionDefs[] = // @@ -79,6 +82,9 @@ CSimpleOpt::SOption TesterOptionDefs[] = // { OPT_TRANSACTION_RETRY_LIMIT, "--transaction-retry-limit", SO_REQ_SEP }, { OPT_BLOB_GRANULE_LOCAL_FILE_PATH, "--blob-granule-local-file-path", SO_REQ_SEP }, { OPT_STATS_INTERVAL, "--stats-interval", SO_REQ_SEP }, + { OPT_TLS_CERT_FILE, "--tls-cert-file", SO_REQ_SEP }, + { OPT_TLS_KEY_FILE, "--tls-key-file", SO_REQ_SEP }, + { OPT_TLS_CA_FILE, "--tls-ca-file", SO_REQ_SEP }, SO_END_OF_OPTIONS }; void printProgramUsage(const char* execName) { @@ -122,6 +128,12 @@ void printProgramUsage(const char* execName) { " Test file to run.\n" " --stats-interval MILLISECONDS\n" " Time interval in milliseconds for printing workload statistics (default: 0 - disabled).\n" + " --tls-cert-file FILE\n" + " Path to file containing client's TLS certificate chain\n" + " --tls-key-file FILE\n" + " Path to file containing client's TLS private key\n" + " --tls-ca-file FILE\n" + " Path to file containing TLS CA certificate\n" " -h, --help Display this help and exit.\n", FDB_API_VERSION); } @@ -221,6 +233,15 @@ bool processArg(TesterOptions& options, const CSimpleOpt& args) { case OPT_STATS_INTERVAL: processIntOption(args.OptionText(), args.OptionArg(), 0, 60000, options.statsIntervalMs); break; + case OPT_TLS_CERT_FILE: + options.tlsCertFile.assign(args.OptionArg()); + break; + case OPT_TLS_KEY_FILE: + options.tlsKeyFile.assign(args.OptionArg()); + break; + case OPT_TLS_CA_FILE: + options.tlsCaFile.assign(args.OptionArg()); + break; } return true; } @@ -299,6 +320,18 @@ void applyNetworkOptions(TesterOptions& options) { fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_KNOB, fmt::format("{}={}", knob.first.c_str(), knob.second.c_str()))); } + + if (!options.tlsCertFile.empty()) { + fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_TLS_CERT_PATH, options.tlsCertFile)); + } + + if (!options.tlsKeyFile.empty()) { + fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_TLS_KEY_PATH, options.tlsKeyFile)); + } + + if (!options.tlsCaFile.empty()) { + fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_TLS_CA_PATH, options.tlsCaFile)); + } } void randomizeOptions(TesterOptions& options) { diff --git a/bindings/c/test/apitester/run_c_api_tests.py b/bindings/c/test/apitester/run_c_api_tests.py index 89cc0d2780..4756117c07 100755 --- a/bindings/c/test/apitester/run_c_api_tests.py +++ b/bindings/c/test/apitester/run_c_api_tests.py @@ -84,6 +84,15 @@ def run_tester(args, test_file): cmd += ["--blob-granule-local-file-path", args.blob_granule_local_file_path] + if args.tls_ca_file is not None: + cmd += ["--tls-ca-file", args.tls_ca_file] + + if args.tls_key_file is not None: + cmd += ["--tls-key-file", args.tls_key_file] + + if args.tls_cert_file is not None: + cmd += ["--tls-cert-file", args.tls_cert_file] + get_logger().info('\nRunning tester \'%s\'...' % ' '.join(cmd)) proc = Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) timed_out = False @@ -149,6 +158,12 @@ def parse_args(argv): help='The directory for storing temporary files (default: None)') parser.add_argument('--blob-granule-local-file-path', type=str, default=None, help='Enable blob granule tests if set, value is path to local blob granule files') + parser.add_argument('--tls-ca-file', type=str, default=None, + help='Path to client\'s TLS CA file: i.e. certificate of CA that signed the server certificate') + parser.add_argument('--tls-cert-file', type=str, default=None, + help='Path to client\'s TLS certificate file') + parser.add_argument('--tls-key-file', type=str, default=None, + help='Path to client\'s TLS private key file') return parser.parse_args(argv) diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake index a796f47b5b..c0440df5ef 100644 --- a/cmake/AddFdbTest.cmake +++ b/cmake/AddFdbTest.cmake @@ -404,8 +404,7 @@ endfunction() # Creates a single cluster before running the specified command (usually a ctest test) function(add_fdbclient_test) - set(options DISABLED ENABLED DISABLE_LOG_DUMP) - set(options DISABLED ENABLED API_TEST_BLOB_GRANULES_ENABLED) + set(options DISABLED ENABLED DISABLE_LOG_DUMP API_TEST_BLOB_GRANULES_ENABLED TLS_ENABLED) set(oneValueArgs NAME PROCESS_NUMBER TEST_TIMEOUT WORKING_DIRECTORY) set(multiValueArgs COMMAND) cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") @@ -435,6 +434,9 @@ function(add_fdbclient_test) if(T_API_TEST_BLOB_GRANULES_ENABLED) list(APPEND TMP_CLUSTER_CMD --blob-granules-enabled) endif() + if(T_TLS_ENABLED) + list(APPEND TMP_CLUSTER_CMD --tls-enabled) + endif() message(STATUS "Adding Client test ${T_NAME}") add_test(NAME "${T_NAME}" WORKING_DIRECTORY ${T_WORKING_DIRECTORY} diff --git a/flow/MkCertCli.cpp b/flow/MkCertCli.cpp index a4c1a4eb24..5369fee028 100644 --- a/flow/MkCertCli.cpp +++ b/flow/MkCertCli.cpp @@ -179,9 +179,6 @@ mkcert::CertChainRef ChainSpec::makeChain(Arena& arena) { ofsCert.write(reinterpret_cast(cert.begin()), cert.size()); auto key = chain[0].privateKeyPem; ofsKey.write(reinterpret_cast(key.begin()), key.size()); - ofsCert.close(); - ofsKey.close(); - ofsCa.close(); return chain; } diff --git a/tests/TestRunner/local_cluster.py b/tests/TestRunner/local_cluster.py index 4e47319b0f..9500e94b67 100644 --- a/tests/TestRunner/local_cluster.py +++ b/tests/TestRunner/local_cluster.py @@ -39,6 +39,17 @@ def is_port_in_use(port): valid_letters_for_secret = string.ascii_letters + string.digits +class TLSConfig: + # Passing a negative chain length generates expired leaf certificate + def __init__( + self, + server_chain_len: int = 3, + client_chain_len: int = 2, + verify_peers = "Check.Valid=1", + ): + self.server_chain_len = server_chain_len + self.client_chain_len = client_chain_len + self.verify_peers = verify_peers def random_secret_string(length): return "".join(random.choice(valid_letters_for_secret) for _ in range(length)) @@ -67,11 +78,12 @@ cluster-file = {etcdir}/fdb.cluster ## Default parameters for individual fdbserver processes [fdbserver] command = {fdbserver_bin} -public-address = {ip_address}:$ID +public-address = {ip_address}:$ID{optional_tls} listen-address = public datadir = {datadir}/$ID logdir = {logdir} {bg_knob_line} +{tls_config} # logsize = 10MiB # maxlogssize = 100MiB # machine-id = @@ -98,12 +110,15 @@ logdir = {logdir} port=None, ip_address=None, blob_granules_enabled: bool = False, - redundancy: str = "single" + redundancy: str = "single", + tls_config: TLSConfig = None, + mkcert_binary: str = "", ): self.basedir = Path(basedir) self.etc = self.basedir.joinpath("etc") self.log = self.basedir.joinpath("log") self.data = self.basedir.joinpath("data") + self.cert = self.basedir.joinpath("cert") self.conf_file = self.etc.joinpath("foundationdb.conf") self.cluster_file = self.etc.joinpath("fdb.cluster") self.fdbserver_binary = Path(fdbserver_binary) @@ -137,11 +152,22 @@ logdir = {logdir} self.use_legacy_conf_syntax = False self.coordinators = set() self.active_servers = set(self.server_ports.keys()) + self.tls_config = tls_config + self.mkcert_binary = Path(mkcert_binary) + self.server_cert_file = self.cert.joinpath("server_cert.pem") + self.client_cert_file = self.cert.joinpath("client_cert.pem") + self.server_key_file = self.cert.joinpath("server_key.pem") + self.client_key_file = self.cert.joinpath("client_key.pem") + self.server_ca_file = self.cert.joinpath("server_ca.pem") + self.client_ca_file = self.cert.joinpath("client_ca.pem") if create_config: self.create_cluster_file() self.save_config() + if self.tls_config is not None: + self.create_tls_cert() + def __next_port(self): if self.first_port is None: return get_free_port() @@ -166,6 +192,8 @@ logdir = {logdir} logdir=self.log, ip_address=self.ip_address, bg_knob_line=bg_knob_line, + tls_config=self.tls_conf_string(), + optional_tls=":tls" if self.tls_config is not None else "", ) ) # By default, the cluster only has one process @@ -190,11 +218,12 @@ logdir = {logdir} def create_cluster_file(self): with open(self.cluster_file, "x") as f: f.write( - "{desc}:{secret}@{ip_addr}:{server_port}".format( + "{desc}:{secret}@{ip_addr}:{server_port}{optional_tls}".format( desc=self.cluster_desc, secret=self.cluster_secret, ip_addr=self.ip_address, server_port=self.server_ports[0], + optional_tls=":tls" if self.tls_config is not None else "", ) ) self.coordinators = {0} @@ -248,6 +277,10 @@ logdir = {logdir} def __fdbcli_exec(self, cmd, stdout, stderr, timeout): args = [self.fdbcli_binary, "-C", self.cluster_file, "--exec", cmd] + if self.tls_config: + args += ["--tls-certificate-file", self.client_cert_file, + "--tls-key-file", self.client_key_file, + "--tls-ca-file", self.server_ca_file] res = subprocess.run(args, env=self.process_env(), stderr=stderr, stdout=stdout, timeout=timeout) assert res.returncode == 0, "fdbcli command {} failed with {}".format(cmd, res.returncode) return res.stdout @@ -271,6 +304,46 @@ logdir = {logdir} if self.blob_granules_enabled: self.fdbcli_exec("blobrange start \\x00 \\xff") + # Generate and install test certificate chains and keys + def create_tls_cert(self): + assert self.tls_config is not None, "TLS not enabled" + assert self.mkcert_binary.exists() and self.mkcert_binary.is_file(), "{} does not exist".format(self.mkcert_binary) + self.cert.mkdir(exist_ok=True) + server_chain_len = abs(self.tls_config.server_chain_len) + client_chain_len = abs(self.tls_config.client_chain_len) + expire_server_cert = (self.tls_config.server_chain_len < 0) + expire_client_cert = (self.tls_config.client_chain_len < 0) + args = [ + str(self.mkcert_binary), + "--server-chain-length", str(server_chain_len), + "--client-chain-length", str(client_chain_len), + "--server-cert-file", str(self.server_cert_file), + "--client-cert-file", str(self.client_cert_file), + "--server-key-file", str(self.server_key_file), + "--client-key-file", str(self.client_key_file), + "--server-ca-file", str(self.server_ca_file), + "--client-ca-file", str(self.client_ca_file), + "--print-args", + ] + if expire_server_cert: + args.append("--expire-server-cert") + if expire_client_cert: + args.append("--expire-client-cert") + subprocess.run(args, check=True) + + # Materialize server's TLS configuration section + def tls_conf_string(self): + if self.tls_config is None: + return "" + else: + conf_map = { + "tls-certificate-file": self.server_cert_file, + "tls-key-file": self.server_key_file, + "tls-ca-file": self.client_ca_file, + "tls-verify-peers": self.tls_config.verify_peers, + } + return "\n".join("{} = {}".format(k, v) for k, v in conf_map.items()) + # Get cluster status using fdbcli def get_status(self): status_output = self.fdbcli_exec_and_get("status json") diff --git a/tests/TestRunner/tmp_cluster.py b/tests/TestRunner/tmp_cluster.py index d341c759e9..99fa698ce2 100755 --- a/tests/TestRunner/tmp_cluster.py +++ b/tests/TestRunner/tmp_cluster.py @@ -5,25 +5,27 @@ import os import shutil import subprocess import sys -from local_cluster import LocalCluster, random_secret_string +from local_cluster import LocalCluster, TLSConfig, random_secret_string from argparse import ArgumentParser, RawDescriptionHelpFormatter from pathlib import Path -class TempCluster: +class TempCluster(LocalCluster): def __init__( self, build_dir: str, process_number: int = 1, port: str = None, blob_granules_enabled: bool = False, + tls_config: TLSConfig = None, ): self.build_dir = Path(build_dir).resolve() assert self.build_dir.exists(), "{} does not exist".format(build_dir) assert self.build_dir.is_dir(), "{} is not a directory".format(build_dir) tmp_dir = self.build_dir.joinpath("tmp", random_secret_string(16)) tmp_dir.mkdir(parents=True) - self.cluster = LocalCluster( + self.tmp_dir = tmp_dir + super().__init__( tmp_dir, self.build_dir.joinpath("bin", "fdbserver"), self.build_dir.joinpath("bin", "fdbmonitor"), @@ -31,23 +33,21 @@ class TempCluster: process_number, port=port, blob_granules_enabled=blob_granules_enabled, + tls_config=tls_config, + mkcert_binary=self.build_dir.joinpath("bin", "mkcert"), ) - self.log = self.cluster.log - self.etc = self.cluster.etc - self.data = self.cluster.data - self.tmp_dir = tmp_dir def __enter__(self): - self.cluster.__enter__() - self.cluster.create_database() + super().__enter__() + super().create_database() return self def __exit__(self, xc_type, exc_value, traceback): - self.cluster.__exit__(xc_type, exc_value, traceback) + super().__exit__(xc_type, exc_value, traceback) shutil.rmtree(self.tmp_dir) def close(self): - self.cluster.__exit__(None, None, None) + super().__exit__(None, None, None) shutil.rmtree(self.tmp_dir) @@ -94,12 +94,37 @@ if __name__ == "__main__": parser.add_argument( "--blob-granules-enabled", help="Enable blob granules", action="store_true" ) + parser.add_argument( + "--tls-enabled", help="Enable TLS (with test-only certificates)", action="store_true") + parser.add_argument( + "--server-cert-chain-len", + help="Length of server TLS certificate chain including root CA. Negative value deliberately generates expired leaf certificate for TLS testing. Only takes effect with --tls-enabled.", + type=int, + default=3, + ) + parser.add_argument( + "--client-cert-chain-len", + help="Length of client TLS certificate chain including root CA. Negative value deliberately generates expired leaf certificate for TLS testing. Only takes effect with --tls-enabled.", + type=int, + default=2, + ) + parser.add_argument( + "--tls-verify-peer", + help="Rules to verify client certificate chain. See https://apple.github.io/foundationdb/tls.html#peer-verification", + type=str, + default="Check.Valid=1", + ) args = parser.parse_args() + tls_config = None + if args.tls_enabled: + tls_config = TLSConfig(server_chain_len=args.server_cert_chain_len, + client_chain_len=args.client_cert_chain_len) errcode = 1 with TempCluster( args.build_dir, args.process_number, blob_granules_enabled=args.blob_granules_enabled, + tls_config=tls_config, ) as cluster: print("log-dir: {}".format(cluster.log)) print("etc-dir: {}".format(cluster.etc)) @@ -117,6 +142,18 @@ if __name__ == "__main__": cmd_args.append(str(cluster.etc)) elif cmd == "@TMP_DIR@": cmd_args.append(str(cluster.tmp_dir)) + elif cmd == "@SERVER_CERT_FILE@": + cmd_args.append(str(cluster.server_cert_file)) + elif cmd == "@SERVER_KEY_FILE@": + cmd_args.append(str(cluster.server_key_file)) + elif cmd == "@SERVER_CA_FILE@": + cmd_args.append(str(cluster.server_ca_file)) + elif cmd == "@CLIENT_CERT_FILE@": + cmd_args.append(str(cluster.client_cert_file)) + elif cmd == "@CLIENT_KEY_FILE@": + cmd_args.append(str(cluster.client_key_file)) + elif cmd == "@CLIENT_CA_FILE@": + cmd_args.append(str(cluster.client_ca_file)) elif cmd.startswith("@DATA_DIR@"): cmd_args.append(str(cluster.data) + cmd[len("@DATA_DIR@") :]) else: