mTLS test helpers (#7218)

* Add TLS option to (local_cluster|tmp_cluster).py

* Add TLS-enabled C API test

bindings/c/CMakeLists.txt

@@ -301,6 +301,31 @@ endif()
             @LOG_DIR@
             )
 
+  add_fdbclient_test(
+    NAME fdb_c_api_tests_with_tls
+    DISABLE_LOG_DUMP
+    TLS_ENABLED
+    COMMAND ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py
+            --cluster-file
+            @CLUSTER_FILE@
+            --tester-binary
+            $<TARGET_FILE:fdb_c_api_tester>
+            --external-client-library
+            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
+            --test-dir
+            ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
+            --tmp-dir
+            @TMP_DIR@
+            --log-dir
+            @LOG_DIR@
+            --tls-cert-file
+            @CLIENT_CERT_FILE@
+            --tls-key-file
+            @CLIENT_KEY_FILE@
+            --tls-ca-file
+            @SERVER_CA_FILE@
+            )
+
   if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT USE_SANITIZER)
     add_test(NAME fdb_c_upgrade_single_threaded_630api
       COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py

bindings/c/test/apitester/TesterOptions.h

@@ -52,6 +52,9 @@ public:
 	std::vector<std::pair<std::string, std::string>> knobs;
 	TestSpec testSpec;
 	std::string bgBasePath;
+	std::string tlsCertFile;
+	std::string tlsKeyFile;
+	std::string tlsCaFile;
 };
 
 } // namespace FdbApiTester

bindings/c/test/apitester/fdb_c_api_tester.cpp

@@ -54,7 +54,10 @@ enum TesterOptionId {
 	OPT_FDB_API_VERSION,
 	OPT_TRANSACTION_RETRY_LIMIT,
 	OPT_BLOB_GRANULE_LOCAL_FILE_PATH,
-	OPT_STATS_INTERVAL
+	OPT_STATS_INTERVAL,
+	OPT_TLS_CERT_FILE,
+	OPT_TLS_KEY_FILE,
+	OPT_TLS_CA_FILE,
 };
 
 CSimpleOpt::SOption TesterOptionDefs[] = //
@@ -79,6 +82,9 @@ CSimpleOpt::SOption TesterOptionDefs[] = //
 	  { OPT_TRANSACTION_RETRY_LIMIT, "--transaction-retry-limit", SO_REQ_SEP },
 	  { OPT_BLOB_GRANULE_LOCAL_FILE_PATH, "--blob-granule-local-file-path", SO_REQ_SEP },
 	  { OPT_STATS_INTERVAL, "--stats-interval", SO_REQ_SEP },
+	  { OPT_TLS_CERT_FILE, "--tls-cert-file", SO_REQ_SEP },
+	  { OPT_TLS_KEY_FILE, "--tls-key-file", SO_REQ_SEP },
+	  { OPT_TLS_CA_FILE, "--tls-ca-file", SO_REQ_SEP },
 	  SO_END_OF_OPTIONS };
 
 void printProgramUsage(const char* execName) {
@@ -122,6 +128,12 @@ void printProgramUsage(const char* execName) {
 	       "                 Test file to run.\n"
 	       "  --stats-interval MILLISECONDS\n"
 	       "                 Time interval in milliseconds for printing workload statistics (default: 0 - disabled).\n"
+	       "  --tls-cert-file FILE\n"
+	       "                 Path to file containing client's TLS certificate chain\n"
+	       "  --tls-key-file FILE\n"
+	       "                 Path to file containing client's TLS private key\n"
+	       "  --tls-ca-file FILE\n"
+	       "                 Path to file containing TLS CA certificate\n"
 	       "  -h, --help     Display this help and exit.\n",
 	       FDB_API_VERSION);
 }
@@ -221,6 +233,15 @@ bool processArg(TesterOptions& options, const CSimpleOpt& args) {
 	case OPT_STATS_INTERVAL:
 		processIntOption(args.OptionText(), args.OptionArg(), 0, 60000, options.statsIntervalMs);
 		break;
+	case OPT_TLS_CERT_FILE:
+		options.tlsCertFile.assign(args.OptionArg());
+		break;
+	case OPT_TLS_KEY_FILE:
+		options.tlsKeyFile.assign(args.OptionArg());
+		break;
+	case OPT_TLS_CA_FILE:
+		options.tlsCaFile.assign(args.OptionArg());
+		break;
 	}
 	return true;
 }
@@ -299,6 +320,18 @@ void applyNetworkOptions(TesterOptions& options) {
 		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_KNOB,
 		                            fmt::format("{}={}", knob.first.c_str(), knob.second.c_str())));
 	}
+
+	if (!options.tlsCertFile.empty()) {
+		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_TLS_CERT_PATH, options.tlsCertFile));
+	}
+
+	if (!options.tlsKeyFile.empty()) {
+		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_TLS_KEY_PATH, options.tlsKeyFile));
+	}
+
+	if (!options.tlsCaFile.empty()) {
+		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_TLS_CA_PATH, options.tlsCaFile));
+	}
 }
 
 void randomizeOptions(TesterOptions& options) {

bindings/c/test/apitester/run_c_api_tests.py

@@ -84,6 +84,15 @@ def run_tester(args, test_file):
         cmd += ["--blob-granule-local-file-path",
                 args.blob_granule_local_file_path]
 
+    if args.tls_ca_file is not None:
+        cmd += ["--tls-ca-file", args.tls_ca_file]
+
+    if args.tls_key_file is not None:
+        cmd += ["--tls-key-file", args.tls_key_file]
+
+    if args.tls_cert_file is not None:
+        cmd += ["--tls-cert-file", args.tls_cert_file]
+
     get_logger().info('\nRunning tester \'%s\'...' % ' '.join(cmd))
     proc = Popen(cmd, stdout=sys.stdout, stderr=sys.stderr)
     timed_out = False
@@ -149,6 +158,12 @@ def parse_args(argv):
                         help='The directory for storing temporary files (default: None)')
     parser.add_argument('--blob-granule-local-file-path', type=str, default=None,
                         help='Enable blob granule tests if set, value is path to local blob granule files')
+    parser.add_argument('--tls-ca-file', type=str, default=None,
+                        help='Path to client\'s TLS CA file: i.e. certificate of CA that signed the server certificate')
+    parser.add_argument('--tls-cert-file', type=str, default=None,
+                        help='Path to client\'s TLS certificate file')
+    parser.add_argument('--tls-key-file', type=str, default=None,
+                        help='Path to client\'s TLS private key file')
 
     return parser.parse_args(argv)
 

cmake/AddFdbTest.cmake

@@ -404,8 +404,7 @@ endfunction()
 
 # Creates a single cluster before running the specified command (usually a ctest test)
 function(add_fdbclient_test)
-  set(options DISABLED ENABLED DISABLE_LOG_DUMP)
+  set(options DISABLED ENABLED DISABLE_LOG_DUMP API_TEST_BLOB_GRANULES_ENABLED TLS_ENABLED)
-  set(options DISABLED ENABLED API_TEST_BLOB_GRANULES_ENABLED)
   set(oneValueArgs NAME PROCESS_NUMBER TEST_TIMEOUT WORKING_DIRECTORY)
   set(multiValueArgs COMMAND)
   cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
@@ -435,6 +434,9 @@ function(add_fdbclient_test)
   if(T_API_TEST_BLOB_GRANULES_ENABLED)
     list(APPEND TMP_CLUSTER_CMD --blob-granules-enabled)
   endif()
+  if(T_TLS_ENABLED)
+    list(APPEND TMP_CLUSTER_CMD --tls-enabled)
+  endif()
   message(STATUS "Adding Client test ${T_NAME}")
   add_test(NAME "${T_NAME}"
     WORKING_DIRECTORY ${T_WORKING_DIRECTORY}

flow/MkCertCli.cpp

@@ -179,9 +179,6 @@ mkcert::CertChainRef ChainSpec::makeChain(Arena& arena) {
 	ofsCert.write(reinterpret_cast<char const*>(cert.begin()), cert.size());
 	auto key = chain[0].privateKeyPem;
 	ofsKey.write(reinterpret_cast<char const*>(key.begin()), key.size());
-	ofsCert.close();
-	ofsKey.close();
-	ofsCa.close();
 	return chain;
 }
 

tests/TestRunner/local_cluster.py

@@ -39,6 +39,17 @@ def is_port_in_use(port):
 
 valid_letters_for_secret = string.ascii_letters + string.digits
 
+class TLSConfig:
+    # Passing a negative chain length generates expired leaf certificate
+    def __init__(
+        self,
+        server_chain_len: int = 3,
+        client_chain_len: int = 2,
+        verify_peers = "Check.Valid=1",
+    ):
+        self.server_chain_len = server_chain_len
+        self.client_chain_len = client_chain_len
+        self.verify_peers = verify_peers
 
 def random_secret_string(length):
     return "".join(random.choice(valid_letters_for_secret) for _ in range(length))
@@ -67,11 +78,12 @@ cluster-file = {etcdir}/fdb.cluster
 ## Default parameters for individual fdbserver processes
 [fdbserver]
 command = {fdbserver_bin}
-public-address = {ip_address}:$ID
+public-address = {ip_address}:$ID{optional_tls}
 listen-address = public
 datadir = {datadir}/$ID
 logdir = {logdir}
 {bg_knob_line}
+{tls_config}
 # logsize = 10MiB
 # maxlogssize = 100MiB
 # machine-id =
@@ -98,12 +110,15 @@ logdir = {logdir}
         port=None,
         ip_address=None,
         blob_granules_enabled: bool = False,
-        redundancy: str = "single"
+        redundancy: str = "single",
+        tls_config: TLSConfig = None,
+        mkcert_binary: str = "",
     ):
         self.basedir = Path(basedir)
         self.etc = self.basedir.joinpath("etc")
         self.log = self.basedir.joinpath("log")
         self.data = self.basedir.joinpath("data")
+        self.cert = self.basedir.joinpath("cert")
         self.conf_file = self.etc.joinpath("foundationdb.conf")
         self.cluster_file = self.etc.joinpath("fdb.cluster")
         self.fdbserver_binary = Path(fdbserver_binary)
@@ -137,11 +152,22 @@ logdir = {logdir}
         self.use_legacy_conf_syntax = False
         self.coordinators = set()
         self.active_servers = set(self.server_ports.keys())
+        self.tls_config = tls_config
+        self.mkcert_binary = Path(mkcert_binary)
+        self.server_cert_file = self.cert.joinpath("server_cert.pem")
+        self.client_cert_file = self.cert.joinpath("client_cert.pem")
+        self.server_key_file = self.cert.joinpath("server_key.pem")
+        self.client_key_file = self.cert.joinpath("client_key.pem")
+        self.server_ca_file = self.cert.joinpath("server_ca.pem")
+        self.client_ca_file = self.cert.joinpath("client_ca.pem")
 
         if create_config:
             self.create_cluster_file()
             self.save_config()
 
+        if self.tls_config is not None:
+            self.create_tls_cert()
+
     def __next_port(self):
         if self.first_port is None:
             return get_free_port()
@@ -166,6 +192,8 @@ logdir = {logdir}
                     logdir=self.log,
                     ip_address=self.ip_address,
                     bg_knob_line=bg_knob_line,
+                    tls_config=self.tls_conf_string(),
+                    optional_tls=":tls" if self.tls_config is not None else "",
                 )
             )
             # By default, the cluster only has one process
@@ -190,11 +218,12 @@ logdir = {logdir}
     def create_cluster_file(self):
         with open(self.cluster_file, "x") as f:
             f.write(
-                "{desc}:{secret}@{ip_addr}:{server_port}".format(
+                "{desc}:{secret}@{ip_addr}:{server_port}{optional_tls}".format(
                     desc=self.cluster_desc,
                     secret=self.cluster_secret,
                     ip_addr=self.ip_address,
                     server_port=self.server_ports[0],
+                    optional_tls=":tls" if self.tls_config is not None else "",
                 )
             )
         self.coordinators = {0}
@@ -248,6 +277,10 @@ logdir = {logdir}
 
     def __fdbcli_exec(self, cmd, stdout, stderr, timeout):
         args = [self.fdbcli_binary, "-C", self.cluster_file, "--exec", cmd]
+        if self.tls_config:
+            args += ["--tls-certificate-file", self.client_cert_file,
+                     "--tls-key-file", self.client_key_file,
+                     "--tls-ca-file", self.server_ca_file]
         res = subprocess.run(args, env=self.process_env(), stderr=stderr, stdout=stdout, timeout=timeout)
         assert res.returncode == 0, "fdbcli command {} failed with {}".format(cmd, res.returncode)
         return res.stdout
@@ -271,6 +304,46 @@ logdir = {logdir}
         if self.blob_granules_enabled:
             self.fdbcli_exec("blobrange start \\x00 \\xff")
 
+    # Generate and install test certificate chains and keys
+    def create_tls_cert(self):
+        assert self.tls_config is not None, "TLS not enabled"
+        assert self.mkcert_binary.exists() and self.mkcert_binary.is_file(), "{} does not exist".format(self.mkcert_binary)
+        self.cert.mkdir(exist_ok=True)
+        server_chain_len = abs(self.tls_config.server_chain_len)
+        client_chain_len = abs(self.tls_config.client_chain_len)
+        expire_server_cert = (self.tls_config.server_chain_len < 0)
+        expire_client_cert = (self.tls_config.client_chain_len < 0)
+        args = [
+            str(self.mkcert_binary),
+            "--server-chain-length", str(server_chain_len),
+            "--client-chain-length", str(client_chain_len),
+            "--server-cert-file", str(self.server_cert_file),
+            "--client-cert-file", str(self.client_cert_file),
+            "--server-key-file", str(self.server_key_file),
+            "--client-key-file", str(self.client_key_file),
+            "--server-ca-file", str(self.server_ca_file),
+            "--client-ca-file", str(self.client_ca_file),
+            "--print-args",
+        ]
+        if expire_server_cert:
+            args.append("--expire-server-cert")
+        if expire_client_cert:
+            args.append("--expire-client-cert")
+        subprocess.run(args, check=True)
+
+    # Materialize server's TLS configuration section
+    def tls_conf_string(self):
+        if self.tls_config is None:
+            return ""
+        else:
+            conf_map = {
+                "tls-certificate-file": self.server_cert_file,
+                "tls-key-file": self.server_key_file,
+                "tls-ca-file": self.client_ca_file,
+                "tls-verify-peers": self.tls_config.verify_peers,
+            }
+            return "\n".join("{} = {}".format(k, v) for k, v in conf_map.items())
+
     # Get cluster status using fdbcli
     def get_status(self):
         status_output = self.fdbcli_exec_and_get("status json")

tests/TestRunner/tmp_cluster.py

@@ -5,25 +5,27 @@ import os
 import shutil
 import subprocess
 import sys
-from local_cluster import LocalCluster, random_secret_string
+from local_cluster import LocalCluster, TLSConfig, random_secret_string
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
 from pathlib import Path
 
 
-class TempCluster:
+class TempCluster(LocalCluster):
     def __init__(
         self,
         build_dir: str,
         process_number: int = 1,
         port: str = None,
         blob_granules_enabled: bool = False,
+        tls_config: TLSConfig = None,
     ):
         self.build_dir = Path(build_dir).resolve()
         assert self.build_dir.exists(), "{} does not exist".format(build_dir)
         assert self.build_dir.is_dir(), "{} is not a directory".format(build_dir)
         tmp_dir = self.build_dir.joinpath("tmp", random_secret_string(16))
         tmp_dir.mkdir(parents=True)
-        self.cluster = LocalCluster(
+        self.tmp_dir = tmp_dir
+        super().__init__(
             tmp_dir,
             self.build_dir.joinpath("bin", "fdbserver"),
             self.build_dir.joinpath("bin", "fdbmonitor"),
@@ -31,23 +33,21 @@ class TempCluster:
             process_number,
             port=port,
             blob_granules_enabled=blob_granules_enabled,
+            tls_config=tls_config,
+            mkcert_binary=self.build_dir.joinpath("bin", "mkcert"),
         )
-        self.log = self.cluster.log
-        self.etc = self.cluster.etc
-        self.data = self.cluster.data
-        self.tmp_dir = tmp_dir
 
     def __enter__(self):
-        self.cluster.__enter__()
+        super().__enter__()
-        self.cluster.create_database()
+        super().create_database()
         return self
 
     def __exit__(self, xc_type, exc_value, traceback):
-        self.cluster.__exit__(xc_type, exc_value, traceback)
+        super().__exit__(xc_type, exc_value, traceback)
         shutil.rmtree(self.tmp_dir)
 
     def close(self):
-        self.cluster.__exit__(None, None, None)
+        super().__exit__(None, None, None)
         shutil.rmtree(self.tmp_dir)
 
 
@@ -94,12 +94,37 @@ if __name__ == "__main__":
     parser.add_argument(
         "--blob-granules-enabled", help="Enable blob granules", action="store_true"
     )
+    parser.add_argument(
+        "--tls-enabled", help="Enable TLS (with test-only certificates)", action="store_true")
+    parser.add_argument(
+        "--server-cert-chain-len",
+        help="Length of server TLS certificate chain including root CA. Negative value deliberately generates expired leaf certificate for TLS testing. Only takes effect with --tls-enabled.",
+        type=int,
+        default=3,
+    )
+    parser.add_argument(
+        "--client-cert-chain-len",
+        help="Length of client TLS certificate chain including root CA. Negative value deliberately generates expired leaf certificate for TLS testing. Only takes effect with --tls-enabled.",
+        type=int,
+        default=2,
+    )
+    parser.add_argument(
+        "--tls-verify-peer",
+        help="Rules to verify client certificate chain. See https://apple.github.io/foundationdb/tls.html#peer-verification",
+        type=str,
+        default="Check.Valid=1",
+    )
     args = parser.parse_args()
+    tls_config = None
+    if args.tls_enabled:
+        tls_config = TLSConfig(server_chain_len=args.server_cert_chain_len,
+                               client_chain_len=args.client_cert_chain_len)
     errcode = 1
     with TempCluster(
         args.build_dir,
         args.process_number,
         blob_granules_enabled=args.blob_granules_enabled,
+        tls_config=tls_config,
     ) as cluster:
         print("log-dir: {}".format(cluster.log))
         print("etc-dir: {}".format(cluster.etc))
@@ -117,6 +142,18 @@ if __name__ == "__main__":
                 cmd_args.append(str(cluster.etc))
             elif cmd == "@TMP_DIR@":
                 cmd_args.append(str(cluster.tmp_dir))
+            elif cmd == "@SERVER_CERT_FILE@":
+                cmd_args.append(str(cluster.server_cert_file))
+            elif cmd == "@SERVER_KEY_FILE@":
+                cmd_args.append(str(cluster.server_key_file))
+            elif cmd == "@SERVER_CA_FILE@":
+                cmd_args.append(str(cluster.server_ca_file))
+            elif cmd == "@CLIENT_CERT_FILE@":
+                cmd_args.append(str(cluster.client_cert_file))
+            elif cmd == "@CLIENT_KEY_FILE@":
+                cmd_args.append(str(cluster.client_key_file))
+            elif cmd == "@CLIENT_CA_FILE@":
+                cmd_args.append(str(cluster.client_ca_file))
             elif cmd.startswith("@DATA_DIR@"):
                 cmd_args.append(str(cluster.data) + cmd[len("@DATA_DIR@") :])
             else: