adding encryption at rest support to local cluster and api tester (#9325)

* adding encryption at rest support to local cluster and api tester

* adding encrypted variant of bg tenant test and renaming ear

bindings/c/test/apitester/run_c_api_tests.py

@@ -164,6 +164,9 @@ class TestConfig:
         server_config = config.get("server", [{}])[0]
         self.tenants_enabled = server_config.get("tenants_enabled", True)
         self.blob_granules_enabled = server_config.get("blob_granules_enabled", False)
+        self.enable_encryption_at_rest = server_config.get(
+            "enable_encryption_at_rest", False
+        )
         self.tls_enabled = server_config.get("tls_enabled", False)
         self.client_chain_len = server_config.get("tls_client_chain_len", 2)
         self.server_chain_len = server_config.get("tls_server_chain_len", 3)
@@ -189,6 +192,7 @@ def run_test(args, test_file):
         config.num_processes,
         enable_tenants=config.tenants_enabled,
         blob_granules_enabled=config.blob_granules_enabled,
+        enable_encryption_at_rest=config.enable_encryption_at_rest,
         tls_config=tls_config,
     ) as cluster:
         ret_code = run_tester(args, cluster, test_file)

bindings/c/test/apitester/tests/CApiBlobGranuleCorrectnessMultiThr.toml

@@ -13,6 +13,7 @@ maxClients = 8
 
 [[server]]
 blob_granules_enabled = true
+# FIXME: enable EAR once other PRs merged
 
 [[test.workload]]
 name = 'ApiBlobGranuleCorrectness'

bindings/c/test/apitester/tests/CApiBlobGranuleTenantEncryptionCorrectnessMultiThr.toml

@@ -0,0 +1,25 @@
+[[test]]
+title = 'Blob Granule API Tenant Correctness Multi Threaded'
+multiThreaded = true
+buggify = true
+minFdbThreads = 2
+maxFdbThreads = 8
+minClients = 1
+maxClients = 8
+minTenants = 1
+maxTenants = 5
+
+[[server]]
+blob_granules_enabled = true
+enable_encryption_at_rest = true
+
+[[test.workload]]
+name = 'ApiBlobGranuleCorrectness'
+minKeyLength = 1
+maxKeyLength = 64
+minValueLength = 1
+maxValueLength = 1000
+maxKeysPerTransaction = 50
+# TODO - increase initialSize and/or buggify down BG_SNAPSHOT_FILE_TARGET_BYTES to force multiple granules
+initialSize = 100
+numRandomOperations = 100

tests/TestRunner/local_cluster.py

@@ -112,6 +112,8 @@ listen-address = public
 datadir = {datadir}/$ID
 logdir = {logdir}
 {bg_knob_line}
+{encrypt_knob_line1}
+{encrypt_knob_line2}
 {tls_config}
 {authz_public_key_config}
 {custom_config}
@@ -142,6 +144,7 @@ logdir = {logdir}
         port=None,
         ip_address=None,
         blob_granules_enabled: bool = False,
+        enable_encryption_at_rest: bool = False,
         use_future_protocol_version: bool = False,
         redundancy: str = "single",
         tls_config: TLSConfig = None,
@@ -172,6 +175,7 @@ logdir = {logdir}
         self.first_port = port
         self.custom_config = custom_config
         self.blob_granules_enabled = blob_granules_enabled
+        self.enable_encryption_at_rest = enable_encryption_at_rest
         if blob_granules_enabled:
             # add extra process for blob_worker
             self.process_number += 1
@@ -250,10 +254,15 @@ logdir = {logdir}
         with open(new_conf_file, "x") as f:
             conf_template = LocalCluster.configuration_template
             bg_knob_line = ""
+            encrypt_knob_line1 = ""
+            encrypt_knob_line2 = ""
             if self.use_legacy_conf_syntax:
                 conf_template = conf_template.replace("-", "_")
             if self.blob_granules_enabled:
                 bg_knob_line = "knob_bg_url=file://" + str(self.data) + "/fdbblob/"
+            if self.enable_encryption_at_rest:
+                encrypt_knob_line1 = "knob_enable_encryption=true"
+                encrypt_knob_line2 = "knob_kms_connector_type=FDBPerfKmsConnector"
             f.write(
                 conf_template.format(
                     etcdir=self.etc,
@@ -262,6 +271,8 @@ logdir = {logdir}
                     logdir=self.log,
                     ip_address=self.ip_address,
                     bg_knob_line=bg_knob_line,
+                    encrypt_knob_line1=encrypt_knob_line1,
+                    encrypt_knob_line2=encrypt_knob_line2,
                     tls_config=self.tls_conf_string(),
                     authz_public_key_config=self.authz_public_key_conf_string(),
                     optional_tls=":tls" if self.tls_config is not None else "",
@@ -397,9 +408,15 @@ logdir = {logdir}
         return self.__fdbcli_exec(cmd, subprocess.PIPE, None, timeout)
 
     def create_database(self, storage="ssd", enable_tenants=True):
+        if self.enable_encryption_at_rest:
+            # only redwood supports EAR
+            storage = "ssd-redwood-1-experimental"
         db_config = "configure new {} {}".format(self.redundancy, storage)
         if enable_tenants:
             db_config += " tenant_mode=optional_experimental"
+        if self.enable_encryption_at_rest:
+            # FIXME: could support domain_aware if tenants are required
+            db_config += " encryption_at_rest_mode=cluster_aware"
         if self.blob_granules_enabled:
             db_config += " blob_granules_enabled:=1"
         self.fdbcli_exec(db_config)

tests/TestRunner/tmp_cluster.py

@@ -25,6 +25,7 @@ class TempCluster(LocalCluster):
         remove_at_exit: bool = True,
         custom_config: dict = {},
         enable_tenants: bool = True,
+        enable_encryption_at_rest: bool = False,
     ):
         self.build_dir = Path(build_dir).resolve()
         assert self.build_dir.exists(), "{} does not exist".format(build_dir)
@@ -34,6 +35,7 @@ class TempCluster(LocalCluster):
         self.tmp_dir = tmp_dir
         self.remove_at_exit = remove_at_exit
         self.enable_tenants = enable_tenants
+        self.enable_encryption_at_rest = enable_encryption_at_rest
         super().__init__(
             tmp_dir,
             self.build_dir.joinpath("bin", "fdbserver"),
@@ -42,6 +44,7 @@ class TempCluster(LocalCluster):
             process_number,
             port=port,
             blob_granules_enabled=blob_granules_enabled,
+            enable_encryption_at_rest=enable_encryption_at_rest,
             tls_config=tls_config,
             mkcert_binary=self.build_dir.joinpath("bin", "mkcert"),
             authorization_kty=authorization_kty,