Merge branch 'main' into feature-metacluster

bindings/c/CMakeLists.txt

@@ -428,10 +428,18 @@ endif()
 # Generate shim library in Linux builds
 if (OPEN_FOR_IDE)
 
-  add_library(fdb_c_shim OBJECT fdb_c_shim.cpp)
+  add_library(fdb_c_shim OBJECT foundationdb/fdb_c_shim.h fdb_c_shim.cpp)
   target_link_libraries(fdb_c_shim PUBLIC dl)
+  target_include_directories(fdb_c_shim PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/foundationdb>)
 
-elseif(NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE AND NOT USE_UBSAN) # Linux, non-ubsan only
+  add_library(fdb_c_shim_lib_tester OBJECT test/shim_lib_tester.cpp)
+  target_link_libraries(fdb_c_shim_lib_tester PRIVATE fdb_c_shim SimpleOpt fdb_cpp Threads::Threads)
+  target_include_directories(fdb_c_shim_lib_tester PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/ ${CMAKE_SOURCE_DIR}/flow/include)
+
+elseif(NOT WIN32 AND NOT APPLE AND NOT USE_UBSAN) # Linux Only, non-ubsan only
 
   set(SHIM_LIB_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
 
@@ -439,16 +447,31 @@ elseif(NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE AND NOT USE_UBSAN) # Linux,
     ${SHIM_LIB_OUTPUT_DIR}/libfdb_c.so.init.c
     ${SHIM_LIB_OUTPUT_DIR}/libfdb_c.so.tramp.S)
 
+  set(IMPLIBSO_SRC_DIR ${CMAKE_SOURCE_DIR}/contrib/Implib.so)
+  set(IMPLIBSO_SRC
+    ${IMPLIBSO_SRC_DIR}/implib-gen.py
+    ${IMPLIBSO_SRC_DIR}/arch/common/init.c.tpl
+    ${IMPLIBSO_SRC_DIR}/arch/${CMAKE_SYSTEM_PROCESSOR}/config.ini
+    ${IMPLIBSO_SRC_DIR}/arch/${CMAKE_SYSTEM_PROCESSOR}/table.S.tpl
+    ${IMPLIBSO_SRC_DIR}/arch/${CMAKE_SYSTEM_PROCESSOR}/trampoline.S.tpl
+    )
+
   add_custom_command(OUTPUT ${SHIM_LIB_GEN_SRC}
-  COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_SOURCE_DIR}/contrib/Implib.so/implib-gen.py
+    COMMAND $<TARGET_FILE:Python::Interpreter> ${IMPLIBSO_SRC_DIR}/implib-gen.py
       --target ${CMAKE_SYSTEM_PROCESSOR}
       --outdir ${SHIM_LIB_OUTPUT_DIR}
       --dlopen-callback=fdb_shim_dlopen_callback
-      $<TARGET_FILE:fdb_c>)
+      $<TARGET_FILE:fdb_c>
+    DEPENDS ${IMPLIBSO_SRC}
+    COMMENT "Generating source code for C shim library")
 
-  add_library(fdb_c_shim SHARED ${SHIM_LIB_GEN_SRC} fdb_c_shim.cpp)
+  add_library(fdb_c_shim SHARED ${SHIM_LIB_GEN_SRC} foundationdb/fdb_c_shim.h fdb_c_shim.cpp)
   target_link_options(fdb_c_shim PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.map,-z,nodelete,-z,noexecstack")
   target_link_libraries(fdb_c_shim PUBLIC dl)
+  target_include_directories(fdb_c_shim PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/foundationdb>)
 
   add_executable(fdb_c_shim_unit_tests)
   target_link_libraries(fdb_c_shim_unit_tests PRIVATE fdb_c_shim fdb_c_unit_tests_impl)
@@ -456,15 +479,20 @@ elseif(NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE AND NOT USE_UBSAN) # Linux,
   add_executable(fdb_c_shim_api_tester)
   target_link_libraries(fdb_c_shim_api_tester PRIVATE fdb_c_shim fdb_c_api_tester_impl)
 
+  add_executable(fdb_c_shim_lib_tester test/shim_lib_tester.cpp)
+  target_link_libraries(fdb_c_shim_lib_tester PRIVATE fdb_c_shim SimpleOpt fdb_cpp Threads::Threads)
+  target_include_directories(fdb_c_shim_lib_tester PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/ ${CMAKE_SOURCE_DIR}/flow/include)
+
   add_test(NAME fdb_c_shim_library_tests
     COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/test/fdb_c_shim_tests.py
           --build-dir ${CMAKE_BINARY_DIR}
           --unit-tests-bin $<TARGET_FILE:fdb_c_shim_unit_tests>
           --api-tester-bin $<TARGET_FILE:fdb_c_shim_api_tester>
+          --shim-lib-tester-bin $<TARGET_FILE:fdb_c_shim_lib_tester>
           --api-test-dir ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
           )
 
-endif() # End Linux, non-ubsan only
+endif() # End Linux only, non-ubsan only
 
 # TODO: re-enable once the old vcxproj-based build system is removed.
 #generate_export_header(fdb_c EXPORT_MACRO_NAME "DLLEXPORT"
@@ -508,3 +536,19 @@ fdb_install(
   DESTINATION lib
   DESTINATION_SUFFIX "/cmake/${targets_export_name}"
   COMPONENT clients)
+
+if(NOT WIN32 AND NOT APPLE AND NOT USE_UBSAN) # Linux Only, non-ubsan only
+
+fdb_install(
+  FILES foundationdb/fdb_c_shim.h
+  DESTINATION include
+  DESTINATION_SUFFIX /foundationdb
+  COMPONENT clients)
+
+fdb_install(
+  TARGETS fdb_c_shim
+  EXPORT ${targets_export_name}
+  DESTINATION lib
+  COMPONENT clients)
+
+endif() # End Linux only, non-ubsan only

bindings/c/fdb_c_shim.cpp

@@ -20,25 +20,42 @@
 
 #if (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
 
+#define DLLEXPORT __attribute__((visibility("default")))
+
+#include "foundationdb/fdb_c_shim.h"
+
 #include <dlfcn.h>
-#include <stdio.h>
-#include <stdlib.h>
 #include <string>
 
-static const char* FDB_C_CLIENT_LIBRARY_PATH = "FDB_C_CLIENT_LIBRARY_PATH";
+namespace {
 
-// Callback that tries different library names
+const char* FDB_LOCAL_CLIENT_LIBRARY_PATH_ENVVAR = "FDB_LOCAL_CLIENT_LIBRARY_PATH";
+std::string g_fdbLocalClientLibraryPath;
+
+} // namespace
+
+extern "C" DLLEXPORT void fdb_shim_set_local_client_library_path(const char* filePath) {
+	g_fdbLocalClientLibraryPath = filePath;
+}
+
+/* The callback of the fdb_c_shim layer that determines the path
+   of the fdb_c library to be dynamically loaded
+ */
 extern "C" void* fdb_shim_dlopen_callback(const char* libName) {
 	std::string libPath;
-	char* val = getenv(FDB_C_CLIENT_LIBRARY_PATH);
+	if (!g_fdbLocalClientLibraryPath.empty()) {
-	if (val) {
+		libPath = g_fdbLocalClientLibraryPath;
-		libPath = val;
 	} else {
-		libPath = libName;
+		char* val = getenv(FDB_LOCAL_CLIENT_LIBRARY_PATH_ENVVAR);
+		if (val) {
+			libPath = val;
+		} else {
+			libPath = libName;
+		}
 	}
 	return dlopen(libPath.c_str(), RTLD_LAZY | RTLD_GLOBAL);
 }
 
 #else
 #error Port me!
-#endif
+#endif

bindings/c/foundationdb/fdb_c.h

@@ -70,6 +70,15 @@ DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_error_t code);
 
 #define /* fdb_error_t */ fdb_select_api_version(v) fdb_select_api_version_impl(v, FDB_API_VERSION)
 
+/*
+ * A variant of fdb_select_api_version that caps the header API version by the maximum API version
+ * supported by the client library. It is intended mainly for use in combination with the shim
+ * layer, which loads the client library dynamically.
+ */
+#define /* fdb_error_t */ fdb_select_api_version_capped(v)                                                             \
+	fdb_select_api_version_impl(                                                                                       \
+	    v, FDB_API_VERSION < fdb_get_max_api_version() ? FDB_API_VERSION : fdb_get_max_api_version())
+
 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_network_set_option(FDBNetworkOption option,
                                                                 uint8_t const* value,
                                                                 int value_length);

bindings/c/foundationdb/fdb_c_shim.h

@@ -0,0 +1,47 @@
+/*
+ * fdb_shim_c.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDB_SHIM_C_H
+#define FDB_SHIM_C_H
+#pragma once
+
+#ifndef DLLEXPORT
+#define DLLEXPORT
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Specify the path of the local libfdb_c.so library to be dynamically loaded by the shim layer
+ *
+ * This enables running the same application code with different client library versions,
+ * e.g. using the latest development build for testing new features, but still using the latest
+ * stable release in production deployments.
+ *
+ * The given path overrides the environment variable FDB_LOCAL_CLIENT_LIBRARY_PATH
+ */
+DLLEXPORT void fdb_shim_set_local_client_library_path(const char* filePath);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

bindings/c/test/apitester/fdb_c_api_tester.cpp

@@ -278,7 +278,7 @@ void fdb_check(fdb::Error e) {
 }
 
 void applyNetworkOptions(TesterOptions& options) {
-	if (!options.tmpDir.empty()) {
+	if (!options.tmpDir.empty() && options.apiVersion >= 720) {
 		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_CLIENT_TMP_DIR, options.tmpDir);
 	}
 	if (!options.externalClientLibrary.empty()) {
@@ -419,7 +419,7 @@ int main(int argc, char** argv) {
 		}
 		randomizeOptions(options);
 
-		fdb::selectApiVersion(options.apiVersion);
+		fdb::selectApiVersionCapped(options.apiVersion);
 		applyNetworkOptions(options);
 		fdb::network::setup();
 

bindings/c/test/fdb_api.hpp

@@ -114,7 +114,7 @@ public:
 
 	explicit Error(CodeType err) noexcept : err(err) {}
 
-	char const* what() noexcept { return native::fdb_get_error(err); }
+	char const* what() const noexcept { return native::fdb_get_error(err); }
 
 	explicit operator bool() const noexcept { return err != 0; }
 
@@ -722,6 +722,20 @@ inline void selectApiVersion(int version) {
 	}
 }
 
+inline Error selectApiVersionCappedNothrow(int version) {
+	if (version < 720) {
+		Tenant::tenantManagementMapPrefix = "\xff\xff/management/tenant_map/";
+	}
+	return Error(
+	    native::fdb_select_api_version_impl(version, std::min(native::fdb_get_max_api_version(), FDB_API_VERSION)));
+}
+
+inline void selectApiVersionCapped(int version) {
+	if (auto err = selectApiVersionCappedNothrow(version)) {
+		throwError(fmt::format("ERROR: fdb_select_api_version_capped({}): ", version), err);
+	}
+}
+
 } // namespace fdb
 
 template <>

bindings/c/test/fdb_c_shim_tests.py

@@ -7,13 +7,18 @@ import subprocess
 import sys
 import os
 
-sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..', '..', '..', 'tests', 'TestRunner')]
+sys.path[:0] = [os.path.join(os.path.dirname(
+    __file__), '..', '..', '..', 'tests', 'TestRunner')]
+
+# fmt: off
 from binary_download import FdbBinaryDownloader, CURRENT_VERSION
 from local_cluster import LocalCluster, random_secret_string
+# fmt: on
 
 LAST_RELEASE_VERSION = "7.1.5"
 TESTER_STATS_INTERVAL_SEC = 5
 DEFAULT_TEST_FILE = "CApiCorrectnessMultiThr.toml"
+IMPLIBSO_ERROR_CODE = -6  # SIGABORT
 
 
 def version_from_str(ver_str):
@@ -55,7 +60,8 @@ class TestEnv(LocalCluster):
         self.set_env_var("LD_LIBRARY_PATH", self.downloader.lib_dir(version))
         client_lib = self.downloader.lib_path(version)
         assert client_lib.exists(), "{} does not exist".format(client_lib)
-        self.client_lib_external = self.tmp_dir.joinpath("libfdb_c_external.so")
+        self.client_lib_external = self.tmp_dir.joinpath(
+            "libfdb_c_external.so")
         shutil.copyfile(client_lib, self.client_lib_external)
 
     def __enter__(self):
@@ -91,6 +97,9 @@ class FdbCShimTests:
         assert self.unit_tests_bin.exists(), "{} does not exist".format(self.unit_tests_bin)
         self.api_tester_bin = Path(args.api_tester_bin).resolve()
         assert self.api_tester_bin.exists(), "{} does not exist".format(self.api_tests_bin)
+        self.shim_lib_tester_bin = Path(args.shim_lib_tester_bin).resolve()
+        assert self.shim_lib_tester_bin.exists(
+        ), "{} does not exist".format(self.shim_lib_tester_bin)
         self.api_test_dir = Path(args.api_test_dir).resolve()
         assert self.api_test_dir.exists(), "{} does not exist".format(self.api_test_dir)
         self.downloader = FdbBinaryDownloader(args.build_dir)
@@ -98,6 +107,7 @@ class FdbCShimTests:
         self.platform = platform.machine()
         if (self.platform == "x86_64"):
             self.downloader.download_old_binaries(LAST_RELEASE_VERSION)
+            self.downloader.download_old_binaries("7.0.0")
 
     def build_c_api_tester_args(self, test_env, test_file):
         test_file_path = self.api_test_dir.joinpath(test_file)
@@ -128,7 +138,8 @@ class FdbCShimTests:
         with TestEnv(self.build_dir, self.downloader, version) as test_env:
             cmd_args = self.build_c_api_tester_args(test_env, test_file)
             env_vars = os.environ.copy()
-            env_vars["LD_LIBRARY_PATH"] = self.downloader.lib_dir(version)
+            env_vars["FDB_LOCAL_CLIENT_LIBRARY_PATH"] = self.downloader.lib_path(
+                version)
             test_env.exec_client_command(cmd_args, env_vars)
 
     def run_c_unit_tests(self, version):
@@ -143,38 +154,118 @@ class FdbCShimTests:
                 test_env.client_lib_external
             ]
             env_vars = os.environ.copy()
-            env_vars["LD_LIBRARY_PATH"] = self.downloader.lib_dir(version)
+            env_vars["FDB_LOCAL_CLIENT_LIBRARY_PATH"] = self.downloader.lib_path(
+                version)
             test_env.exec_client_command(cmd_args, env_vars)
 
-    def test_invalid_c_client_lib_env_var(self, version):
+    def run_c_shim_lib_tester(
+        self,
+        version,
+        test_env,
+        api_version=None,
+        invalid_lib_path=False,
+        call_set_path=False,
+        set_env_path=False,
+        set_ld_lib_path=False,
+        use_external_lib=True,
+        expected_ret_code=0
+    ):
         print('-' * 80)
-        print("Test invalid FDB_C_CLIENT_LIBRARY_PATH value")
+        if api_version is None:
+            api_version = api_version_from_str(version)
+        test_flags = []
+        if invalid_lib_path:
+            test_flags.append("invalid_lib_path")
+        if call_set_path:
+            test_flags.append("call_set_path")
+        if set_ld_lib_path:
+            test_flags.append("set_ld_lib_path")
+        if use_external_lib:
+            test_flags.append("use_external_lib")
+        else:
+            test_flags.append("use_local_lib")
+        print("C Shim Tests - version: {}, API version: {}, {}".format(version,
+              api_version, ", ".join(test_flags)))
         print('-' * 80)
-        with TestEnv(self.build_dir, self.downloader, version) as test_env:
+        cmd_args = [
-            cmd_args = self.build_c_api_tester_args(test_env, DEFAULT_TEST_FILE)
+            self.shim_lib_tester_bin,
-            env_vars = os.environ.copy()
+            "--cluster-file",
-            env_vars["FDB_C_CLIENT_LIBRARY_PATH"] = "dummy"
+            test_env.cluster_file,
-            test_env.exec_client_command(cmd_args, env_vars, 1)
+            "--api-version",
-
+            str(api_version),
-    def test_valid_c_client_lib_env_var(self, version):
+        ]
-        print('-' * 80)
+        if call_set_path:
-        print("Test valid FDB_C_CLIENT_LIBRARY_PATH value")
+            cmd_args = cmd_args + [
-        print('-' * 80)
+                "--local-client-library",
-        with TestEnv(self.build_dir, self.downloader, version) as test_env:
+                ("dummy" if invalid_lib_path else self.downloader.lib_path(version))
-            cmd_args = self.build_c_api_tester_args(test_env, DEFAULT_TEST_FILE)
+            ]
-            env_vars = os.environ.copy()
+        if use_external_lib:
-            env_vars["FDB_C_CLIENT_LIBRARY_PATH"] = self.downloader.lib_path(version)
+            cmd_args = cmd_args + [
-            test_env.exec_client_command(cmd_args, env_vars)
+                "--disable-local-client",
+                "--external-client-library",
+                test_env.client_lib_external
+            ]
+        env_vars = os.environ.copy()
+        env_vars["LD_LIBRARY_PATH"] = (
+            self.downloader.lib_dir(version) if set_ld_lib_path else "")
+        if set_env_path:
+            env_vars["FDB_LOCAL_CLIENT_LIBRARY_PATH"] = (
+                "dummy" if invalid_lib_path else self.downloader.lib_path(version))
+        test_env.exec_client_command(cmd_args, env_vars, expected_ret_code)
 
     def run_tests(self):
+        # Test the API workload with the dev version
+        self.run_c_api_test(CURRENT_VERSION, DEFAULT_TEST_FILE)
+
+        # Run unit tests with the dev version
+        self.run_c_unit_tests(CURRENT_VERSION)
+
+        with TestEnv(self.build_dir, self.downloader, CURRENT_VERSION) as test_env:
+            # Test lookup of the client library over LD_LIBRARY_PATH
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, set_ld_lib_path=True)
+
+            # Test setting the client library path over an API call
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, call_set_path=True)
+
+            # Test setting the client library path over an environment variable
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, set_env_path=True)
+
+            # Test using the loaded client library as the local client
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, call_set_path=True, use_external_lib=False)
+
+            # Test setting an invalid client library path over an API call
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, call_set_path=True, invalid_lib_path=True, expected_ret_code=IMPLIBSO_ERROR_CODE)
+
+            # Test setting an invalid client library path over an environment variable
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, set_env_path=True, invalid_lib_path=True, expected_ret_code=IMPLIBSO_ERROR_CODE)
+
+            # Test calling a function that exists in the loaded library, but not for the selected API version
+            self.run_c_shim_lib_tester(
+                CURRENT_VERSION, test_env, call_set_path=True, api_version=700)
+
         # binary downloads are currently available only for x86_64
-        if (self.platform == "x86_64"):
+        if self.platform == "x86_64":
+            # Test the API workload with the release version
             self.run_c_api_test(LAST_RELEASE_VERSION, DEFAULT_TEST_FILE)
 
-        self.run_c_api_test(CURRENT_VERSION, DEFAULT_TEST_FILE)
+            with TestEnv(self.build_dir, self.downloader, LAST_RELEASE_VERSION) as test_env:
-        self.run_c_unit_tests(CURRENT_VERSION)
+                # Test using the loaded client library as the local client
-        self.test_invalid_c_client_lib_env_var(CURRENT_VERSION)
+                self.run_c_shim_lib_tester(
-        self.test_valid_c_client_lib_env_var(CURRENT_VERSION)
+                    LAST_RELEASE_VERSION, test_env, call_set_path=True, use_external_lib=False)
+
+                # Test the client library of the release version in combination with the dev API version
+                self.run_c_shim_lib_tester(
+                    LAST_RELEASE_VERSION, test_env, call_set_path=True, api_version=api_version_from_str(CURRENT_VERSION), expected_ret_code=1)
+
+                # Test calling a function that does not exist in the loaded library
+                self.run_c_shim_lib_tester(
+                    "7.0.0", test_env, call_set_path=True, api_version=700, expected_ret_code=IMPLIBSO_ERROR_CODE)
 
 
 if __name__ == "__main__":
@@ -194,12 +285,26 @@ if __name__ == "__main__":
         help="FDB build directory",
         required=True,
     )
-    parser.add_argument('--unit-tests-bin', type=str,
+    parser.add_argument(
-                        help='Path to the fdb_c_shim_unit_tests executable.')
+        '--unit-tests-bin',
-    parser.add_argument('--api-tester-bin', type=str,
+        type=str,
-                        help='Path to the fdb_c_shim_api_tester executable.')
+        help='Path to the fdb_c_shim_unit_tests executable.',
-    parser.add_argument('--api-test-dir', type=str,
+        required=True)
-                        help='Path to a directory with api test definitions.')
+    parser.add_argument(
+        '--api-tester-bin',
+        type=str,
+        help='Path to the fdb_c_shim_api_tester executable.',
+        required=True)
+    parser.add_argument(
+        '--shim-lib-tester-bin',
+        type=str,
+        help='Path to the fdb_c_shim_lib_tester executable.',
+        required=True)
+    parser.add_argument(
+        '--api-test-dir',
+        type=str,
+        help='Path to a directory with api test definitions.',
+        required=True)
     args = parser.parse_args()
     test = FdbCShimTests(args)
     test.run_tests()

bindings/c/test/shim_lib_tester.cpp

@@ -0,0 +1,253 @@
+/*
+ * shim_lib_tester.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * A utility for testing shim library usage with various valid and invalid configurations
+ */
+
+#include "fmt/core.h"
+#include "test/fdb_api.hpp"
+#include "SimpleOpt/SimpleOpt.h"
+#include <thread>
+#include <string_view>
+#include "foundationdb/fdb_c_shim.h"
+
+#undef ERROR
+#define ERROR(name, number, description) enum { error_code_##name = number };
+
+#include "flow/error_definitions.h"
+
+using namespace std::string_view_literals;
+
+namespace {
+
+enum TesterOptionId {
+	OPT_HELP,
+	OPT_CONNFILE,
+	OPT_LOCAL_CLIENT_LIBRARY,
+	OPT_EXTERNAL_CLIENT_LIBRARY,
+	OPT_EXTERNAL_CLIENT_DIRECTORY,
+	OPT_DISABLE_LOCAL_CLIENT,
+	OPT_API_VERSION
+};
+
+const int MIN_TESTABLE_API_VERSION = 400;
+
+CSimpleOpt::SOption TesterOptionDefs[] = //
+    { { OPT_HELP, "-h", SO_NONE },
+	  { OPT_HELP, "--help", SO_NONE },
+	  { OPT_CONNFILE, "-C", SO_REQ_SEP },
+	  { OPT_CONNFILE, "--cluster-file", SO_REQ_SEP },
+	  { OPT_LOCAL_CLIENT_LIBRARY, "--local-client-library", SO_REQ_SEP },
+	  { OPT_EXTERNAL_CLIENT_LIBRARY, "--external-client-library", SO_REQ_SEP },
+	  { OPT_EXTERNAL_CLIENT_DIRECTORY, "--external-client-dir", SO_REQ_SEP },
+	  { OPT_DISABLE_LOCAL_CLIENT, "--disable-local-client", SO_NONE },
+	  { OPT_API_VERSION, "--api-version", SO_REQ_SEP },
+	  SO_END_OF_OPTIONS };
+
+class TesterOptions {
+public:
+	// FDB API version, using the latest version by default
+	int apiVersion = FDB_API_VERSION;
+	std::string clusterFile;
+	std::string localClientLibrary;
+	std::string externalClientLibrary;
+	std::string externalClientDir;
+	bool disableLocalClient = false;
+};
+
+void printProgramUsage(const char* execName) {
+	printf("usage: %s [OPTIONS]\n"
+	       "\n",
+	       execName);
+	printf("  -C, --cluster-file FILE\n"
+	       "                 The path of a file containing the connection string for the\n"
+	       "                 FoundationDB cluster. The default is `fdb.cluster'\n"
+	       "  --local-client-library FILE\n"
+	       "                 Path to the local client library.\n"
+	       "  --external-client-library FILE\n"
+	       "                 Path to the external client library.\n"
+	       "  --external-client-dir DIR\n"
+	       "                 Directory containing external client libraries.\n"
+	       "  --disable-local-client DIR\n"
+	       "                 Disable the local client, i.e. use only external client libraries.\n"
+	       "  --api-version VERSION\n"
+	       "                 Required FDB API version (default %d).\n"
+	       "  -h, --help     Display this help and exit.\n",
+	       FDB_API_VERSION);
+}
+
+bool processIntOption(const std::string& optionName, const std::string& value, int minValue, int maxValue, int& res) {
+	char* endptr;
+	res = strtol(value.c_str(), &endptr, 10);
+	if (*endptr != '\0') {
+		fmt::print(stderr, "Invalid value {} for {}", value, optionName);
+		return false;
+	}
+	if (res < minValue || res > maxValue) {
+		fmt::print(stderr, "Value for {} must be between {} and {}", optionName, minValue, maxValue);
+		return false;
+	}
+	return true;
+}
+
+bool processArg(TesterOptions& options, const CSimpleOpt& args) {
+	switch (args.OptionId()) {
+	case OPT_CONNFILE:
+		options.clusterFile = args.OptionArg();
+		break;
+	case OPT_LOCAL_CLIENT_LIBRARY:
+		options.localClientLibrary = args.OptionArg();
+		break;
+	case OPT_EXTERNAL_CLIENT_LIBRARY:
+		options.externalClientLibrary = args.OptionArg();
+		break;
+	case OPT_EXTERNAL_CLIENT_DIRECTORY:
+		options.externalClientDir = args.OptionArg();
+		break;
+	case OPT_DISABLE_LOCAL_CLIENT:
+		options.disableLocalClient = true;
+		break;
+	case OPT_API_VERSION:
+		if (!processIntOption(
+		        args.OptionText(), args.OptionArg(), MIN_TESTABLE_API_VERSION, FDB_API_VERSION, options.apiVersion)) {
+			return false;
+		}
+		break;
+	}
+	return true;
+}
+
+bool parseArgs(TesterOptions& options, int argc, char** argv) {
+	// declare our options parser, pass in the arguments from main
+	// as well as our array of valid options.
+	CSimpleOpt args(argc, argv, TesterOptionDefs);
+
+	// while there are arguments left to process
+	while (args.Next()) {
+		if (args.LastError() == SO_SUCCESS) {
+			if (args.OptionId() == OPT_HELP) {
+				printProgramUsage(argv[0]);
+				return false;
+			}
+			if (!processArg(options, args)) {
+				return false;
+			}
+		} else {
+			fmt::print(stderr, "ERROR: Invalid argument: {}\n", args.OptionText());
+			printProgramUsage(argv[0]);
+			return false;
+		}
+	}
+	return true;
+}
+
+void fdb_check(fdb::Error e, std::string_view msg, fdb::Error::CodeType expectedError = error_code_success) {
+	if (e.code()) {
+		fmt::print(stderr, "{}, Error: {}({})\n", msg, e.code(), e.what());
+		std::abort();
+	}
+}
+
+void applyNetworkOptions(TesterOptions& options) {
+	if (!options.externalClientLibrary.empty()) {
+		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT);
+		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_LIBRARY,
+		                        options.externalClientLibrary);
+	} else if (!options.externalClientDir.empty()) {
+		if (options.disableLocalClient) {
+			fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT);
+		}
+		fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_EXTERNAL_CLIENT_DIRECTORY, options.externalClientDir);
+	} else {
+		if (options.disableLocalClient) {
+			fmt::print(stderr, "Invalid options: Cannot disable local client if no external library is provided");
+			exit(1);
+		}
+	}
+}
+
+void testBasicApi(const TesterOptions& options) {
+	fdb::Database db(options.clusterFile);
+	fdb::Transaction tx = db.createTransaction();
+	while (true) {
+		try {
+			// Set a time out to avoid long delays when testing invalid configurations
+			tx.setOption(FDB_TR_OPTION_TIMEOUT, 1000);
+			tx.set(fdb::toBytesRef("key1"sv), fdb::toBytesRef("val1"sv));
+			fdb_check(tx.commit().blockUntilReady(), "Wait on commit failed");
+			break;
+		} catch (const fdb::Error& err) {
+			if (err.code() == error_code_timed_out) {
+				exit(1);
+			}
+			auto onErrorFuture = tx.onError(err);
+			fdb_check(onErrorFuture.blockUntilReady(), "Wait on onError failed");
+			fdb_check(onErrorFuture.error(), "onError failed");
+		}
+	}
+}
+
+void test710Api(const TesterOptions& options) {
+	fdb::Database db(options.clusterFile);
+	try {
+		db.openTenant(fdb::toBytesRef("not_existing_tenant"sv));
+	} catch (const fdb::Error& err) {
+		fdb_check(err, "Tenant not found expected", error_code_tenant_not_found);
+	}
+}
+
+} // namespace
+
+int main(int argc, char** argv) {
+	int retCode = 0;
+	try {
+		TesterOptions options;
+		if (!parseArgs(options, argc, argv)) {
+			return 1;
+		}
+
+		if (!options.localClientLibrary.empty()) {
+			// Must be called before the first FDB API call
+			fdb_shim_set_local_client_library_path(options.localClientLibrary.c_str());
+		}
+
+		fdb::selectApiVersionCapped(options.apiVersion);
+		applyNetworkOptions(options);
+		fdb::network::setup();
+
+		std::thread network_thread{ &fdb::network::run };
+
+		// Try calling some basic functionality that is available
+		// in all recent API versions
+		testBasicApi(options);
+
+		// Try calling 710-specific API. This enables testing what
+		// happens if a library is missing a function
+		test710Api(options);
+
+		fdb_check(fdb::network::stop(), "Stop network failed");
+		network_thread.join();
+	} catch (const std::runtime_error& err) {
+		fmt::print(stderr, "runtime error caught: {}\n", err.what());
+		retCode = 1;
+	}
+	return retCode;
+}

bindings/python/tests/fdbcli_tests.py

@@ -628,8 +628,9 @@ def tenants(logger):
     assert(len(json_output) == 2)
     assert('tenant' in json_output)
     assert(json_output['type'] == 'success')
-    assert(len(json_output['tenant']) == 3)
+    assert(len(json_output['tenant']) == 4)
     assert('id' in json_output['tenant'])
+    assert('encrypted' in json_output['tenant'])
     assert('prefix' in json_output['tenant'])
     assert(len(json_output['tenant']['prefix']) == 2)
     assert('base64' in json_output['tenant']['prefix'])
@@ -649,8 +650,9 @@ def tenants(logger):
     assert(len(json_output) == 2)
     assert('tenant' in json_output)
     assert(json_output['type'] == 'success')
-    assert(len(json_output['tenant']) == 4)
+    assert(len(json_output['tenant']) == 5)
     assert('id' in json_output['tenant'])
+    assert('encrypted' in json_output['tenant'])
     assert('prefix' in json_output['tenant'])
     assert(json_output['tenant']['tenant_state'] == 'ready')
     assert('tenant_group' in json_output['tenant'])

contrib/Implib.so/arch/common/init.c.tpl

@@ -26,8 +26,7 @@ extern "C" {
 #define CHECK(cond, fmt, ...) do { \
     if(!(cond)) { \
       fprintf(stderr, "implib-gen: $load_name: " fmt "\n", ##__VA_ARGS__); \
-      assert(0 && "Assertion in generated code"); \
+      abort(); \
-      exit(1); \
     } \
   } while(0)
 

fdbcli/BlobRangeCommand.actor.cpp

@@ -58,6 +58,69 @@ ACTOR Future<Void> setBlobRange(Database db, Key startKey, Key endKey, Value val
 	}
 }
 
+ACTOR Future<Version> getLatestReadVersion(Database db) {
+	state Transaction tr(db);
+	loop {
+		try {
+			Version rv = wait(tr.getReadVersion());
+			fmt::print("Resolved latest read version as {0}\n", rv);
+			return rv;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+}
+
+// print after delay if not cancelled
+ACTOR Future<Void> printAfterDelay(double delaySeconds, std::string message) {
+	wait(delay(delaySeconds));
+	fmt::print("{}\n", message);
+	return Void();
+}
+
+ACTOR Future<Void> doBlobPurge(Database db, Key startKey, Key endKey, Optional<Version> version) {
+	state Version purgeVersion;
+	if (version.present()) {
+		purgeVersion = version.get();
+	} else {
+		wait(store(purgeVersion, getLatestReadVersion(db)));
+	}
+
+	state Key purgeKey = wait(db->purgeBlobGranules(KeyRange(KeyRangeRef(startKey, endKey)), purgeVersion, {}));
+
+	fmt::print("Blob purge registered for [{0} - {1}) @ {2}\n", startKey.printable(), endKey.printable(), purgeVersion);
+
+	state Future<Void> printWarningActor = printAfterDelay(
+	    5.0, "Waiting for purge to complete. (interrupting this wait with CTRL+C will not cancel the purge)");
+	wait(db->waitPurgeGranulesComplete(purgeKey));
+
+	fmt::print("Blob purge complete for [{0} - {1}) @ {2}\n", startKey.printable(), endKey.printable(), purgeVersion);
+
+	return Void();
+}
+
+ACTOR Future<Void> doBlobCheck(Database db, Key startKey, Key endKey, Optional<Version> version) {
+	state Transaction tr(db);
+	state Version readVersionOut = invalidVersion;
+	state double elapsed = -timer_monotonic();
+	loop {
+		try {
+			wait(success(tr.readBlobGranules(KeyRange(KeyRangeRef(startKey, endKey)), 0, version, &readVersionOut)));
+			elapsed += timer_monotonic();
+			break;
+		} catch (Error& e) {
+			wait(tr.onError(e));
+		}
+	}
+
+	fmt::print("Blob check complete for [{0} - {1}) @ {2} in {3:.6f} seconds\n",
+	           startKey.printable(),
+	           endKey.printable(),
+	           readVersionOut,
+	           elapsed);
+	return Void();
+}
+
 } // namespace
 
 namespace fdb_cli {
@@ -66,7 +129,7 @@ ACTOR Future<bool> blobRangeCommandActor(Database localDb,
                                          Optional<TenantMapEntry> tenantEntry,
                                          std::vector<StringRef> tokens) {
 	// enables blob writing for the given range
-	if (tokens.size() != 4) {
+	if (tokens.size() != 4 && tokens.size() != 5) {
 		printUsage(tokens[0]);
 		return false;
 	}
@@ -84,29 +147,60 @@ ACTOR Future<bool> blobRangeCommandActor(Database localDb,
 
 	if (end > LiteralStringRef("\xff")) {
 		// TODO is this something we want?
-		printf("Cannot blobbify system keyspace! Problematic End Key: %s\n", tokens[3].printable().c_str());
+		fmt::print("Cannot blobbify system keyspace! Problematic End Key: {0}\n", tokens[3].printable());
 		return false;
 	} else if (tokens[2] >= tokens[3]) {
-		printf("Invalid blob range [%s - %s)\n", tokens[2].printable().c_str(), tokens[3].printable().c_str());
+		fmt::print("Invalid blob range [{0} - {1})\n", tokens[2].printable(), tokens[3].printable());
 	} else {
-		if (tokencmp(tokens[1], "start")) {
+		if (tokencmp(tokens[1], "start") || tokencmp(tokens[1], "stop")) {
-			printf("Starting blobbify range for [%s - %s)\n",
+			bool starting = tokencmp(tokens[1], "start");
-			       tokens[2].printable().c_str(),
+			if (tokens.size() > 4) {
-			       tokens[3].printable().c_str());
+				printUsage(tokens[0]);
-			wait(setBlobRange(localDb, begin, end, LiteralStringRef("1")));
+				return false;
-		} else if (tokencmp(tokens[1], "stop")) {
+			}
-			printf("Stopping blobbify range for [%s - %s)\n",
+			fmt::print("{0} blobbify range for [{1} - {2})\n",
-			       tokens[2].printable().c_str(),
+			           starting ? "Starting" : "Stopping",
-			       tokens[3].printable().c_str());
+			           tokens[2].printable().c_str(),
-			wait(setBlobRange(localDb, begin, end, StringRef()));
+			           tokens[3].printable().c_str());
+			wait(setBlobRange(localDb, begin, end, starting ? LiteralStringRef("1") : StringRef()));
+		} else if (tokencmp(tokens[1], "purge") || tokencmp(tokens[1], "check")) {
+			bool purge = tokencmp(tokens[1], "purge");
+
+			Optional<Version> version;
+			if (tokens.size() > 4) {
+				Version v;
+				int n = 0;
+				if (sscanf(tokens[4].toString().c_str(), "%" PRId64 "%n", &v, &n) != 1 || n != tokens[4].size()) {
+					printUsage(tokens[0]);
+					return false;
+				}
+				version = v;
+			}
+
+			fmt::print("{0} blob range [{1} - {2})",
+			           purge ? "Purging" : "Checking",
+			           tokens[2].printable(),
+			           tokens[3].printable());
+			if (version.present()) {
+				fmt::print(" @ {0}", version.get());
+			}
+			fmt::print("\n");
+
+			if (purge) {
+				wait(doBlobPurge(localDb, begin, end, version));
+			} else {
+				wait(doBlobCheck(localDb, begin, end, version));
+			}
 		} else {
 			printUsage(tokens[0]);
-			printf("Usage: blobrange <start|stop> <startkey> <endkey>");
 			return false;
 		}
 	}
 	return true;
 }
 
-CommandFactory blobRangeFactory("blobrange", CommandHelp("blobrange <start|stop> <startkey> <endkey>", "", ""));
+CommandFactory blobRangeFactory("blobrange",
+                                CommandHelp("blobrange <start|stop|purge|check> <startkey> <endkey> [version]",
+                                            "",
+                                            ""));
 } // namespace fdb_cli

fdbclient/ClientKnobs.cpp

@@ -82,7 +82,7 @@ void ClientKnobs::initialize(Randomize randomize) {
 	init( METADATA_VERSION_CACHE_SIZE,            1000 );
 	init( CHANGE_FEED_LOCATION_LIMIT,            10000 );
 	init( CHANGE_FEED_CACHE_SIZE,               100000 ); if( randomize && BUGGIFY ) CHANGE_FEED_CACHE_SIZE = 1;
-	init( CHANGE_FEED_POP_TIMEOUT,                 5.0 );
+	init( CHANGE_FEED_POP_TIMEOUT,                10.0 );
 	init( CHANGE_FEED_STREAM_MIN_BYTES,            1e4 ); if( randomize && BUGGIFY ) CHANGE_FEED_STREAM_MIN_BYTES = 1;
 
 	init( MAX_BATCH_SIZE,                         1000 ); if( randomize && BUGGIFY ) MAX_BATCH_SIZE = 1;

fdbclient/ManagementAPI.actor.cpp

@@ -804,6 +804,8 @@ ACTOR Future<Optional<ClusterConnectionString>> getConnectionString(Database cx)
 	}
 }
 
+static std::vector<std::string> connectionStrings;
+
 namespace {
 
 ACTOR Future<Optional<ClusterConnectionString>> getClusterConnectionStringFromStorageServer(Transaction* tr) {
@@ -821,6 +823,19 @@ ACTOR Future<Optional<ClusterConnectionString>> getClusterConnectionStringFromSt
 
 		Version readVersion = wait(tr->getReadVersion());
 		state Optional<Value> currentKey = wait(tr->get(coordinatorsKey));
+		if (g_network->isSimulated() && currentKey.present()) {
+			// If the change coordinators request succeeded, the coordinators
+			// should have changed to the connection string of the most
+			// recently issued request. If instead the connection string is
+			// equal to one of the previously issued requests, there is a bug
+			// and we are breaking the promises we make with
+			// commit_unknown_result (the transaction must no longer be in
+			// progress when receiving this error).
+			int n = connectionStrings.size() > 0 ? connectionStrings.size() - 1 : 0; // avoid underflow
+			for (int i = 0; i < n; ++i) {
+				ASSERT(currentKey.get() != connectionStrings.at(i));
+			}
+		}
 
 		if (!currentKey.present()) {
 			// Someone deleted this key entirely?
@@ -879,10 +894,12 @@ ACTOR Future<Optional<CoordinatorsResult>> changeQuorumChecker(Transaction* tr,
 	std::sort(old.hostnames.begin(), old.hostnames.end());
 	std::sort(old.coords.begin(), old.coords.end());
 	if (conn->hostnames == old.hostnames && conn->coords == old.coords && old.clusterKeyName() == newName) {
+		connectionStrings.clear();
 		return CoordinatorsResult::SAME_NETWORK_ADDRESSES;
 	}
 
 	conn->parseKey(newName + ':' + deterministicRandom()->randomAlphaNumeric(32));
+	connectionStrings.push_back(conn->toString());
 
 	if (g_network->isSimulated()) {
 		int i = 0;

fdbclient/Tenant.cpp

@@ -71,11 +71,15 @@ TenantState TenantMapEntry::stringToTenantState(std::string stateStr) {
 }
 
 TenantMapEntry::TenantMapEntry() {}
-TenantMapEntry::TenantMapEntry(int64_t id, TenantState tenantState) : tenantState(tenantState) {
+TenantMapEntry::TenantMapEntry(int64_t id, TenantState tenantState, bool encrypted)
+  : tenantState(tenantState), encrypted(encrypted) {
 	setId(id);
 }
-TenantMapEntry::TenantMapEntry(int64_t id, TenantState tenantState, Optional<TenantGroupName> tenantGroup)
+TenantMapEntry::TenantMapEntry(int64_t id,
-  : tenantState(tenantState), tenantGroup(tenantGroup) {
+                               TenantState tenantState,
+                               Optional<TenantGroupName> tenantGroup,
+                               bool encrypted)
+  : tenantState(tenantState), tenantGroup(tenantGroup), encrypted(encrypted) {
 	setId(id);
 }
 
@@ -88,6 +92,7 @@ void TenantMapEntry::setId(int64_t id) {
 std::string TenantMapEntry::toJson(int apiVersion) const {
 	json_spirit::mObject tenantEntry;
 	tenantEntry["id"] = id;
+	tenantEntry["encrypted"] = encrypted;
 
 	if (apiVersion >= 720 || apiVersion == Database::API_VERSION_LATEST) {
 		json_spirit::mObject prefixObject;
@@ -135,12 +140,12 @@ void TenantMapEntry::configure(Standalone<StringRef> parameter, Optional<Value>
 }
 
 TEST_CASE("/fdbclient/TenantMapEntry/Serialization") {
-	TenantMapEntry entry1(1, TenantState::READY);
+	TenantMapEntry entry1(1, TenantState::READY, false);
 	ASSERT(entry1.prefix == "\x00\x00\x00\x00\x00\x00\x00\x01"_sr);
 	TenantMapEntry entry2 = TenantMapEntry::decode(entry1.encode());
 	ASSERT(entry1.id == entry2.id && entry1.prefix == entry2.prefix);
 
-	TenantMapEntry entry3(std::numeric_limits<int64_t>::max(), TenantState::READY);
+	TenantMapEntry entry3(std::numeric_limits<int64_t>::max(), TenantState::READY, false);
 	ASSERT(entry3.prefix == "\x7f\xff\xff\xff\xff\xff\xff\xff"_sr);
 	TenantMapEntry entry4 = TenantMapEntry::decode(entry3.encode());
 	ASSERT(entry3.id == entry4.id && entry3.prefix == entry4.prefix);
@@ -151,7 +156,7 @@ TEST_CASE("/fdbclient/TenantMapEntry/Serialization") {
 		int64_t maxPlusOne = std::min<uint64_t>(UINT64_C(1) << bits, std::numeric_limits<int64_t>::max());
 		int64_t id = deterministicRandom()->randomInt64(min, maxPlusOne);
 
-		TenantMapEntry entry(id, TenantState::READY);
+		TenantMapEntry entry(id, TenantState::READY, false);
 		int64_t bigEndianId = bigEndian64(id);
 		ASSERT(entry.id == id && entry.prefix == StringRef(reinterpret_cast<uint8_t*>(&bigEndianId), 8));
 

fdbclient/include/fdbclient/BlobWorkerCommon.h

@@ -59,7 +59,7 @@ struct BlobWorkerStats {
 	    bytesReadFromFDBForInitialSnapshot("BytesReadFromFDBForInitialSnapshot", cc),
 	    bytesReadFromS3ForCompaction("BytesReadFromS3ForCompaction", cc),
 	    rangeAssignmentRequests("RangeAssignmentRequests", cc), readRequests("ReadRequests", cc),
-	    wrongShardServer("WrongShardServer", cc), changeFeedInputBytes("RangeFeedInputBytes", cc),
+	    wrongShardServer("WrongShardServer", cc), changeFeedInputBytes("ChangeFeedInputBytes", cc),
 	    readReqTotalFilesReturned("ReadReqTotalFilesReturned", cc),
 	    readReqDeltaBytesReturned("ReadReqDeltaBytesReturned", cc), commitVersionChecks("CommitVersionChecks", cc),
 	    granuleUpdateErrors("GranuleUpdateErrors", cc), granuleRequestTimeouts("GranuleRequestTimeouts", cc),

fdbclient/include/fdbclient/CommitProxyInterface.h

@@ -117,6 +117,7 @@ struct ClientDBInfo {
 	Optional<Value> forward;
 	std::vector<VersionHistory> history;
 	UID clusterId;
+	bool isEncryptionEnabled = false;
 
 	TenantMode tenantMode;
 	ClusterType clusterType = ClusterType::STANDALONE;
@@ -132,8 +133,17 @@ struct ClientDBInfo {
 		if constexpr (!is_fb_function<Archive>) {
 			ASSERT(ar.protocolVersion().isValid());
 		}
-		serializer(
+		serializer(ar,
-		    ar, grvProxies, commitProxies, id, forward, history, tenantMode, clusterId, clusterType, metaclusterName);
+		           grvProxies,
+		           commitProxies,
+		           id,
+		           forward,
+		           history,
+		           tenantMode,
+		           isEncryptionEnabled,
+		           clusterId,
+		           clusterType,
+		           metaclusterName);
 	}
 };
 

fdbclient/include/fdbclient/Tenant.h

@@ -66,14 +66,15 @@ struct TenantMapEntry {
 	Key prefix;
 	TenantState tenantState = TenantState::READY;
 	Optional<TenantGroupName> tenantGroup;
+	bool encrypted = false;
 	Optional<ClusterName> assignedCluster;
 	int64_t configurationSequenceNum = 0;
 
 	constexpr static int PREFIX_SIZE = sizeof(id);
 
 	TenantMapEntry();
-	TenantMapEntry(int64_t id, TenantState tenantState);
+	TenantMapEntry(int64_t id, TenantState tenantState, bool encrypted);
-	TenantMapEntry(int64_t id, TenantState tenantState, Optional<TenantGroupName> tenantGroup);
+	TenantMapEntry(int64_t id, TenantState tenantState, Optional<TenantGroupName> tenantGroup, bool encrypted);
 
 	void setId(int64_t id);
 	std::string toJson(int apiVersion) const;
@@ -88,7 +89,7 @@ struct TenantMapEntry {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, id, tenantState, tenantGroup, assignedCluster, configurationSequenceNum);
+		serializer(ar, id, tenantState, tenantGroup, encrypted, assignedCluster, configurationSequenceNum);
 		if constexpr (Ar::isDeserializing) {
 			if (id >= 0) {
 				prefix = idToPrefix(id);

fdbclient/include/fdbclient/TenantSpecialKeys.actor.h

@@ -116,6 +116,7 @@ private:
 	    std::map<TenantGroupName, int>* tenantGroupNetTenantDelta) {
 		state TenantMapEntry tenantEntry;
 		tenantEntry.setId(tenantId);
+		tenantEntry.encrypted = ryw->getTransactionState()->cx->clientInfo->get().isEncryptionEnabled;
 
 		for (auto const& [name, value] : configMutations) {
 			tenantEntry.configure(name, value);

fdbserver/ClusterController.actor.cpp

@@ -249,6 +249,7 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster,
 			dbInfo.latencyBandConfig = db->serverInfo->get().latencyBandConfig;
 			dbInfo.myLocality = db->serverInfo->get().myLocality;
 			dbInfo.client = ClientDBInfo();
+			dbInfo.client.isEncryptionEnabled = SERVER_KNOBS->ENABLE_ENCRYPTION;
 			dbInfo.client.tenantMode = TenantAPI::tenantModeForClusterType(db->clusterType, db->config.tenantMode);
 			dbInfo.client.clusterId = db->serverInfo->get().client.clusterId;
 			dbInfo.client.clusterType = db->clusterType;
@@ -1015,6 +1016,7 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co
 	if (db->clientInfo->get().commitProxies != req.commitProxies ||
 	    db->clientInfo->get().grvProxies != req.grvProxies ||
 	    db->clientInfo->get().tenantMode != db->config.tenantMode || db->clientInfo->get().clusterId != req.clusterId ||
+	    db->clientInfo->get().isEncryptionEnabled != SERVER_KNOBS->ENABLE_ENCRYPTION ||
 	    db->clientInfo->get().clusterType != db->clusterType ||
 	    db->clientInfo->get().metaclusterName != db->metaclusterName) {
 		TraceEvent("PublishNewClientInfo", self->id)
@@ -1027,6 +1029,7 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co
 		    .detail("ReqTenantMode", db->config.tenantMode.toString())
 		    .detail("ClusterId", db->clientInfo->get().clusterId)
 		    .detail("ReqClusterId", req.clusterId)
+		    .detail("EncryptionEnabled", SERVER_KNOBS->ENABLE_ENCRYPTION)
 		    .detail("ClusterType", db->clientInfo->get().clusterType)
 		    .detail("ReqClusterType", db->clusterType)
 		    .detail("MetaclusterName", db->clientInfo->get().metaclusterName)
@@ -1035,6 +1038,7 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co
 		// TODO why construct a new one and not just copy the old one and change proxies + id?
 		ClientDBInfo clientInfo;
 		clientInfo.id = deterministicRandom()->randomUniqueID();
+		clientInfo.isEncryptionEnabled = SERVER_KNOBS->ENABLE_ENCRYPTION;
 		clientInfo.commitProxies = req.commitProxies;
 		clientInfo.grvProxies = req.grvProxies;
 		clientInfo.tenantMode = TenantAPI::tenantModeForClusterType(db->clusterType, db->config.tenantMode);

fdbserver/ClusterRecovery.actor.cpp

@@ -522,6 +522,19 @@ ACTOR Future<Void> changeCoordinators(Reference<ClusterRecoveryData> self) {
 		TraceEvent("ChangeCoordinators", self->dbgid).log();
 		++self->changeCoordinatorsRequests;
 		state ChangeCoordinatorsRequest changeCoordinatorsRequest = req;
+		if (self->masterInterface.id() != changeCoordinatorsRequest.masterId) {
+			// Make sure the request is coming from a proxy from the same
+			// generation. If not, throw coordinators_changed - this is OK
+			// because the client will still receive commit_unknown_result, and
+			// will retry the request. This check is necessary because
+			// otherwise in rare circumstances where a recovery occurs between
+			// the change coordinators request from the client and the cstate
+			// actually being moved, the client may think the change
+			// coordinators command failed when it is still in progress. So we
+			// preempt the issue here and force failure if the generations
+			// don't match.
+			throw coordinators_changed();
+		}
 
 		// Kill cluster controller to facilitate coordinator registration update
 		if (self->controllerData->shouldCommitSuicide) {

fdbserver/CommitProxyServer.actor.cpp

@@ -1139,7 +1139,8 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 	if (!self->isMyFirstBatch &&
 	    pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get() != self->oldCoordinators.get()) {
 		wait(brokenPromiseToNever(pProxyCommitData->db->get().clusterInterface.changeCoordinators.getReply(
-		    ChangeCoordinatorsRequest(pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get()))));
+		    ChangeCoordinatorsRequest(pProxyCommitData->txnStateStore->readValue(coordinatorsKey).get().get(),
+		                              self->pProxyCommitData->master.id()))));
 		ASSERT(false); // ChangeCoordinatorsRequest should always throw
 	}
 

fdbserver/Coordination.actor.cpp

@@ -393,6 +393,7 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
 				notify[i].send(newInfo);
 			notify.clear();
 			ClientDBInfo outInfo;
+			outInfo.isEncryptionEnabled = SERVER_KNOBS->ENABLE_ENCRYPTION;
 			outInfo.id = deterministicRandom()->randomUniqueID();
 			outInfo.forward = req.conn.toString();
 			clientData.clientInfo->set(CachedSerialization<ClientDBInfo>(outInfo));
@@ -632,6 +633,7 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 			Optional<LeaderInfo> forward = regs.getForward(req.clusterKey);
 			if (forward.present()) {
 				ClientDBInfo info;
+				info.isEncryptionEnabled = SERVER_KNOBS->ENABLE_ENCRYPTION;
 				info.id = deterministicRandom()->randomUniqueID();
 				info.forward = forward.get().serializedInfo;
 				req.reply.send(CachedSerialization<ClientDBInfo>(info));

fdbserver/DataDistributionQueue.actor.cpp

@@ -1964,33 +1964,19 @@ ACTOR Future<Void> BgDDLoadRebalance(DDQueueData* self, int teamCollectionIndex,
 			traceEvent.detail("QueuedRelocations", self->priority_relocations[ddPriority]);
 
 			if (self->priority_relocations[ddPriority] < SERVER_KNOBS->DD_REBALANCE_PARALLELISM) {
-				if (isDataMovementForMountainChopper(reason)) {
+				bool mcMove = isDataMovementForMountainChopper(reason);
-					srcReq = GetTeamRequest(WantNewServers::True,
+				srcReq = GetTeamRequest(WantNewServers::True,
-					                        WantTrueBest::True,
+				                        WantTrueBest(mcMove),
-					                        PreferLowerDiskUtil::False,
+				                        PreferLowerDiskUtil::False,
-					                        TeamMustHaveShards::True,
+				                        TeamMustHaveShards::True,
-					                        ForReadBalance(readRebalance),
+				                        ForReadBalance(readRebalance),
-					                        PreferLowerReadUtil::False);
+				                        PreferLowerReadUtil::False);
-					destReq = GetTeamRequest(WantNewServers::True,
+				destReq = GetTeamRequest(WantNewServers::True,
-					                         WantTrueBest::False,
+				                         WantTrueBest(!mcMove),
-					                         PreferLowerDiskUtil::True,
+				                         PreferLowerDiskUtil::True,
-					                         TeamMustHaveShards::False,
+				                         TeamMustHaveShards::False,
-					                         ForReadBalance(readRebalance),
+				                         ForReadBalance(readRebalance),
-					                         PreferLowerReadUtil::True);
+				                         PreferLowerReadUtil::True);
-				} else {
-					srcReq = GetTeamRequest(WantNewServers::True,
-					                        WantTrueBest::False,
-					                        PreferLowerDiskUtil::False,
-					                        TeamMustHaveShards::True,
-					                        ForReadBalance(readRebalance),
-					                        PreferLowerReadUtil::False);
-					destReq = GetTeamRequest(WantNewServers::True,
-					                         WantTrueBest::True,
-					                         PreferLowerDiskUtil::True,
-					                         TeamMustHaveShards::False,
-					                         ForReadBalance(readRebalance),
-					                         PreferLowerReadUtil::True);
-				}
 				state Future<SrcDestTeamPair> getTeamFuture =
 				    getSrcDestTeams(self, teamCollectionIndex, srcReq, destReq, ddPriority, &traceEvent);
 				wait(ready(getTeamFuture));

fdbserver/TenantCache.actor.cpp

@@ -217,7 +217,8 @@ public:
 
 		for (uint16_t i = 0; i < tenantCount; i++) {
 			TenantName tenantName(format("%s_%08d", "ddtc_test_tenant", tenantNumber + i));
-			TenantMapEntry tenant(tenantNumber + i, TenantState::READY);
+			TenantMapEntry tenant(tenantNumber + i, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION);
+
 			tenantCache.insert(tenantName, tenant);
 		}
 
@@ -244,7 +245,8 @@ public:
 
 		for (uint16_t i = 0; i < tenantCount; i++) {
 			TenantName tenantName(format("%s_%08d", "ddtc_test_tenant", tenantNumber + i));
-			TenantMapEntry tenant(tenantNumber + i, TenantState::READY);
+			TenantMapEntry tenant(tenantNumber + i, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION);
+
 			tenantCache.insert(tenantName, tenant);
 		}
 
@@ -257,7 +259,7 @@ public:
 
 			if (tenantOrdinal % staleTenantFraction != 0) {
 				TenantName tenantName(format("%s_%08d", "ddtc_test_tenant", tenantOrdinal));
-				TenantMapEntry tenant(tenantOrdinal, TenantState::READY);
+				TenantMapEntry tenant(tenantOrdinal, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION);
 				bool newTenant = tenantCache.update(tenantName, tenant);
 				ASSERT(!newTenant);
 				keepCount++;

fdbserver/include/fdbserver/MasterInterface.h

@@ -83,13 +83,15 @@ struct ChangeCoordinatorsRequest {
 	constexpr static FileIdentifier file_identifier = 13605416;
 	Standalone<StringRef> newConnectionString;
 	ReplyPromise<Void> reply; // normally throws even on success!
+	UID masterId;
 
 	ChangeCoordinatorsRequest() {}
-	ChangeCoordinatorsRequest(Standalone<StringRef> newConnectionString) : newConnectionString(newConnectionString) {}
+	ChangeCoordinatorsRequest(Standalone<StringRef> newConnectionString, UID masterId)
+	  : newConnectionString(newConnectionString), masterId(masterId) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, newConnectionString, reply);
+		serializer(ar, newConnectionString, reply, masterId);
 	}
 };
 

fdbserver/storageserver.actor.cpp

@@ -3818,6 +3818,9 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 	state double getValuesStart = g_network->timer();
 	getRange.begin = firstGreaterOrEqual(KeyRef(*a, prefix));
 	getRange.end = firstGreaterOrEqual(strinc(prefix, *a));
+	if (pOriginalReq->debugID.present())
+		g_traceBatch.addEvent(
+		    "TransactionDebug", pOriginalReq->debugID.get().first(), "storageserver.quickGetKeyValues.Before");
 	try {
 		// TODO: Use a lower level API may be better? Or tweak priorities?
 		GetKeyValuesRequest req;
@@ -3848,6 +3851,10 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 			getRange.result = RangeResultRef(reply.data, reply.more);
 			const double duration = g_network->timer() - getValuesStart;
 			data->counters.mappedRangeLocalSample.addMeasurement(duration);
+			if (pOriginalReq->debugID.present())
+				g_traceBatch.addEvent("TransactionDebug",
+				                      pOriginalReq->debugID.get().first(),
+				                      "storageserver.quickGetKeyValues.AfterLocalFetch");
 			return getRange;
 		}
 		// Otherwise fallback.
@@ -3869,6 +3876,10 @@ ACTOR Future<GetRangeReqAndResultRef> quickGetKeyValues(
 		getRange.result = rangeResult;
 		const double duration = g_network->timer() - getValuesStart;
 		data->counters.mappedRangeRemoteSample.addMeasurement(duration);
+		if (pOriginalReq->debugID.present())
+			g_traceBatch.addEvent("TransactionDebug",
+			                      pOriginalReq->debugID.get().first(),
+			                      "storageserver.quickGetKeyValues.AfterRemoteFetch");
 		return getRange;
 	} else {
 		throw quick_get_key_values_miss();
@@ -4156,7 +4167,9 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 	result.arena.dependsOn(input.arena);
 
 	result.data.reserve(result.arena, input.data.size());
-
+	if (pOriginalReq->debugID.present())
+		g_traceBatch.addEvent(
+		    "TransactionDebug", pOriginalReq->debugID.get().first(), "storageserver.mapKeyValues.Start");
 	state Tuple mappedKeyFormatTuple;
 	state Tuple mappedKeyTuple;
 
@@ -4175,6 +4188,9 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 	state std::vector<MappedKeyValueRef> kvms(k);
 	state std::vector<Future<Void>> subqueries;
 	state int offset = 0;
+	if (pOriginalReq->debugID.present())
+		g_traceBatch.addEvent(
+		    "TransactionDebug", pOriginalReq->debugID.get().first(), "storageserver.mapKeyValues.BeforeLoop");
 	for (; offset < sz; offset += SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE) {
 		// Divide into batches of MAX_PARALLEL_QUICK_GET_VALUE subqueries
 		for (int i = 0; i + offset < sz && i < SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE; i++) {
@@ -4210,11 +4226,17 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
 			                                 mappedKey));
 		}
 		wait(waitForAll(subqueries));
+		if (pOriginalReq->debugID.present())
+			g_traceBatch.addEvent(
+			    "TransactionDebug", pOriginalReq->debugID.get().first(), "storageserver.mapKeyValues.AfterBatch");
 		subqueries.clear();
 		for (int i = 0; i + offset < sz && i < SERVER_KNOBS->MAX_PARALLEL_QUICK_GET_VALUE; i++) {
 			result.data.push_back(result.arena, kvms[i]);
 		}
 	}
+	if (pOriginalReq->debugID.present())
+		g_traceBatch.addEvent(
+		    "TransactionDebug", pOriginalReq->debugID.get().first(), "storageserver.mapKeyValues.AfterAll");
 	return result;
 }
 
@@ -4240,11 +4262,11 @@ bool rangeIntersectsAnyTenant(TenantPrefixIndex& prefixIndex, KeyRangeRef range,
 
 TEST_CASE("/fdbserver/storageserver/rangeIntersectsAnyTenant") {
 	std::map<TenantName, TenantMapEntry> entries = {
-		std::make_pair("tenant0"_sr, TenantMapEntry(0, TenantState::READY)),
+		std::make_pair("tenant0"_sr, TenantMapEntry(0, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION)),
-		std::make_pair("tenant2"_sr, TenantMapEntry(2, TenantState::READY)),
+		std::make_pair("tenant2"_sr, TenantMapEntry(2, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION)),
-		std::make_pair("tenant3"_sr, TenantMapEntry(3, TenantState::READY)),
+		std::make_pair("tenant3"_sr, TenantMapEntry(3, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION)),
-		std::make_pair("tenant4"_sr, TenantMapEntry(4, TenantState::READY)),
+		std::make_pair("tenant4"_sr, TenantMapEntry(4, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION)),
-		std::make_pair("tenant6"_sr, TenantMapEntry(6, TenantState::READY))
+		std::make_pair("tenant6"_sr, TenantMapEntry(6, TenantState::READY, SERVER_KNOBS->ENABLE_ENCRYPTION))
 	};
 	TenantPrefixIndex index;
 	index.createNewVersion(1);

fdbserver/tester.actor.cpp

@@ -1643,6 +1643,7 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 			if (deterministicRandom()->coinflip()) {
 				entry.tenantGroup = "TestTenantGroup"_sr;
 			}
+			entry.encrypted = SERVER_KNOBS->ENABLE_ENCRYPTION;
 			TraceEvent("CreatingTenant").detail("Tenant", tenant).detail("TenantGroup", entry.tenantGroup);
 			tenantFutures.push_back(success(TenantAPI::createTenant(cx.getReference(), tenant, entry)));
 		}

fdbserver/workloads/FuzzApiCorrectness.actor.cpp

@@ -24,6 +24,7 @@
 #include <sstream>
 
 #include "fdbclient/FDBOptions.g.h"
+#include "fdbserver/Knobs.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbclient/GenericManagementAPI.actor.h"
 #include "fdbclient/TenantManagement.actor.h"
@@ -240,6 +241,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
 			if (i < self->numTenants) {
 				TenantMapEntry entry;
 				entry.tenantGroup = self->getTenantGroup(i);
+				entry.encrypted = SERVER_KNOBS->ENABLE_ENCRYPTION;
 				tenantFutures.push_back(::success(TenantAPI::createTenant(cx.getReference(), tenantName, entry)));
 				self->createdTenants.insert(tenantName);
 			}

fdbserver/workloads/TenantManagementWorkload.actor.cpp

@@ -46,10 +46,11 @@ struct TenantManagementWorkload : TestWorkload {
 		int64_t id;
 		Optional<TenantGroupName> tenantGroup;
 		bool empty;
+		bool encrypted;
 
 		TenantData() : id(-1), empty(true) {}
-		TenantData(int64_t id, Optional<TenantGroupName> tenantGroup, bool empty)
+		TenantData(int64_t id, Optional<TenantGroupName> tenantGroup, bool empty, bool encrypted)
-		  : id(id), tenantGroup(tenantGroup), empty(empty) {}
+		  : id(id), tenantGroup(tenantGroup), empty(empty), encrypted(encrypted) {}
 	};
 
 	struct TenantGroupData {
@@ -328,6 +329,11 @@ struct TenantManagementWorkload : TestWorkload {
 
 			TenantMapEntry entry;
 			entry.tenantGroup = self->chooseTenantGroup(true);
+			if (operationType == OperationType::SPECIAL_KEYS) {
+				entry.encrypted = SERVER_KNOBS->ENABLE_ENCRYPTION;
+			} else {
+				entry.encrypted = deterministicRandom()->coinflip();
+			}
 
 			if (self->createdTenants.count(tenant)) {
 				alreadyExists = true;
@@ -453,7 +459,7 @@ struct TenantManagementWorkload : TestWorkload {
 					// Update our local tenant state to include the newly created one
 					self->maxId = entry.get().id;
 					self->createdTenants[tenantItr->first] =
-					    TenantData(entry.get().id, tenantItr->second.tenantGroup, true);
+					    TenantData(entry.get().id, tenantItr->second.tenantGroup, true, tenantItr->second.encrypted);
 
 					// If this tenant has a tenant group, create or update the entry for it
 					if (tenantItr->second.tenantGroup.present()) {
@@ -854,11 +860,13 @@ struct TenantManagementWorkload : TestWorkload {
 		std::string tenantStateStr;
 		std::string base64TenantGroup;
 		std::string printableTenantGroup;
+		bool encrypted;
 		std::string assignedClusterStr;
 
 		jsonDoc.get("id", id);
 		jsonDoc.get("prefix.base64", base64Prefix);
 		jsonDoc.get("prefix.printable", printablePrefix);
+		jsonDoc.get("prefix.encrypted", encrypted);
 
 		prefix = base64::decoder::from_string(base64Prefix);
 		ASSERT(prefix == unprintable(printablePrefix));
@@ -878,7 +886,7 @@ struct TenantManagementWorkload : TestWorkload {
 			assignedCluster = ClusterNameRef(assignedClusterStr);
 		}
 
-		TenantMapEntry entry(id, TenantMapEntry::stringToTenantState(tenantStateStr), tenantGroup);
+		TenantMapEntry entry(id, TenantMapEntry::stringToTenantState(tenantStateStr), tenantGroup, encrypted);
 		ASSERT(entry.prefix == prefix);
 		return entry;
 	}
@@ -1461,6 +1469,7 @@ struct TenantManagementWorkload : TestWorkload {
 				ASSERT(localItr != self->createdTenants.end());
 				ASSERT(dataItr->first == localItr->first);
 				ASSERT(dataItr->second.tenantGroup == localItr->second.tenantGroup);
+				ASSERT(dataItr->second.encrypted == localItr->second.encrypted);
 
 				checkTenants.push_back(checkTenantContents(self, dataItr->first, localItr->second));
 				lastTenant = dataItr->first;

packaging/deb/builddebs.sh

@@ -52,8 +52,8 @@ mkdir -p -m 0755 $CLIENTSDIR/usr/share/doc/foundationdb-clients
 mkdir -p -m 0755 $CLIENTSDIR/usr/lib/foundationdb/backup_agent
 
 install -m 0755 bin/fdbcli $CLIENTSDIR/usr/bin
-install -m 0644 lib/libfdb_c.so $CLIENTSDIR/usr/lib
+install -m 0644 lib/libfdb_c.so lib/libfdb_c_shim.so $CLIENTSDIR/usr/lib
-install -m 0644 bindings/c/foundationdb/fdb_c.h bindings/c/foundationdb/fdb_c_types.h bindings/c/foundationdb/fdb_c_internal.h bindings/c/foundationdb/fdb_c_options.g.h fdbclient/vexillographer/fdb.options $CLIENTSDIR/usr/include/foundationdb
+install -m 0644 bindings/c/foundationdb/fdb_c.h bindings/c/foundationdb/fdb_c_types.h bindings/c/foundationdb/fdb_c_internal.h bindings/c/foundationdb/fdb_c_options.g.h fdbclient/vexillographer/fdb.options bindings/c/foundationdb/fdb_c_shim.h $CLIENTSDIR/usr/include/foundationdb
 dos2unix -q -n README.md $CLIENTSDIR/usr/share/doc/foundationdb-clients/README
 chmod 0644 $CLIENTSDIR/usr/share/doc/foundationdb-clients/README
 install -m 0755 bin/fdbbackup $CLIENTSDIR/usr/lib/foundationdb/backup_agent/backup_agent

packaging/rpm/buildrpms.sh

@@ -33,7 +33,8 @@ install -m 0755 bin/fdbcli $INSTDIR/usr/bin
 install -m 0755 bin/fdbserver $INSTDIR/usr/sbin
 install -m 0755 bin/fdbmonitor $INSTDIR/usr/lib/foundationdb
 install -m 0755 lib/libfdb_c.so $INSTDIR/usr/lib64
-install -m 0644 bindings/c/foundationdb/fdb_c.h bindings/c/foundationdb/fdb_c_options.g.h bindings/c/foundationdb/fdb_c_types.h bindings/c/foundationdb/fdb_c_internal.h fdbclient/vexillographer/fdb.options $INSTDIR/usr/include/foundationdb
+install -m 0755 lib/libfdb_c_shim.so $INSTDIR/usr/lib64
+install -m 0644 bindings/c/foundationdb/fdb_c.h bindings/c/foundationdb/fdb_c_options.g.h bindings/c/foundationdb/fdb_c_types.h bindings/c/foundationdb/fdb_c_internal.h bindings/c/foundationdb/fdb_c_shim.h fdbclient/vexillographer/fdb.options $INSTDIR/usr/include/foundationdb
 dos2unix -q -n README.md $INSTDIR/usr/share/doc/foundationdb-clients/README
 dos2unix -q -n README.md $INSTDIR/usr/share/doc/foundationdb-server/README
 chmod 0644 $INSTDIR/usr/share/doc/foundationdb-clients/README