Merge remote-tracking branch 'origin/main' into fix-knob-typo

.git-blame-ignore-revs

@@ -0,0 +1,4 @@
+# clang-format the entire codebase
+df90cc89de67ea4748c8cadd18e6fc4ce7fda12e
+2c788c233db56ccec4ed90d7da31887487b9f3b7
+69508b980f3cc5aabea6322f292e53b07bb27544

bindings/c/CMakeLists.txt

@@ -137,7 +137,6 @@ if(NOT WIN32)
     test/apitester/TesterUtil.h
     test/apitester/TesterWorkload.cpp
     test/apitester/TesterWorkload.h
-    ../../flow/SimpleOpt.h
   )
 
   if(OPEN_FOR_IDE)
@@ -193,7 +192,8 @@ if(NOT WIN32)
   else()
     target_link_libraries(fdb_c_api_tester PRIVATE fdb_c fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
   endif()
-  target_include_directories(fdb_c_api_tester PRIVATE "${CMAKE_BINARY_DIR}/flow/include")
+  target_include_directories(fdb_c_api_tester PRIVATE "${CMAKE_SOURCE_DIR}/flow/include" "${CMAKE_BINARY_DIR}/flow/include")
+  target_link_libraries(fdb_c_api_tester PRIVATE SimpleOpt)
 
   # do not set RPATH for mako
   set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)

bindings/c/test/apitester/fdb_c_api_tester.cpp

@@ -24,7 +24,7 @@
 #include "TesterTransactionExecutor.h"
 #include "TesterTestSpec.h"
 #include "TesterUtil.h"
-#include "flow/SimpleOpt.h"
+#include "SimpleOpt/SimpleOpt.h"
 #include "test/fdb_api.hpp"
 
 #include <memory>

bindings/flow/CMakeLists.txt

@@ -16,7 +16,7 @@ set(SRCS
   fdb_flow.actor.cpp
   fdb_flow.h)
 
-add_flow_target(STATIC_LIBRARY NAME fdb_flow SRCS ${SRCS} NO_COPY_HDR)
+add_flow_target(STATIC_LIBRARY NAME fdb_flow SRCS ${SRCS})
 target_link_libraries(fdb_flow PUBLIC fdb_c)
 target_link_libraries(fdb_flow PUBLIC fdbclient)
 target_include_directories(fdb_flow PUBLIC

bindings/python/tests/fdbcli_tests.py

@@ -620,6 +620,15 @@ def tenants(logger):
     assert lines[0].strip().startswith('id: ')
     assert lines[1].strip().startswith('prefix: ')
 
+    output = run_fdbcli_command('gettenant tenant JSON')
+    json_output = json.loads(output, strict=False)
+    assert(len(json_output) == 2)
+    assert('tenant' in json_output)
+    assert(json_output['type'] == 'success')
+    assert(len(json_output['tenant']) == 2)
+    assert('id' in json_output['tenant'])
+    assert('prefix' in json_output['tenant'])
+
     output = run_fdbcli_command('usetenant')
     assert output == 'Using the default tenant'
 

cmake/FlowCommands.cmake

@@ -64,9 +64,8 @@ function(generate_coverage_xml)
       WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
       COMMENT "Generate coverage xml")
   endif()
-  add_custom_target(coverage_${target_name} DEPENDS ${target_file})
+  add_custom_target(coverage_${target_name} ALL DEPENDS ${target_file})
   add_dependencies(coverage_${target_name} coveragetool)
-  add_dependencies(${target_name} coverage_${target_name})
 endfunction()
 
 # This function asserts that `versions.h` does not exist in the source
@@ -190,7 +189,7 @@ endfunction()
 
 function(add_flow_target)
   set(options EXECUTABLE STATIC_LIBRARY
-    DYNAMIC_LIBRARY NO_COPY_HDR)
+    DYNAMIC_LIBRARY)
   set(oneValueArgs NAME)
   set(multiValueArgs SRCS COVERAGE_FILTER_OUT DISABLE_ACTOR_DIAGNOSTICS ADDL_SRCS)
   cmake_parse_arguments(AFT "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
@@ -200,9 +199,6 @@ function(add_flow_target)
   if(NOT AFT_SRCS)
     message(FATAL_ERROR "No sources provided")
   endif()
-  if(NOT AFT_NO_COPY_HDR)
-    copy_headers(NAME ${AFT_NAME} SRCS "${AFT_SRCS};${AFT_DISABLE_ACTOR_DIAGNOSTICS}" OUT_DIR incl_dir INC_DIR include_dir)
-  endif()
   #foreach(src IN LISTS AFT_SRCS)
   #  is_header(h "${src}")
   #  if(NOT h)
@@ -214,6 +210,7 @@ function(add_flow_target)
     set(sources ${AFT_SRCS} ${AFT_ADDL_SRCS})
     add_library(${AFT_NAME} OBJECT ${sources})
   else()
+    create_build_dirs(${AFT_SRCS} ${AFT_DISABLE_ACTOR_DIAGNOSTICS})
     foreach(src IN LISTS AFT_SRCS AFT_DISABLE_ACTOR_DIAGNOSTICS)
       is_header(hdr ${src})
       set(in_filename "${src}")
@@ -229,26 +226,11 @@ function(add_flow_target)
         set(out_filename "${src}")
       endif()
 
-      if(hdr AND NOT AFT_NO_COPY_HDR)
-        set(in_file "${incl_dir}/${in_filename}")
-        set(out_file "${incl_dir}/${out_filename}")
+      set(in_file "${CMAKE_CURRENT_SOURCE_DIR}/${in_filename}")
+      if(is_actor_file)
+        set(out_file "${CMAKE_CURRENT_BINARY_DIR}/${out_filename}")
       else()
-        set(in_file "${CMAKE_CURRENT_SOURCE_DIR}/${in_filename}")
-        if(is_actor_file)
-          set(out_file "${CMAKE_CURRENT_BINARY_DIR}/${out_filename}")
-        else()
-          set(out_file "${in_file}")
-        endif()
-      endif()
-
-      is_prefix(in_src_dir "${CMAKE_CURRENT_SOURCE_DIR}" ${src})
-      is_prefix(in_bin_dir "${CMAKE_CURRENT_BINARY_DIR}" ${src})
-      if(NOT AFT_NO_COPY_HDR)
-        is_prefix(in_incl_dir "${incl_dir}" ${src})
-      endif()
-      if(in_src_dir OR in_bin_dir)
-        set(in_file "${src}")
-        set(out_file "${src}")
+        set(out_file "${in_file}")
       endif()
 
       list(APPEND sources ${out_file})
@@ -307,10 +289,6 @@ function(add_flow_target)
     set_property(TARGET ${AFT_NAME} PROPERTY COVERAGE_FILTERS ${AFT_SRCS})
 
     add_custom_target(${AFT_NAME}_actors DEPENDS ${generated_files})
-    if(NOT AFT_NO_COPY_HDR)
-      target_include_directories("${AFT_NAME}" PUBLIC "${include_dir}")
-      add_dependencies(${AFT_NAME}_actors actorcompiler "${AFT_NAME}_incl")
-    endif()
     add_dependencies(${AFT_NAME} ${AFT_NAME}_actors)
     if(NOT WIN32)
       assert_no_version_h(${AFT_NAME}_actors)

cmake/utils.cmake

@@ -29,3 +29,40 @@ function(is_prefix out prefix str)
   endif()
   set(${out} ${res} PARENT_SCOPE)
 endfunction()
+
+function(create_build_dirs)
+  foreach(src IN LISTS ARGV)
+    get_filename_component(d "${src}" DIRECTORY)
+    if(IS_ABSOLUTE "${d}")
+      file(RELATIVE_PATH d "${CMAKE_CURRENT_SOURCE_DIR}" "${src}")
+    endif()
+    list(APPEND dirs "${d}")
+  endforeach()
+  list(REMOVE_DUPLICATES dirs)
+  foreach(dir IN LISTS dirs)
+    make_directory("${CMAKE_CURRENT_BINARY_DIR}/${dir}")
+  endforeach()
+endfunction()
+
+function(fdb_find_sources out)
+  file(GLOB res
+    LIST_DIRECTORIES false
+    RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
+    CONFIGURE_DEPENDS "*.cpp" "*.c" "*.h" "*.hpp")
+  file(GLOB_RECURSE res_includes
+    LIST_DIRECTORIES false
+    RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/include"
+    CONFIGURE_DEPENDS "include/*.cpp" "include/*.c" "include/*.h" "include/*.hpp")
+  file(GLOB_RECURSE res_workloads
+    LIST_DIRECTORIES false
+    RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/workloads"
+    CONFIGURE_DEPENDS "workloads/*.cpp" "workloads/*.c" "workloads/*.h" "workloads/*.hpp")
+
+  foreach(f IN LISTS res_includes)
+    list(APPEND res "include/${f}")
+  endforeach()
+  foreach(f IN LISTS res_workloads)
+    list(APPEND res "workloads/${f}")
+  endforeach()
+  set(${out} "${res}" PARENT_SCOPE)
+endfunction()

contrib/CMakeLists.txt

@@ -1,8 +1,14 @@
 add_library(rapidjson INTERFACE)
 target_include_directories(rapidjson INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/rapidjson)
 
+add_subdirectory(crc32)
+add_subdirectory(stacktrace)
+add_subdirectory(folly_memcpy)
+add_subdirectory(sqlite)
+add_subdirectory(SimpleOpt)
 add_subdirectory(fmt-8.1.1)
 if(NOT WIN32)
+  add_subdirectory(linenoise)
   add_subdirectory(debug_determinism)
   add_subdirectory(monitoring)
   add_subdirectory(TraceLogHelper)

contrib/SimpleOpt/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_library(SimpleOpt INTERFACE)
+target_include_directories(SimpleOpt INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include")

contrib/SimpleOpt/include/SimpleOpt/SimpleOpt.h

@@ -78,7 +78,7 @@
     <li> Include the SimpleOpt.h header file
 
         <pre>
-        \#include "flow/SimpleOpt.h"
+        \#include "SimpleOpt/SimpleOpt.h"
         </pre>
 
     <li> Define an array of valid options for your program.

contrib/crc32/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_library(crc32 STATIC crc32.S crc32_wrapper.c crc32c.cpp)
+target_include_directories(crc32 PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")

contrib/crc32/crc32.S

@@ -58,7 +58,7 @@
 #ifdef CRC32_CONSTANTS_HEADER
 #include CRC32_CONSTANTS_HEADER
 #else
-#include "crc32_constants.h"
+#include "crc32/crc32_constants.h"
 #endif
 
 	.text

contrib/crc32/crc32_wrapper.c

@@ -15,7 +15,7 @@
 #ifdef CRC32_CONSTANTS_HEADER
 #include CRC32_CONSTANTS_HEADER
 #else
-#include "crc32_constants.h"
+#include "crc32/crc32_constants.h"
 #endif
 
 #define VMX_ALIGN 16

contrib/crc32/crc32c.cpp

@@ -25,7 +25,17 @@
 #define _CRT_SECURE_NO_WARNINGS
 #endif
 
-#include "flow/crc32c.h"
+#if (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
+#define __unixish__ 1
+#endif
+
+#ifdef __unixish__
+#if !defined(__aarch64__) && !defined(__powerpc64__)
+#include <cpuid.h>
+#endif
+#endif
+
+#include "crc32/crc32c.h"
 
 #if !defined(__aarch64__) && !defined(__powerpc64__)
 #include <nmmintrin.h>
@@ -34,9 +44,40 @@
 #include <stdlib.h>
 #include <random>
 #include <algorithm>
-#include "flow/Platform.h"
 #include "crc32c-generated-constants.cpp"
 
+// CRC32C
+#ifdef __aarch64__
+// aarch64
+#include <inttypes.h>
+static inline uint32_t hwCrc32cU8(unsigned int crc, unsigned char v) {
+	uint32_t ret;
+	asm volatile("crc32cb %w[r], %w[c], %w[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
+	return ret;
+}
+static inline uint32_t hwCrc32cU32(unsigned int crc, unsigned int v) {
+	uint32_t ret;
+	asm volatile("crc32cw %w[r], %w[c], %w[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
+	return ret;
+}
+#ifdef _M_X64
+static inline uint64_t hwCrc32cU64(uint64_t crc, uint64_t v) {
+	uint64_t ret;
+	asm volatile("crc32cx %w[r], %w[c], %x[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
+	return ret;
+}
+#endif
+#else
+#ifndef __powerpc64__
+// Intel
+#define hwCrc32cU8(c, v) _mm_crc32_u8(c, v)
+#define hwCrc32cU32(c, v) _mm_crc32_u32(c, v)
+#ifdef _M_X64
+#define hwCrc32cU64(c, v) _mm_crc32_u64(c, v)
+#endif
+#endif
+#endif
+
 [[maybe_unused]] static uint32_t append_trivial(uint32_t crc, const uint8_t* input, size_t length) {
 	for (size_t i = 0; i < length; ++i) {
 		crc = crc ^ input[i];
@@ -278,7 +319,25 @@ uint32_t ppc_hw(uint32_t crc, const uint8_t* input, size_t length) {
 }
 #endif
 
-static bool hw_available = platform::isHwCrcSupported();
+bool isHwCrcSupported() {
+#if defined(_WIN32)
+	int info[4];
+	__cpuid(info, 1);
+	return (info[2] & (1 << 20)) != 0;
+#elif defined(__aarch64__)
+	return true; /* force to use crc instructions */
+#elif defined(__powerpc64__)
+	return false; /* force not to use crc instructions */
+#elif defined(__unixish__)
+	uint32_t eax, ebx, ecx, edx, level = 1, count = 0;
+	__cpuid_count(level, count, eax, ebx, ecx, edx);
+	return ((ecx >> 20) & 1) != 0;
+#else
+#error Port me!
+#endif
+}
+
+static bool hw_available = isHwCrcSupported();
 
 extern "C" uint32_t crc32c_append(uint32_t crc, const uint8_t* input, size_t length) {
 	if (hw_available) {

contrib/folly_memcpy/CMakeLists.txt

@@ -0,0 +1,4 @@
+if(UNIX AND NOT APPLE)
+  add_library(folly_memcpy STATIC folly_memcpy.S)
+  target_include_directories(folly_memcpy PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")
+endif()

contrib/linenoise/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_library(linenoise STATIC linenoise.c)
+target_include_directories(linenoise PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")

contrib/linenoise/linenoise.c

@@ -115,7 +115,7 @@
 #include <sys/types.h>
 #include <sys/ioctl.h>
 #include <unistd.h>
-#include "linenoise.h"
+#include "linenoise/linenoise.h"
 
 #define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100
 #define LINENOISE_MAX_LINE 4096

contrib/sqlite/CMakeLists.txt

@@ -0,0 +1,16 @@
+add_library(sqlite STATIC
+  btree.h
+  hash.h
+  sqlite3.h
+  sqlite3ext.h
+  sqliteInt.h
+  sqliteLimit.h
+  sqlite3.amalgamation.c)
+
+
+target_include_directories(sqlite PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+# Suppress warnings in sqlite since it's third party
+if(NOT WIN32)
+  target_compile_definitions(sqlite PRIVATE $<$<CONFIG:Debug>:NDEBUG>)
+  target_compile_options(sqlite BEFORE PRIVATE -w) # disable warnings for third party
+endif()

contrib/stacktrace/CMakeLists.txt

@@ -0,0 +1,11 @@
+add_library(stacktrace STATIC stacktrace.amalgamation.cpp)
+target_include_directories(stacktrace PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
+if (USE_ASAN)
+  target_compile_definitions(stacktrace PRIVATE ADDRESS_SANITIZER)
+elseif(USE_MSAN)
+  target_compile_definitions(stacktrace PRIVATE MEMORY_SANITIZER)
+elseif(USE_UBSAN)
+  target_compile_definitions(stacktrace PRIVATE UNDEFINED_BEHAVIOR_SANITIZER)
+elseif(USE_TSAN)
+  target_compile_definitions(stacktrace PRIVATE THREAD_SANITIZER DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL=1)
+endif()

documentation/sphinx/source/command-line-interface.rst

@@ -234,12 +234,27 @@ Note that :ref:`characters can be escaped <cli-escaping>` when specifying keys (
 gettenant
 ---------
 
-The ``gettenant`` command fetches metadata for a given tenant and displays it. Its syntax is ``gettenant <TENANT_NAME>``.
+The ``gettenant`` command fetches metadata for a given tenant and displays it. Its syntax is ``gettenant <TENANT_NAME> [JSON]``.
 
-Included in the output of this command are the ``id`` and ``prefix`` assigned to the tenant. If the tenant does not exist, ``fdbcli`` will report an error.
+Included in the output of this command are the ``id`` and ``prefix`` assigned to the tenant. If the tenant does not exist, ``fdbcli`` will report an error. If ``JSON`` is specified, then the output will be written as a JSON document::
 
-getversion
-----------
+    {
+        "tenant": {
+            "id": 0,
+            "prefix": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"
+        },
+        "type": "success"
+    }
+
+In the event of an error, the output will include an error message::
+
+    {
+        "error": "...",
+        "type": "error"
+    }
+
+    getversion
+    ----------
 
 The ``getversion`` command fetches the current read version of the cluster or currently running transaction.
 

documentation/sphinx/source/release-notes/release-notes-710.rst

@@ -4,6 +4,18 @@
 Release Notes
 #############
 
+7.1.11
+======
+* Same as 7.1.10 release with AVX enabled.
+
+7.1.10
+======
+* Released with AVX disabled.
+* Fixed a sequencer crash when DC ID is a string. `(PR #7393) <https://github.com/apple/foundationdb/pull/7393>`_
+* Fixed a client performance regression by removing unnecessary transaction initialization. `(PR #7365) <https://github.com/apple/foundationdb/pull/7365>`_
+* Safely removed fdb_transaction_get_range_and_flat_map C API. `(PR #7379) <https://github.com/apple/foundationdb/pull/7379>`_
+* Fixed an unknown error bug when hostname resolving fails. `(PR #7380) <https://github.com/apple/foundationdb/pull/7380>`_
+
 7.1.9
 =====
 * Same as 7.1.8 release with AVX enabled.
@@ -15,7 +27,7 @@ Release Notes
 * Added RSS bytes for processes in status json output and corrected available_bytes calculation. `(PR #7348) <https://github.com/apple/foundationdb/pull/7348>`_
 * Added versionstamp support in tuples. `(PR #7313) <https://github.com/apple/foundationdb/pull/7313>`_
 * Fixed some spammy trace events. `(PR #7300) <https://github.com/apple/foundationdb/pull/7300>`_
-* Fixed a memory corruption bug for using streaming peeks. `(PR #7288) <https://github.com/apple/foundationdb/pull/7288>`_
+* Avoided a memory corruption bug by disabling streaming peeks. `(PR #7288) <https://github.com/apple/foundationdb/pull/7288>`_
 * Fixed a hang bug in fdbcli exclude command. `(PR #7268) <https://github.com/apple/foundationdb/pull/7268>`_
 * Fixed an issue that a remote TLog blocks peeks. `(PR #7255) <https://github.com/apple/foundationdb/pull/7255>`_
 * Fixed a connection issue using hostnames. `(PR #7264) <https://github.com/apple/foundationdb/pull/7264>`_

fdbbackup/CMakeLists.txt

@@ -1,23 +1,22 @@
 set(FDBBACKUP_SRCS
-	BackupTLSConfig.h
 	BackupTLSConfig.cpp
 	backup.actor.cpp)
 
 add_flow_target(EXECUTABLE NAME fdbbackup SRCS ${FDBBACKUP_SRCS})
+target_include_directories(fdbbackup PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 target_link_libraries(fdbbackup PRIVATE fdbclient)
 
 set(FDBCONVERT_SRCS
-	FileConverter.actor.cpp
-	FileConverter.h)
+	FileConverter.actor.cpp)
 add_flow_target(EXECUTABLE NAME fdbconvert SRCS ${FDBCONVERT_SRCS})
+target_include_directories(fdbconvert PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 target_link_libraries(fdbconvert PRIVATE fdbclient)
 
 set(FDBDECODE_SRCS
-	BackupTLSConfig.h
 	BackupTLSConfig.cpp
-	FileDecoder.actor.cpp
-	FileConverter.h)
+	FileDecoder.actor.cpp)
 add_flow_target(EXECUTABLE NAME fdbdecode SRCS ${FDBDECODE_SRCS})
+target_include_directories(fdbdecode PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 target_link_libraries(fdbdecode PRIVATE fdbclient)
 
 if(NOT OPEN_FOR_IDE)

fdbbackup/backup.actor.cpp

@@ -74,7 +74,7 @@
 #include "fdbclient/versions.h"
 #include "fdbclient/BuildFlags.h"
 
-#include "flow/SimpleOpt.h"
+#include "SimpleOpt/SimpleOpt.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 // Type of program being executed

fdbbackup/include/fdbbackup/FileConverter.h

@@ -23,7 +23,7 @@
 #pragma once
 
 #include <cinttypes>
-#include "flow/SimpleOpt.h"
+#include "SimpleOpt/SimpleOpt.h"
 #include "flow/TLSConfig.actor.h"
 
 namespace file_converter {

fdbcli/CMakeLists.txt

@@ -1,43 +1,12 @@
-set(FDBCLI_SRCS
-  fdbcli.actor.cpp
-  fdbcli.actor.h
-  AdvanceVersionCommand.actor.cpp
-  BlobRangeCommand.actor.cpp
-  CacheRangeCommand.actor.cpp
-  ConfigureCommand.actor.cpp
-  ConsistencyCheckCommand.actor.cpp
-  CoordinatorsCommand.actor.cpp
-  DataDistributionCommand.actor.cpp
-  ExcludeCommand.actor.cpp
-  ExpensiveDataCheckCommand.actor.cpp
-  FileConfigureCommand.actor.cpp
-  FlowLineNoise.actor.cpp
-  FlowLineNoise.h
-  ForceRecoveryWithDataLossCommand.actor.cpp
-  IncludeCommand.actor.cpp
-  KillCommand.actor.cpp
-  LockCommand.actor.cpp
-  ChangeFeedCommand.actor.cpp
-  MaintenanceCommand.actor.cpp
-  ProfileCommand.actor.cpp
-  SetClassCommand.actor.cpp
-  SnapshotCommand.actor.cpp
-  StatusCommand.actor.cpp
-  SuspendCommand.actor.cpp
-  TenantCommands.actor.cpp
-  ThrottleCommand.actor.cpp
-  TriggerDDTeamInfoLogCommand.actor.cpp
-  TssqCommand.actor.cpp
-  Util.actor.cpp
-  VersionEpochCommand.actor.cpp
-  linenoise/linenoise.h)
-
-if(NOT WIN32)
-  list(APPEND FDBCLI_SRCS linenoise/linenoise.c)
-endif()
+fdb_find_sources(FDBCLI_SRCS)
 
 add_flow_target(EXECUTABLE NAME fdbcli SRCS ${FDBCLI_SRCS})
-target_link_libraries(fdbcli PRIVATE fdbclient)
+target_include_directories(fdbcli PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
+target_link_libraries(fdbcli PRIVATE fdbclient SimpleOpt)
+
+if(NOT WIN32)
+  target_link_libraries(fdbcli PRIVATE linenoise)
+endif()
 
 if(NOT OPEN_FOR_IDE)
   if(GENERATE_DEBUG_PACKAGES)

fdbcli/FlowLineNoise.actor.cpp

@@ -30,7 +30,7 @@
 
 #if __unixish__
 #define HAVE_LINENOISE 1
-#include "fdbcli/linenoise/linenoise.h"
+#include "linenoise/linenoise.h"
 #else
 #define HAVE_LINENOISE 0
 #endif

fdbcli/TenantCommands.actor.cpp

@@ -209,11 +209,12 @@ CommandFactory listTenantsFactory(
 
 // gettenant command
 ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
-	if (tokens.size() != 2) {
+	if (tokens.size() < 2 || tokens.size() > 3 || (tokens.size() == 3 && tokens[2] != "JSON"_sr)) {
 		printUsage(tokens[0]);
 		return false;
 	}
 
+	state bool useJson = tokens.size() == 3;
 	state Key tenantNameKey = fdb_cli::tenantSpecialKeyRange.begin.withSuffix(tokens[1]);
 	state Reference<ITransaction> tr = db->createTransaction();
 
@@ -228,30 +229,58 @@ ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<St
 
 			json_spirit::mValue jsonObject;
 			json_spirit::read_string(tenant.get().toString(), jsonObject);
-			JSONDoc doc(jsonObject);
 
-			int64_t id;
-			std::string prefix;
-			doc.get("id", id);
-			doc.get("prefix", prefix);
+			if (useJson) {
+				json_spirit::mObject resultObj;
+				resultObj["tenant"] = jsonObject;
+				resultObj["type"] = "success";
+				printf("%s\n",
+				       json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print).c_str());
+			} else {
+				JSONDoc doc(jsonObject);
+
+				int64_t id;
+				std::string prefix;
+				doc.get("id", id);
+				doc.get("prefix", prefix);
+
+				printf("  id: %" PRId64 "\n", id);
+				printf("  prefix: %s\n", printable(prefix).c_str());
+			}
 
-			printf("  id: %" PRId64 "\n", id);
-			printf("  prefix: %s\n", printable(prefix).c_str());
 			return true;
 		} catch (Error& e) {
-			state Error err(e);
-			if (e.code() == error_code_special_keys_api_failure) {
-				std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
-				fprintf(stderr, "ERROR: %s\n", errorMsgStr.c_str());
-				return false;
+			try {
+				wait(safeThreadFutureToFuture(tr->onError(e)));
+			} catch (Error& finalErr) {
+				state std::string errorStr;
+				if (finalErr.code() == error_code_special_keys_api_failure) {
+					std::string str = wait(getSpecialKeysFailureErrorMessage(tr));
+					errorStr = str;
+				} else if (useJson) {
+					errorStr = finalErr.what();
+				} else {
+					throw finalErr;
+				}
+
+				if (useJson) {
+					json_spirit::mObject resultObj;
+					resultObj["type"] = "error";
+					resultObj["error"] = errorStr;
+					printf(
+					    "%s\n",
+					    json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print).c_str());
+				} else {
+					fprintf(stderr, "ERROR: %s\n", errorStr.c_str());
+				}
 			}
-			wait(safeThreadFutureToFuture(tr->onError(err)));
 		}
 	}
 }
 
-CommandFactory getTenantFactory("gettenant",
-                                CommandHelp("gettenant <TENANT_NAME>",
-                                            "prints the metadata for a tenant",
-                                            "Prints the metadata for a tenant."));
+CommandFactory getTenantFactory(
+    "gettenant",
+    CommandHelp("gettenant <TENANT_NAME> [JSON]",
+                "prints the metadata for a tenant",
+                "Prints the metadata for a tenant. If JSON is specified, then the output will be in JSON format."));
 } // namespace fdb_cli

fdbcli/fdbcli.actor.cpp

@@ -39,6 +39,7 @@
 #include "fdbclient/FDBOptions.g.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/TagThrottle.actor.h"
+#include "fdbclient/TenantManagement.actor.h"
 #include "fdbclient/Tuple.h"
 
 #include "fdbclient/ThreadSafeTransaction.h"
@@ -51,7 +52,7 @@
 
 #include "flow/TLSConfig.actor.h"
 #include "flow/ThreadHelper.actor.h"
-#include "flow/SimpleOpt.h"
+#include "SimpleOpt/SimpleOpt.h"
 
 #include "fdbcli/FlowLineNoise.h"
 #include "fdbcli/fdbcli.actor.h"
@@ -63,7 +64,7 @@
 
 #ifdef __unixish__
 #include <stdio.h>
-#include "fdbcli/linenoise/linenoise.h"
+#include "linenoise/linenoise.h"
 #endif
 
 #include "fdbclient/versions.h"
@@ -1049,7 +1050,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 	state Database localDb;
 	state Reference<IDatabase> db;
 	state Reference<ITenant> tenant;
-	state Optional<Standalone<StringRef>> tenantName;
+	state Optional<TenantName> tenantName;
 	state Optional<TenantMapEntry> tenantEntry;
 
 	// This tenant is kept empty for operations that perform management tasks (e.g. killing a process)
@@ -1840,7 +1841,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 						}
 					} else {
 						Optional<TenantMapEntry> entry =
-						    wait(makeInterruptable(ManagementAPI::tryGetTenant(db, tokens[1])));
+						    wait(makeInterruptable(TenantAPI::tryGetTenant(db, tokens[1])));
 						if (!entry.present()) {
 							fprintf(stderr, "ERROR: Tenant `%s' does not exist\n", printable(tokens[1]).c_str());
 							is_error = true;

fdbclient/CMakeLists.txt

@@ -1,197 +1,8 @@
-set(FDBCLIENT_SRCS
-  ActorLineageProfiler.h
-  ActorLineageProfiler.cpp
-  AnnotateActor.h
-  AnnotateActor.cpp
-  AsyncFileS3BlobStore.actor.cpp
-  AsyncFileS3BlobStore.actor.h
-  AsyncTaskThread.actor.cpp
-  AsyncTaskThread.h
-  Atomic.h
-  AutoPublicAddress.cpp
-  BackupAgent.actor.h
-  BackupAgentBase.actor.cpp
-  BackupContainer.actor.cpp
-  BackupContainer.h
-  BackupContainerFileSystem.actor.cpp
-  BackupContainerFileSystem.h
-  BackupContainerLocalDirectory.actor.cpp
-  BackupContainerLocalDirectory.h
-  BackupContainerS3BlobStore.actor.cpp
-  BackupContainerS3BlobStore.h
-  BlobConnectionProvider.h
-  BlobConnectionProvider.cpp
-  BlobWorkerInterface.h
-  BlobGranuleReader.actor.cpp
-  BlobGranuleReader.actor.h
-  BlobGranuleCommon.h
-  BlobGranuleFiles.cpp
-  BlobGranuleFiles.h
-  BlobMetadataUtils.h
-  BlobWorkerCommon.h
-  ClientBooleanParams.cpp
-  ClientBooleanParams.h
-  ClientKnobCollection.cpp
-  ClientKnobCollection.h
-  ClientKnobs.cpp
-  ClientKnobs.h
-  ClientLogEvents.h
-  ClientVersion.h
-  ClientWorkerInterface.h
-  ClusterConnectionFile.actor.cpp
-  ClusterConnectionFile.h
-  ClusterConnectionKey.actor.cpp
-  ClusterConnectionKey.actor.h
-  ClusterConnectionMemoryRecord.actor.cpp
-  ClusterConnectionMemoryRecord.h
-  ClusterInterface.h
-  CommitProxyInterface.h
-  CommitTransaction.h
-  ConfigKnobs.cpp
-  ConfigKnobs.h
-  ConfigTransactionInterface.cpp
-  ConfigTransactionInterface.h
-  ConvertUTF.h
-  CoordinationInterface.h
-  DatabaseBackupAgent.actor.cpp
-  DatabaseConfiguration.cpp
-  DatabaseConfiguration.h
-  DatabaseContext.h
-  EventTypes.actor.h
-  FDBOptions.h
-  FDBTypes.cpp
-  FDBTypes.h
-  FluentDSampleIngestor.cpp
-  FileBackupAgent.actor.cpp
-  GenericManagementAPI.actor.h
-  GlobalConfig.h
-  GlobalConfig.actor.h
-  GlobalConfig.actor.cpp
-  GrvProxyInterface.h
-  HTTP.h
-  HTTP.actor.cpp
-  HighContentionPrefixAllocator.actor.h
-  IClientApi.h
-  IConfigTransaction.cpp
-  IConfigTransaction.h
-  ISingleThreadTransaction.cpp
-  ISingleThreadTransaction.h
-  JSONDoc.h
-  JsonBuilder.cpp
-  JsonBuilder.h
-  KeyBackedTypes.h
-  KeyRangeMap.actor.cpp
-  KeyRangeMap.h
-  Knobs.h
-  IKnobCollection.cpp
-  IKnobCollection.h
-  LocalClientAPI.cpp
-  LocalClientAPI.h
-  ManagementAPI.actor.cpp
-  ManagementAPI.actor.h
-  MonitorLeader.actor.cpp
-  MonitorLeader.h
-  MultiVersionAssignmentVars.h
-  MultiVersionTransaction.actor.cpp
-  MultiVersionTransaction.h
-  MutationList.h
-  MutationLogReader.actor.cpp
-  MutationLogReader.actor.h
-  NameLineage.h
-  NameLineage.cpp
-  NativeAPI.actor.cpp
-  NativeAPI.actor.h
-  Notified.h
-  ParallelStream.actor.cpp
-  ParallelStream.actor.h
-  PaxosConfigTransaction.actor.cpp
-  PaxosConfigTransaction.h
-  PImpl.h
-  ProcessInterface.h
-  SimpleConfigTransaction.actor.cpp
-  SpecialKeySpace.actor.cpp
-  SpecialKeySpace.actor.h
-  RESTClient.h
-  RESTClient.actor.cpp
-  RESTUtils.h
-  RESTUtils.actor.cpp
-  ReadYourWrites.actor.cpp
-  ReadYourWrites.h
-  RestoreInterface.cpp
-  RestoreInterface.h
-  RunTransaction.actor.h
-  RYWIterator.cpp
-  RYWIterator.h
-  S3BlobStore.h
-  S3BlobStore.actor.cpp
-  Schemas.cpp
-  Schemas.h
-  ServerKnobCollection.cpp
-  ServerKnobCollection.h
-  ServerKnobs.cpp
-  ServerKnobs.h
-  SimpleConfigTransaction.h
-  SimpleIni.h
-  SnapshotCache.h
-  SpecialKeySpace.actor.cpp
-  SpecialKeySpace.actor.h
-  Status.h
-  StatusClient.actor.cpp
-  StatusClient.h
-  StorageServerInterface.cpp
-  StorageServerInterface.h
-  StorageCheckpoint.h
-  Subspace.cpp
-  Subspace.h
-  StackLineage.h
-  StackLineage.cpp
-  SystemData.cpp
-  SystemData.h
-  TagThrottle.actor.cpp
-  TagThrottle.actor.h
-  TaskBucket.actor.cpp
-  TaskBucket.h
-  Tenant.cpp
-  Tenant.h
-  TestKnobCollection.cpp
-  TestKnobCollection.h
-  ThreadSafeTransaction.cpp
-  ThreadSafeTransaction.h
-  Tracing.h
-  Tracing.actor.cpp
-  TransactionLineage.h
-  Tuple.cpp
-  Tuple.h
-  VersionedMap.actor.h
-  VersionedMap.h
-  VersionedMap.cpp
-  Versionstamp.cpp
-  Versionstamp.h
-  VersionVector.h
-  VersionVector.cpp
-  WriteMap.h
-  WriteMap.cpp
-  json_spirit/json_spirit_error_position.h
-  json_spirit/json_spirit_reader_template.h
-  json_spirit/json_spirit_value.h
-  json_spirit/json_spirit_writer_options.h
-  json_spirit/json_spirit_writer_template.h
-  libb64/encode.h
-  libb64/decode.h
-  libb64/cdecode.h
-  libb64/cdecode.c
-  libb64/cencode.h
-  libb64/cencode.c
-  md5/md5.h
-  md5/md5.c
-  rapidxml/rapidxml.hpp
-  rapidxml/rapidxml_iterators.hpp
-  rapidxml/rapidxml_print.hpp
-  rapidxml/rapidxml_utils.hpp
-  sha1/SHA1.h
+fdb_find_sources(FDBCLIENT_SRCS)
+list(APPEND FDBCLIENT_SRCS
   sha1/SHA1.cpp
-  zipf.c
-  zipf.h)
+  libb64/cdecode.c
+  libb64/cencode.c)
 
 message(STATUS "FDB version is ${FDB_VERSION}")
 message(STATUS "FDB package name is ${FDB_PACKAGE_NAME}")
@@ -224,8 +35,7 @@ if(BUILD_AZURE_BACKUP)
   add_compile_definitions(BUILD_AZURE_BACKUP)
   set(FDBCLIENT_SRCS
     ${FDBCLIENT_SRCS}
-    BackupContainerAzureBlobStore.actor.cpp
-    BackupContainerAzureBlobStore.h)
+    azure_backup/BackupContainerAzureBlobStore.actor.cpp)
 
   configure_file(azurestorage.cmake azurestorage-download/CMakeLists.txt)
 
@@ -267,6 +77,7 @@ if(WITH_AWS_BACKUP)
 endif()
 
 add_flow_target(STATIC_LIBRARY NAME fdbclient SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
+target_include_directories(fdbclient PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/versions.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/versions.h)
 add_dependencies(fdbclient fdboptions)
 target_link_libraries(fdbclient PUBLIC fdbrpc msgpack)
@@ -275,6 +86,7 @@ target_link_libraries(fdbclient PUBLIC fdbrpc msgpack)
 # fdbserver retain sampling functionality in client code while disabling
 # sampling for pure clients.
 add_flow_target(STATIC_LIBRARY NAME fdbclient_sampling SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
+target_include_directories(fdbclient_sampling PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
 add_dependencies(fdbclient_sampling fdboptions)
 target_link_libraries(fdbclient_sampling PUBLIC fdbrpc_sampling msgpack)
 target_compile_definitions(fdbclient_sampling PRIVATE -DENABLE_SAMPLING)

fdbclient/DatabaseConfiguration.cpp

@@ -637,8 +637,7 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 		parse((&type), value);
 		storageMigrationType = (StorageMigrationType::MigrationType)type;
 	} else if (ck == LiteralStringRef("tenant_mode")) {
-		parse((&type), value);
-		tenantMode = (TenantMode::Mode)type;
+		tenantMode = TenantMode::fromValue(value);
 	} else if (ck == LiteralStringRef("proxies")) {
 		overwriteProxiesCount();
 	} else if (ck == LiteralStringRef("blob_granules_enabled")) {

fdbclient/MultiVersionTransaction.actor.cpp

@@ -1340,7 +1340,7 @@ bool MultiVersionTransaction::isValid() {
 }
 
 // MultiVersionTenant
-MultiVersionTenant::MultiVersionTenant(Reference<MultiVersionDatabase> db, StringRef tenantName)
+MultiVersionTenant::MultiVersionTenant(Reference<MultiVersionDatabase> db, TenantNameRef tenantName)
   : tenantState(makeReference<TenantState>(db, tenantName)) {}
 
 MultiVersionTenant::~MultiVersionTenant() {
@@ -1363,7 +1363,7 @@ ThreadFuture<Void> MultiVersionTenant::waitPurgeGranulesComplete(const KeyRef& p
 	return abortableFuture(f, tenantState->db->dbState->dbVar->get().onChange);
 }
 
-MultiVersionTenant::TenantState::TenantState(Reference<MultiVersionDatabase> db, StringRef tenantName)
+MultiVersionTenant::TenantState::TenantState(Reference<MultiVersionDatabase> db, TenantNameRef tenantName)
   : tenantVar(new ThreadSafeAsyncVar<Reference<ITenant>>(Reference<ITenant>(nullptr))), tenantName(tenantName), db(db),
     closed(false) {
 	updateTenant();

fdbclient/NativeAPI.actor.cpp

@@ -7015,7 +7015,7 @@ ACTOR Future<ProtocolVersion> getClusterProtocolImpl(
 		} else {
 			state NetworkAddress coordinatorAddress;
 			if (coordinator->get().get().hostname.present()) {
-				Hostname h = coordinator->get().get().hostname.get();
+				state Hostname h = coordinator->get().get().hostname.get();
 				wait(store(coordinatorAddress, h.resolveWithRetry()));
 			} else {
 				coordinatorAddress = coordinator->get().get().getLeader.getEndpoint().getPrimaryAddress();
@@ -9588,3 +9588,27 @@ int64_t getMaxWriteKeySize(KeyRef const& key, bool hasRawAccess) {
 int64_t getMaxClearKeySize(KeyRef const& key) {
 	return getMaxKeySize(key);
 }
+
+namespace NativeAPI {
+
+ACTOR Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> getServerListAndProcessClasses(
+    Transaction* tr) {
+	state Future<std::vector<ProcessData>> workers = getWorkers(tr);
+	state Future<RangeResult> serverList = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
+	wait(success(workers) && success(serverList));
+	ASSERT(!serverList.get().more && serverList.get().size() < CLIENT_KNOBS->TOO_MANY);
+
+	std::map<Optional<Standalone<StringRef>>, ProcessData> id_data;
+	for (int i = 0; i < workers.get().size(); i++)
+		id_data[workers.get()[i].locality.processId()] = workers.get()[i];
+
+	std::vector<std::pair<StorageServerInterface, ProcessClass>> results;
+	for (int i = 0; i < serverList.get().size(); i++) {
+		auto ssi = decodeServerListValue(serverList.get()[i].value);
+		results.emplace_back(ssi, id_data[ssi.locality.processId()].processClass);
+	}
+
+	return results;
+}
+
+} // namespace NativeAPI

fdbclient/ServerKnobs.cpp

@@ -801,7 +801,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( FASTRESTORE_NUM_LOADERS,                                 3 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_LOADERS = deterministicRandom()->random01() * 10 + 1; }
 	init( FASTRESTORE_NUM_APPLIERS,                                3 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_APPLIERS = deterministicRandom()->random01() * 10 + 1; }
 	init( FASTRESTORE_TXN_BATCH_MAX_BYTES,           1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_BATCH_MAX_BYTES = deterministicRandom()->random01() * 1024.0 * 1024.0 + 1.0; }
-	init( FASTRESTORE_VERSIONBATCH_MAX_BYTES, 10.0 * 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VERSIONBATCH_MAX_BYTES = deterministicRandom()->random01() < 0.2 ? 10 * 1024 : deterministicRandom()->random01() < 0.4 ? 100 * 1024 * 1024 : deterministicRandom()->random01() * 1000.0 * 1024.0 * 1024.0; } // too small value may increase chance of TooManyFile error
+	init( FASTRESTORE_VERSIONBATCH_MAX_BYTES, 10.0 * 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VERSIONBATCH_MAX_BYTES = deterministicRandom()->random01() < 0.2 ? 50 * 1024 : deterministicRandom()->random01() < 0.4 ? 100 * 1024 * 1024 : deterministicRandom()->random01() * 1000.0 * 1024.0 * 1024.0; } // too small value may increase chance of TooManyFile error
 	init( FASTRESTORE_VB_PARALLELISM,                              5 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_PARALLELISM = deterministicRandom()->random01() < 0.2 ? 2 : deterministicRandom()->random01() * 10 + 1; }
 	init( FASTRESTORE_VB_MONITOR_DELAY,                           30 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_MONITOR_DELAY = deterministicRandom()->random01() * 20 + 1; }
 	init( FASTRESTORE_VB_LAUNCH_DELAY,                           1.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_LAUNCH_DELAY = deterministicRandom()->random01() < 0.2 ? 0.1 : deterministicRandom()->random01() * 10.0 + 1; }

fdbclient/SpecialKeySpace.actor.cpp

@@ -2730,123 +2730,3 @@ Future<Optional<std::string>> FailedLocalitiesRangeImpl::commit(ReadYourWritesTr
 	// exclude locality with failed option as true.
 	return excludeLocalityCommitActor(ryw, true);
 }
-
-ACTOR Future<RangeResult> getTenantList(ReadYourWritesTransaction* ryw, KeyRangeRef kr, GetRangeLimits limitsHint) {
-	state KeyRef managementPrefix =
-	    kr.begin.substr(0,
-	                    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin.size() +
-	                        TenantMapRangeImpl::submoduleRange.begin.size());
-
-	kr = kr.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
-	TenantNameRef beginTenant = kr.begin.removePrefix(TenantMapRangeImpl::submoduleRange.begin);
-
-	TenantNameRef endTenant = kr.end;
-	if (endTenant.startsWith(TenantMapRangeImpl::submoduleRange.begin)) {
-		endTenant = endTenant.removePrefix(TenantMapRangeImpl::submoduleRange.begin);
-	} else {
-		endTenant = "\xff"_sr;
-	}
-
-	std::map<TenantName, TenantMapEntry> tenants =
-	    wait(ManagementAPI::listTenantsTransaction(&ryw->getTransaction(), beginTenant, endTenant, limitsHint.rows));
-
-	RangeResult results;
-	for (auto tenant : tenants) {
-		json_spirit::mObject tenantEntry;
-		tenantEntry["id"] = tenant.second.id;
-		tenantEntry["prefix"] = tenant.second.prefix.toString();
-		std::string tenantEntryString = json_spirit::write_string(json_spirit::mValue(tenantEntry));
-		ValueRef tenantEntryBytes(results.arena(), tenantEntryString);
-		results.push_back(results.arena(),
-		                  KeyValueRef(tenant.first.withPrefix(managementPrefix, results.arena()), tenantEntryBytes));
-	}
-
-	return results;
-}
-
-TenantMapRangeImpl::TenantMapRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
-
-Future<RangeResult> TenantMapRangeImpl::getRange(ReadYourWritesTransaction* ryw,
-                                                 KeyRangeRef kr,
-                                                 GetRangeLimits limitsHint) const {
-	return getTenantList(ryw, kr, limitsHint);
-}
-
-ACTOR Future<Void> createTenants(ReadYourWritesTransaction* ryw, std::vector<TenantNameRef> tenants) {
-	Optional<Value> lastIdVal = wait(ryw->getTransaction().get(tenantLastIdKey));
-	int64_t previousId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) : -1;
-
-	std::vector<Future<Void>> createFutures;
-	for (auto tenant : tenants) {
-		createFutures.push_back(
-		    success(ManagementAPI::createTenantTransaction(&ryw->getTransaction(), tenant, ++previousId)));
-	}
-
-	ryw->getTransaction().set(tenantLastIdKey, TenantMapEntry::idToPrefix(previousId));
-	wait(waitForAll(createFutures));
-	return Void();
-}
-
-ACTOR Future<Void> deleteTenantRange(ReadYourWritesTransaction* ryw,
-                                     TenantNameRef beginTenant,
-                                     TenantNameRef endTenant) {
-	std::map<TenantName, TenantMapEntry> tenants = wait(
-	    ManagementAPI::listTenantsTransaction(&ryw->getTransaction(), beginTenant, endTenant, CLIENT_KNOBS->TOO_MANY));
-
-	if (tenants.size() == CLIENT_KNOBS->TOO_MANY) {
-		TraceEvent(SevWarn, "DeleteTenantRangeTooLange")
-		    .detail("BeginTenant", beginTenant)
-		    .detail("EndTenant", endTenant);
-		ryw->setSpecialKeySpaceErrorMsg("too many tenants to range delete");
-		throw special_keys_api_failure();
-	}
-
-	std::vector<Future<Void>> deleteFutures;
-	for (auto tenant : tenants) {
-		deleteFutures.push_back(ManagementAPI::deleteTenantTransaction(&ryw->getTransaction(), tenant.first));
-	}
-
-	wait(waitForAll(deleteFutures));
-	return Void();
-}
-
-Future<Optional<std::string>> TenantMapRangeImpl::commit(ReadYourWritesTransaction* ryw) {
-	auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range);
-	std::vector<TenantNameRef> tenantsToCreate;
-	std::vector<Future<Void>> tenantManagementFutures;
-	for (auto range : ranges) {
-		if (!range.value().first) {
-			continue;
-		}
-
-		TenantNameRef tenantName =
-		    range.begin()
-		        .removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)
-		        .removePrefix(TenantMapRangeImpl::submoduleRange.begin);
-
-		if (range.value().second.present()) {
-			tenantsToCreate.push_back(tenantName);
-		} else {
-			// For a single key clear, just issue the delete
-			if (KeyRangeRef(range.begin(), range.end()).singleKeyRange()) {
-				tenantManagementFutures.push_back(
-				    ManagementAPI::deleteTenantTransaction(&ryw->getTransaction(), tenantName));
-			} else {
-				TenantNameRef endTenant = range.end().removePrefix(
-				    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
-				if (endTenant.startsWith(submoduleRange.begin)) {
-					endTenant = endTenant.removePrefix(submoduleRange.begin);
-				} else {
-					endTenant = "\xff"_sr;
-				}
-				tenantManagementFutures.push_back(deleteTenantRange(ryw, tenantName, endTenant));
-			}
-		}
-	}
-
-	if (tenantsToCreate.size()) {
-		tenantManagementFutures.push_back(createTenants(ryw, tenantsToCreate));
-	}
-
-	return tag(waitForAll(tenantManagementFutures), Optional<std::string>());
-}

fdbclient/StorageServerInterface.cpp

@@ -21,7 +21,7 @@
 // TODO this should really be renamed "TSSComparison.cpp"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/BlobWorkerInterface.h"
-#include "flow/crc32c.h" // for crc32c_append, to checksum values in tss trace events
+#include "crc32/crc32c.h" // for crc32c_append, to checksum values in tss trace events
 
 // Includes template specializations for all tss operations on storage server types.
 // New StorageServerInterface reply types must be added here or it won't compile.

fdbclient/Tenant.cpp

@@ -22,6 +22,35 @@
 #include "fdbclient/Tenant.h"
 #include "flow/UnitTest.h"
 
+Key TenantMapEntry::idToPrefix(int64_t id) {
+	int64_t swapped = bigEndian64(id);
+	return StringRef(reinterpret_cast<const uint8_t*>(&swapped), 8);
+}
+
+int64_t TenantMapEntry::prefixToId(KeyRef prefix) {
+	ASSERT(prefix.size() == 8);
+	int64_t id = *reinterpret_cast<const int64_t*>(prefix.begin());
+	id = bigEndian64(id);
+	ASSERT(id >= 0);
+	return id;
+}
+
+void TenantMapEntry::initPrefix(KeyRef subspace) {
+	ASSERT(id >= 0);
+	prefix = makeString(8 + subspace.size());
+	uint8_t* data = mutateString(prefix);
+	if (subspace.size() > 0) {
+		memcpy(data, subspace.begin(), subspace.size());
+	}
+	int64_t swapped = bigEndian64(id);
+	memcpy(data + subspace.size(), &swapped, 8);
+}
+
+TenantMapEntry::TenantMapEntry() : id(-1) {}
+TenantMapEntry::TenantMapEntry(int64_t id, KeyRef subspace) : id(id) {
+	initPrefix(subspace);
+}
+
 TEST_CASE("/fdbclient/TenantMapEntry/Serialization") {
 	TenantMapEntry entry1(1, ""_sr);
 	ASSERT(entry1.prefix == "\x00\x00\x00\x00\x00\x00\x00\x01"_sr);

fdbclient/TenantSpecialKeys.actor.cpp

@@ -0,0 +1,149 @@
+/*
+ * TenantSpecialKeys.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ActorLineageProfiler.h"
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/Knobs.h"
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/SpecialKeySpace.actor.h"
+#include "fdbclient/TenantManagement.actor.h"
+#include "flow/Arena.h"
+#include "flow/UnitTest.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+ACTOR Future<RangeResult> getTenantList(ReadYourWritesTransaction* ryw, KeyRangeRef kr, GetRangeLimits limitsHint) {
+	state KeyRef managementPrefix =
+	    kr.begin.substr(0,
+	                    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin.size() +
+	                        TenantMapRangeImpl::submoduleRange.begin.size());
+
+	kr = kr.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
+	TenantNameRef beginTenant = kr.begin.removePrefix(TenantMapRangeImpl::submoduleRange.begin);
+
+	TenantNameRef endTenant = kr.end;
+	if (endTenant.startsWith(TenantMapRangeImpl::submoduleRange.begin)) {
+		endTenant = endTenant.removePrefix(TenantMapRangeImpl::submoduleRange.begin);
+	} else {
+		endTenant = "\xff"_sr;
+	}
+
+	std::map<TenantName, TenantMapEntry> tenants =
+	    wait(TenantAPI::listTenantsTransaction(&ryw->getTransaction(), beginTenant, endTenant, limitsHint.rows));
+
+	RangeResult results;
+	for (auto tenant : tenants) {
+		json_spirit::mObject tenantEntry;
+		tenantEntry["id"] = tenant.second.id;
+		tenantEntry["prefix"] = tenant.second.prefix.toString();
+		std::string tenantEntryString = json_spirit::write_string(json_spirit::mValue(tenantEntry));
+		ValueRef tenantEntryBytes(results.arena(), tenantEntryString);
+		results.push_back(results.arena(),
+		                  KeyValueRef(tenant.first.withPrefix(managementPrefix, results.arena()), tenantEntryBytes));
+	}
+
+	return results;
+}
+
+TenantMapRangeImpl::TenantMapRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
+
+Future<RangeResult> TenantMapRangeImpl::getRange(ReadYourWritesTransaction* ryw,
+                                                 KeyRangeRef kr,
+                                                 GetRangeLimits limitsHint) const {
+	return getTenantList(ryw, kr, limitsHint);
+}
+
+ACTOR Future<Void> createTenants(ReadYourWritesTransaction* ryw, std::vector<TenantNameRef> tenants) {
+	Optional<Value> lastIdVal = wait(ryw->getTransaction().get(tenantLastIdKey));
+	int64_t previousId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) : -1;
+
+	std::vector<Future<Void>> createFutures;
+	for (auto tenant : tenants) {
+		createFutures.push_back(
+		    success(TenantAPI::createTenantTransaction(&ryw->getTransaction(), tenant, ++previousId)));
+	}
+
+	ryw->getTransaction().set(tenantLastIdKey, TenantMapEntry::idToPrefix(previousId));
+	wait(waitForAll(createFutures));
+	return Void();
+}
+
+ACTOR Future<Void> deleteTenantRange(ReadYourWritesTransaction* ryw,
+                                     TenantNameRef beginTenant,
+                                     TenantNameRef endTenant) {
+	std::map<TenantName, TenantMapEntry> tenants =
+	    wait(TenantAPI::listTenantsTransaction(&ryw->getTransaction(), beginTenant, endTenant, CLIENT_KNOBS->TOO_MANY));
+
+	if (tenants.size() == CLIENT_KNOBS->TOO_MANY) {
+		TraceEvent(SevWarn, "DeleteTenantRangeTooLange")
+		    .detail("BeginTenant", beginTenant)
+		    .detail("EndTenant", endTenant);
+		ryw->setSpecialKeySpaceErrorMsg("too many tenants to range delete");
+		throw special_keys_api_failure();
+	}
+
+	std::vector<Future<Void>> deleteFutures;
+	for (auto tenant : tenants) {
+		deleteFutures.push_back(TenantAPI::deleteTenantTransaction(&ryw->getTransaction(), tenant.first));
+	}
+
+	wait(waitForAll(deleteFutures));
+	return Void();
+}
+
+Future<Optional<std::string>> TenantMapRangeImpl::commit(ReadYourWritesTransaction* ryw) {
+	auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range);
+	std::vector<TenantNameRef> tenantsToCreate;
+	std::vector<Future<Void>> tenantManagementFutures;
+	for (auto range : ranges) {
+		if (!range.value().first) {
+			continue;
+		}
+
+		TenantNameRef tenantName =
+		    range.begin()
+		        .removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)
+		        .removePrefix(TenantMapRangeImpl::submoduleRange.begin);
+
+		if (range.value().second.present()) {
+			tenantsToCreate.push_back(tenantName);
+		} else {
+			// For a single key clear, just issue the delete
+			if (KeyRangeRef(range.begin(), range.end()).singleKeyRange()) {
+				tenantManagementFutures.push_back(
+				    TenantAPI::deleteTenantTransaction(&ryw->getTransaction(), tenantName));
+			} else {
+				TenantNameRef endTenant = range.end().removePrefix(
+				    SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
+				if (endTenant.startsWith(submoduleRange.begin)) {
+					endTenant = endTenant.removePrefix(submoduleRange.begin);
+				} else {
+					endTenant = "\xff"_sr;
+				}
+				tenantManagementFutures.push_back(deleteTenantRange(ryw, tenantName, endTenant));
+			}
+		}
+	}
+
+	if (tenantsToCreate.size()) {
+		tenantManagementFutures.push_back(createTenants(ryw, tenantsToCreate));
+	}
+
+	return tag(waitForAll(tenantManagementFutures), Optional<std::string>());
+}

fdbclient/ThreadSafeTransaction.cpp

@@ -175,7 +175,7 @@ Reference<ITransaction> ThreadSafeTenant::createTransaction() {
 
 ThreadFuture<Key> ThreadSafeTenant::purgeBlobGranules(const KeyRangeRef& keyRange, Version purgeVersion, bool force) {
 	DatabaseContext* db = this->db->db;
-	Standalone<StringRef> tenantName = this->name;
+	TenantName tenantName = this->name;
 	KeyRange range = keyRange;
 	return onMainThread([db, range, purgeVersion, tenantName, force]() -> Future<Key> {
 		return db->purgeBlobGranules(range, purgeVersion, tenantName, force);