Merge remote-tracking branch 'origin/main' into fix-knob-typo

This commit is contained in:
sfc-gh-tclinkenbeard 2022-06-28 15:58:18 -07:00
commit be9a2002c3
462 changed files with 1179 additions and 2572 deletions

4
.git-blame-ignore-revs Normal file
View File

@ -0,0 +1,4 @@
# clang-format the entire codebase
df90cc89de67ea4748c8cadd18e6fc4ce7fda12e
2c788c233db56ccec4ed90d7da31887487b9f3b7
69508b980f3cc5aabea6322f292e53b07bb27544

View File

@ -137,7 +137,6 @@ if(NOT WIN32)
test/apitester/TesterUtil.h
test/apitester/TesterWorkload.cpp
test/apitester/TesterWorkload.h
../../flow/SimpleOpt.h
)
if(OPEN_FOR_IDE)
@ -193,7 +192,8 @@ if(NOT WIN32)
else()
target_link_libraries(fdb_c_api_tester PRIVATE fdb_c fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
endif()
target_include_directories(fdb_c_api_tester PRIVATE "${CMAKE_BINARY_DIR}/flow/include")
target_include_directories(fdb_c_api_tester PRIVATE "${CMAKE_SOURCE_DIR}/flow/include" "${CMAKE_BINARY_DIR}/flow/include")
target_link_libraries(fdb_c_api_tester PRIVATE SimpleOpt)
# do not set RPATH for mako
set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)

View File

@ -24,7 +24,7 @@
#include "TesterTransactionExecutor.h"
#include "TesterTestSpec.h"
#include "TesterUtil.h"
#include "flow/SimpleOpt.h"
#include "SimpleOpt/SimpleOpt.h"
#include "test/fdb_api.hpp"
#include <memory>

View File

@ -16,7 +16,7 @@ set(SRCS
fdb_flow.actor.cpp
fdb_flow.h)
add_flow_target(STATIC_LIBRARY NAME fdb_flow SRCS ${SRCS} NO_COPY_HDR)
add_flow_target(STATIC_LIBRARY NAME fdb_flow SRCS ${SRCS})
target_link_libraries(fdb_flow PUBLIC fdb_c)
target_link_libraries(fdb_flow PUBLIC fdbclient)
target_include_directories(fdb_flow PUBLIC

View File

@ -620,6 +620,15 @@ def tenants(logger):
assert lines[0].strip().startswith('id: ')
assert lines[1].strip().startswith('prefix: ')
output = run_fdbcli_command('gettenant tenant JSON')
json_output = json.loads(output, strict=False)
assert(len(json_output) == 2)
assert('tenant' in json_output)
assert(json_output['type'] == 'success')
assert(len(json_output['tenant']) == 2)
assert('id' in json_output['tenant'])
assert('prefix' in json_output['tenant'])
output = run_fdbcli_command('usetenant')
assert output == 'Using the default tenant'

View File

@ -64,9 +64,8 @@ function(generate_coverage_xml)
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "Generate coverage xml")
endif()
add_custom_target(coverage_${target_name} DEPENDS ${target_file})
add_custom_target(coverage_${target_name} ALL DEPENDS ${target_file})
add_dependencies(coverage_${target_name} coveragetool)
add_dependencies(${target_name} coverage_${target_name})
endfunction()
# This function asserts that `versions.h` does not exist in the source
@ -190,7 +189,7 @@ endfunction()
function(add_flow_target)
set(options EXECUTABLE STATIC_LIBRARY
DYNAMIC_LIBRARY NO_COPY_HDR)
DYNAMIC_LIBRARY)
set(oneValueArgs NAME)
set(multiValueArgs SRCS COVERAGE_FILTER_OUT DISABLE_ACTOR_DIAGNOSTICS ADDL_SRCS)
cmake_parse_arguments(AFT "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
@ -200,9 +199,6 @@ function(add_flow_target)
if(NOT AFT_SRCS)
message(FATAL_ERROR "No sources provided")
endif()
if(NOT AFT_NO_COPY_HDR)
copy_headers(NAME ${AFT_NAME} SRCS "${AFT_SRCS};${AFT_DISABLE_ACTOR_DIAGNOSTICS}" OUT_DIR incl_dir INC_DIR include_dir)
endif()
#foreach(src IN LISTS AFT_SRCS)
# is_header(h "${src}")
# if(NOT h)
@ -214,6 +210,7 @@ function(add_flow_target)
set(sources ${AFT_SRCS} ${AFT_ADDL_SRCS})
add_library(${AFT_NAME} OBJECT ${sources})
else()
create_build_dirs(${AFT_SRCS} ${AFT_DISABLE_ACTOR_DIAGNOSTICS})
foreach(src IN LISTS AFT_SRCS AFT_DISABLE_ACTOR_DIAGNOSTICS)
is_header(hdr ${src})
set(in_filename "${src}")
@ -229,26 +226,11 @@ function(add_flow_target)
set(out_filename "${src}")
endif()
if(hdr AND NOT AFT_NO_COPY_HDR)
set(in_file "${incl_dir}/${in_filename}")
set(out_file "${incl_dir}/${out_filename}")
set(in_file "${CMAKE_CURRENT_SOURCE_DIR}/${in_filename}")
if(is_actor_file)
set(out_file "${CMAKE_CURRENT_BINARY_DIR}/${out_filename}")
else()
set(in_file "${CMAKE_CURRENT_SOURCE_DIR}/${in_filename}")
if(is_actor_file)
set(out_file "${CMAKE_CURRENT_BINARY_DIR}/${out_filename}")
else()
set(out_file "${in_file}")
endif()
endif()
is_prefix(in_src_dir "${CMAKE_CURRENT_SOURCE_DIR}" ${src})
is_prefix(in_bin_dir "${CMAKE_CURRENT_BINARY_DIR}" ${src})
if(NOT AFT_NO_COPY_HDR)
is_prefix(in_incl_dir "${incl_dir}" ${src})
endif()
if(in_src_dir OR in_bin_dir)
set(in_file "${src}")
set(out_file "${src}")
set(out_file "${in_file}")
endif()
list(APPEND sources ${out_file})
@ -307,10 +289,6 @@ function(add_flow_target)
set_property(TARGET ${AFT_NAME} PROPERTY COVERAGE_FILTERS ${AFT_SRCS})
add_custom_target(${AFT_NAME}_actors DEPENDS ${generated_files})
if(NOT AFT_NO_COPY_HDR)
target_include_directories("${AFT_NAME}" PUBLIC "${include_dir}")
add_dependencies(${AFT_NAME}_actors actorcompiler "${AFT_NAME}_incl")
endif()
add_dependencies(${AFT_NAME} ${AFT_NAME}_actors)
if(NOT WIN32)
assert_no_version_h(${AFT_NAME}_actors)

View File

@ -29,3 +29,40 @@ function(is_prefix out prefix str)
endif()
set(${out} ${res} PARENT_SCOPE)
endfunction()
function(create_build_dirs)
foreach(src IN LISTS ARGV)
get_filename_component(d "${src}" DIRECTORY)
if(IS_ABSOLUTE "${d}")
file(RELATIVE_PATH d "${CMAKE_CURRENT_SOURCE_DIR}" "${src}")
endif()
list(APPEND dirs "${d}")
endforeach()
list(REMOVE_DUPLICATES dirs)
foreach(dir IN LISTS dirs)
make_directory("${CMAKE_CURRENT_BINARY_DIR}/${dir}")
endforeach()
endfunction()
function(fdb_find_sources out)
file(GLOB res
LIST_DIRECTORIES false
RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
CONFIGURE_DEPENDS "*.cpp" "*.c" "*.h" "*.hpp")
file(GLOB_RECURSE res_includes
LIST_DIRECTORIES false
RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/include"
CONFIGURE_DEPENDS "include/*.cpp" "include/*.c" "include/*.h" "include/*.hpp")
file(GLOB_RECURSE res_workloads
LIST_DIRECTORIES false
RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/workloads"
CONFIGURE_DEPENDS "workloads/*.cpp" "workloads/*.c" "workloads/*.h" "workloads/*.hpp")
foreach(f IN LISTS res_includes)
list(APPEND res "include/${f}")
endforeach()
foreach(f IN LISTS res_workloads)
list(APPEND res "workloads/${f}")
endforeach()
set(${out} "${res}" PARENT_SCOPE)
endfunction()

View File

@ -1,8 +1,14 @@
add_library(rapidjson INTERFACE)
target_include_directories(rapidjson INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/rapidjson)
add_subdirectory(crc32)
add_subdirectory(stacktrace)
add_subdirectory(folly_memcpy)
add_subdirectory(sqlite)
add_subdirectory(SimpleOpt)
add_subdirectory(fmt-8.1.1)
if(NOT WIN32)
add_subdirectory(linenoise)
add_subdirectory(debug_determinism)
add_subdirectory(monitoring)
add_subdirectory(TraceLogHelper)

View File

@ -0,0 +1,2 @@
add_library(SimpleOpt INTERFACE)
target_include_directories(SimpleOpt INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include")

View File

@ -78,7 +78,7 @@
<li> Include the SimpleOpt.h header file
<pre>
\#include "flow/SimpleOpt.h"
\#include "SimpleOpt/SimpleOpt.h"
</pre>
<li> Define an array of valid options for your program.

View File

@ -0,0 +1,2 @@
add_library(crc32 STATIC crc32.S crc32_wrapper.c crc32c.cpp)
target_include_directories(crc32 PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")

View File

@ -58,7 +58,7 @@
#ifdef CRC32_CONSTANTS_HEADER
#include CRC32_CONSTANTS_HEADER
#else
#include "crc32_constants.h"
#include "crc32/crc32_constants.h"
#endif
.text

View File

@ -15,7 +15,7 @@
#ifdef CRC32_CONSTANTS_HEADER
#include CRC32_CONSTANTS_HEADER
#else
#include "crc32_constants.h"
#include "crc32/crc32_constants.h"
#endif
#define VMX_ALIGN 16

View File

@ -25,7 +25,17 @@
#define _CRT_SECURE_NO_WARNINGS
#endif
#include "flow/crc32c.h"
#if (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
#define __unixish__ 1
#endif
#ifdef __unixish__
#if !defined(__aarch64__) && !defined(__powerpc64__)
#include <cpuid.h>
#endif
#endif
#include "crc32/crc32c.h"
#if !defined(__aarch64__) && !defined(__powerpc64__)
#include <nmmintrin.h>
@ -34,9 +44,40 @@
#include <stdlib.h>
#include <random>
#include <algorithm>
#include "flow/Platform.h"
#include "crc32c-generated-constants.cpp"
// CRC32C
#ifdef __aarch64__
// aarch64
#include <inttypes.h>
static inline uint32_t hwCrc32cU8(unsigned int crc, unsigned char v) {
uint32_t ret;
asm volatile("crc32cb %w[r], %w[c], %w[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
return ret;
}
static inline uint32_t hwCrc32cU32(unsigned int crc, unsigned int v) {
uint32_t ret;
asm volatile("crc32cw %w[r], %w[c], %w[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
return ret;
}
#ifdef _M_X64
static inline uint64_t hwCrc32cU64(uint64_t crc, uint64_t v) {
uint64_t ret;
asm volatile("crc32cx %w[r], %w[c], %x[v]" : [r] "=r"(ret) : [c] "r"(crc), [v] "r"(v));
return ret;
}
#endif
#else
#ifndef __powerpc64__
// Intel
#define hwCrc32cU8(c, v) _mm_crc32_u8(c, v)
#define hwCrc32cU32(c, v) _mm_crc32_u32(c, v)
#ifdef _M_X64
#define hwCrc32cU64(c, v) _mm_crc32_u64(c, v)
#endif
#endif
#endif
[[maybe_unused]] static uint32_t append_trivial(uint32_t crc, const uint8_t* input, size_t length) {
for (size_t i = 0; i < length; ++i) {
crc = crc ^ input[i];
@ -278,7 +319,25 @@ uint32_t ppc_hw(uint32_t crc, const uint8_t* input, size_t length) {
}
#endif
static bool hw_available = platform::isHwCrcSupported();
bool isHwCrcSupported() {
#if defined(_WIN32)
int info[4];
__cpuid(info, 1);
return (info[2] & (1 << 20)) != 0;
#elif defined(__aarch64__)
return true; /* force to use crc instructions */
#elif defined(__powerpc64__)
return false; /* force not to use crc instructions */
#elif defined(__unixish__)
uint32_t eax, ebx, ecx, edx, level = 1, count = 0;
__cpuid_count(level, count, eax, ebx, ecx, edx);
return ((ecx >> 20) & 1) != 0;
#else
#error Port me!
#endif
}
static bool hw_available = isHwCrcSupported();
extern "C" uint32_t crc32c_append(uint32_t crc, const uint8_t* input, size_t length) {
if (hw_available) {

View File

@ -0,0 +1,4 @@
if(UNIX AND NOT APPLE)
add_library(folly_memcpy STATIC folly_memcpy.S)
target_include_directories(folly_memcpy PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")
endif()

View File

@ -0,0 +1,2 @@
add_library(linenoise STATIC linenoise.c)
target_include_directories(linenoise PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")

View File

@ -115,7 +115,7 @@
#include <sys/types.h>
#include <sys/ioctl.h>
#include <unistd.h>
#include "linenoise.h"
#include "linenoise/linenoise.h"
#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100
#define LINENOISE_MAX_LINE 4096

View File

@ -0,0 +1,16 @@
add_library(sqlite STATIC
btree.h
hash.h
sqlite3.h
sqlite3ext.h
sqliteInt.h
sqliteLimit.h
sqlite3.amalgamation.c)
target_include_directories(sqlite PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
# Suppress warnings in sqlite since it's third party
if(NOT WIN32)
target_compile_definitions(sqlite PRIVATE $<$<CONFIG:Debug>:NDEBUG>)
target_compile_options(sqlite BEFORE PRIVATE -w) # disable warnings for third party
endif()

View File

@ -0,0 +1,11 @@
add_library(stacktrace STATIC stacktrace.amalgamation.cpp)
target_include_directories(stacktrace PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
if (USE_ASAN)
target_compile_definitions(stacktrace PRIVATE ADDRESS_SANITIZER)
elseif(USE_MSAN)
target_compile_definitions(stacktrace PRIVATE MEMORY_SANITIZER)
elseif(USE_UBSAN)
target_compile_definitions(stacktrace PRIVATE UNDEFINED_BEHAVIOR_SANITIZER)
elseif(USE_TSAN)
target_compile_definitions(stacktrace PRIVATE THREAD_SANITIZER DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL=1)
endif()

View File

@ -234,12 +234,27 @@ Note that :ref:`characters can be escaped <cli-escaping>` when specifying keys (
gettenant
---------
The ``gettenant`` command fetches metadata for a given tenant and displays it. Its syntax is ``gettenant <TENANT_NAME>``.
The ``gettenant`` command fetches metadata for a given tenant and displays it. Its syntax is ``gettenant <TENANT_NAME> [JSON]``.
Included in the output of this command are the ``id`` and ``prefix`` assigned to the tenant. If the tenant does not exist, ``fdbcli`` will report an error.
Included in the output of this command are the ``id`` and ``prefix`` assigned to the tenant. If the tenant does not exist, ``fdbcli`` will report an error. If ``JSON`` is specified, then the output will be written as a JSON document::
getversion
----------
{
"tenant": {
"id": 0,
"prefix": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"
},
"type": "success"
}
In the event of an error, the output will include an error message::
{
"error": "...",
"type": "error"
}
getversion
----------
The ``getversion`` command fetches the current read version of the cluster or currently running transaction.

View File

@ -4,6 +4,18 @@
Release Notes
#############
7.1.11
======
* Same as 7.1.10 release with AVX enabled.
7.1.10
======
* Released with AVX disabled.
* Fixed a sequencer crash when DC ID is a string. `(PR #7393) <https://github.com/apple/foundationdb/pull/7393>`_
* Fixed a client performance regression by removing unnecessary transaction initialization. `(PR #7365) <https://github.com/apple/foundationdb/pull/7365>`_
* Safely removed fdb_transaction_get_range_and_flat_map C API. `(PR #7379) <https://github.com/apple/foundationdb/pull/7379>`_
* Fixed an unknown error bug when hostname resolving fails. `(PR #7380) <https://github.com/apple/foundationdb/pull/7380>`_
7.1.9
=====
* Same as 7.1.8 release with AVX enabled.
@ -15,7 +27,7 @@ Release Notes
* Added RSS bytes for processes in status json output and corrected available_bytes calculation. `(PR #7348) <https://github.com/apple/foundationdb/pull/7348>`_
* Added versionstamp support in tuples. `(PR #7313) <https://github.com/apple/foundationdb/pull/7313>`_
* Fixed some spammy trace events. `(PR #7300) <https://github.com/apple/foundationdb/pull/7300>`_
* Fixed a memory corruption bug for using streaming peeks. `(PR #7288) <https://github.com/apple/foundationdb/pull/7288>`_
* Avoided a memory corruption bug by disabling streaming peeks. `(PR #7288) <https://github.com/apple/foundationdb/pull/7288>`_
* Fixed a hang bug in fdbcli exclude command. `(PR #7268) <https://github.com/apple/foundationdb/pull/7268>`_
* Fixed an issue that a remote TLog blocks peeks. `(PR #7255) <https://github.com/apple/foundationdb/pull/7255>`_
* Fixed a connection issue using hostnames. `(PR #7264) <https://github.com/apple/foundationdb/pull/7264>`_

View File

@ -1,23 +1,22 @@
set(FDBBACKUP_SRCS
BackupTLSConfig.h
BackupTLSConfig.cpp
backup.actor.cpp)
add_flow_target(EXECUTABLE NAME fdbbackup SRCS ${FDBBACKUP_SRCS})
target_include_directories(fdbbackup PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
target_link_libraries(fdbbackup PRIVATE fdbclient)
set(FDBCONVERT_SRCS
FileConverter.actor.cpp
FileConverter.h)
FileConverter.actor.cpp)
add_flow_target(EXECUTABLE NAME fdbconvert SRCS ${FDBCONVERT_SRCS})
target_include_directories(fdbconvert PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
target_link_libraries(fdbconvert PRIVATE fdbclient)
set(FDBDECODE_SRCS
BackupTLSConfig.h
BackupTLSConfig.cpp
FileDecoder.actor.cpp
FileConverter.h)
FileDecoder.actor.cpp)
add_flow_target(EXECUTABLE NAME fdbdecode SRCS ${FDBDECODE_SRCS})
target_include_directories(fdbdecode PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
target_link_libraries(fdbdecode PRIVATE fdbclient)
if(NOT OPEN_FOR_IDE)

View File

@ -74,7 +74,7 @@
#include "fdbclient/versions.h"
#include "fdbclient/BuildFlags.h"
#include "flow/SimpleOpt.h"
#include "SimpleOpt/SimpleOpt.h"
#include "flow/actorcompiler.h" // This must be the last #include.
// Type of program being executed

View File

@ -23,7 +23,7 @@
#pragma once
#include <cinttypes>
#include "flow/SimpleOpt.h"
#include "SimpleOpt/SimpleOpt.h"
#include "flow/TLSConfig.actor.h"
namespace file_converter {

View File

@ -1,43 +1,12 @@
set(FDBCLI_SRCS
fdbcli.actor.cpp
fdbcli.actor.h
AdvanceVersionCommand.actor.cpp
BlobRangeCommand.actor.cpp
CacheRangeCommand.actor.cpp
ConfigureCommand.actor.cpp
ConsistencyCheckCommand.actor.cpp
CoordinatorsCommand.actor.cpp
DataDistributionCommand.actor.cpp
ExcludeCommand.actor.cpp
ExpensiveDataCheckCommand.actor.cpp
FileConfigureCommand.actor.cpp
FlowLineNoise.actor.cpp
FlowLineNoise.h
ForceRecoveryWithDataLossCommand.actor.cpp
IncludeCommand.actor.cpp
KillCommand.actor.cpp
LockCommand.actor.cpp
ChangeFeedCommand.actor.cpp
MaintenanceCommand.actor.cpp
ProfileCommand.actor.cpp
SetClassCommand.actor.cpp
SnapshotCommand.actor.cpp
StatusCommand.actor.cpp
SuspendCommand.actor.cpp
TenantCommands.actor.cpp
ThrottleCommand.actor.cpp
TriggerDDTeamInfoLogCommand.actor.cpp
TssqCommand.actor.cpp
Util.actor.cpp
VersionEpochCommand.actor.cpp
linenoise/linenoise.h)
if(NOT WIN32)
list(APPEND FDBCLI_SRCS linenoise/linenoise.c)
endif()
fdb_find_sources(FDBCLI_SRCS)
add_flow_target(EXECUTABLE NAME fdbcli SRCS ${FDBCLI_SRCS})
target_link_libraries(fdbcli PRIVATE fdbclient)
target_include_directories(fdbcli PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
target_link_libraries(fdbcli PRIVATE fdbclient SimpleOpt)
if(NOT WIN32)
target_link_libraries(fdbcli PRIVATE linenoise)
endif()
if(NOT OPEN_FOR_IDE)
if(GENERATE_DEBUG_PACKAGES)

View File

@ -30,7 +30,7 @@
#if __unixish__
#define HAVE_LINENOISE 1
#include "fdbcli/linenoise/linenoise.h"
#include "linenoise/linenoise.h"
#else
#define HAVE_LINENOISE 0
#endif

View File

@ -209,11 +209,12 @@ CommandFactory listTenantsFactory(
// gettenant command
ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens) {
if (tokens.size() != 2) {
if (tokens.size() < 2 || tokens.size() > 3 || (tokens.size() == 3 && tokens[2] != "JSON"_sr)) {
printUsage(tokens[0]);
return false;
}
state bool useJson = tokens.size() == 3;
state Key tenantNameKey = fdb_cli::tenantSpecialKeyRange.begin.withSuffix(tokens[1]);
state Reference<ITransaction> tr = db->createTransaction();
@ -228,30 +229,58 @@ ACTOR Future<bool> getTenantCommandActor(Reference<IDatabase> db, std::vector<St
json_spirit::mValue jsonObject;
json_spirit::read_string(tenant.get().toString(), jsonObject);
JSONDoc doc(jsonObject);
int64_t id;
std::string prefix;
doc.get("id", id);
doc.get("prefix", prefix);
if (useJson) {
json_spirit::mObject resultObj;
resultObj["tenant"] = jsonObject;
resultObj["type"] = "success";
printf("%s\n",
json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print).c_str());
} else {
JSONDoc doc(jsonObject);
int64_t id;
std::string prefix;
doc.get("id", id);
doc.get("prefix", prefix);
printf(" id: %" PRId64 "\n", id);
printf(" prefix: %s\n", printable(prefix).c_str());
}
printf(" id: %" PRId64 "\n", id);
printf(" prefix: %s\n", printable(prefix).c_str());
return true;
} catch (Error& e) {
state Error err(e);
if (e.code() == error_code_special_keys_api_failure) {
std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr));
fprintf(stderr, "ERROR: %s\n", errorMsgStr.c_str());
return false;
try {
wait(safeThreadFutureToFuture(tr->onError(e)));
} catch (Error& finalErr) {
state std::string errorStr;
if (finalErr.code() == error_code_special_keys_api_failure) {
std::string str = wait(getSpecialKeysFailureErrorMessage(tr));
errorStr = str;
} else if (useJson) {
errorStr = finalErr.what();
} else {
throw finalErr;
}
if (useJson) {
json_spirit::mObject resultObj;
resultObj["type"] = "error";
resultObj["error"] = errorStr;
printf(
"%s\n",
json_spirit::write_string(json_spirit::mValue(resultObj), json_spirit::pretty_print).c_str());
} else {
fprintf(stderr, "ERROR: %s\n", errorStr.c_str());
}
}
wait(safeThreadFutureToFuture(tr->onError(err)));
}
}
}
CommandFactory getTenantFactory("gettenant",
CommandHelp("gettenant <TENANT_NAME>",
"prints the metadata for a tenant",
"Prints the metadata for a tenant."));
CommandFactory getTenantFactory(
"gettenant",
CommandHelp("gettenant <TENANT_NAME> [JSON]",
"prints the metadata for a tenant",
"Prints the metadata for a tenant. If JSON is specified, then the output will be in JSON format."));
} // namespace fdb_cli

View File

@ -39,6 +39,7 @@
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/TagThrottle.actor.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbclient/Tuple.h"
#include "fdbclient/ThreadSafeTransaction.h"
@ -51,7 +52,7 @@
#include "flow/TLSConfig.actor.h"
#include "flow/ThreadHelper.actor.h"
#include "flow/SimpleOpt.h"
#include "SimpleOpt/SimpleOpt.h"
#include "fdbcli/FlowLineNoise.h"
#include "fdbcli/fdbcli.actor.h"
@ -63,7 +64,7 @@
#ifdef __unixish__
#include <stdio.h>
#include "fdbcli/linenoise/linenoise.h"
#include "linenoise/linenoise.h"
#endif
#include "fdbclient/versions.h"
@ -1049,7 +1050,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
state Database localDb;
state Reference<IDatabase> db;
state Reference<ITenant> tenant;
state Optional<Standalone<StringRef>> tenantName;
state Optional<TenantName> tenantName;
state Optional<TenantMapEntry> tenantEntry;
// This tenant is kept empty for operations that perform management tasks (e.g. killing a process)
@ -1840,7 +1841,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
}
} else {
Optional<TenantMapEntry> entry =
wait(makeInterruptable(ManagementAPI::tryGetTenant(db, tokens[1])));
wait(makeInterruptable(TenantAPI::tryGetTenant(db, tokens[1])));
if (!entry.present()) {
fprintf(stderr, "ERROR: Tenant `%s' does not exist\n", printable(tokens[1]).c_str());
is_error = true;

View File

@ -1,197 +1,8 @@
set(FDBCLIENT_SRCS
ActorLineageProfiler.h
ActorLineageProfiler.cpp
AnnotateActor.h
AnnotateActor.cpp
AsyncFileS3BlobStore.actor.cpp
AsyncFileS3BlobStore.actor.h
AsyncTaskThread.actor.cpp
AsyncTaskThread.h
Atomic.h
AutoPublicAddress.cpp
BackupAgent.actor.h
BackupAgentBase.actor.cpp
BackupContainer.actor.cpp
BackupContainer.h
BackupContainerFileSystem.actor.cpp
BackupContainerFileSystem.h
BackupContainerLocalDirectory.actor.cpp
BackupContainerLocalDirectory.h
BackupContainerS3BlobStore.actor.cpp
BackupContainerS3BlobStore.h
BlobConnectionProvider.h
BlobConnectionProvider.cpp
BlobWorkerInterface.h
BlobGranuleReader.actor.cpp
BlobGranuleReader.actor.h
BlobGranuleCommon.h
BlobGranuleFiles.cpp
BlobGranuleFiles.h
BlobMetadataUtils.h
BlobWorkerCommon.h
ClientBooleanParams.cpp
ClientBooleanParams.h
ClientKnobCollection.cpp
ClientKnobCollection.h
ClientKnobs.cpp
ClientKnobs.h
ClientLogEvents.h
ClientVersion.h
ClientWorkerInterface.h
ClusterConnectionFile.actor.cpp
ClusterConnectionFile.h
ClusterConnectionKey.actor.cpp
ClusterConnectionKey.actor.h
ClusterConnectionMemoryRecord.actor.cpp
ClusterConnectionMemoryRecord.h
ClusterInterface.h
CommitProxyInterface.h
CommitTransaction.h
ConfigKnobs.cpp
ConfigKnobs.h
ConfigTransactionInterface.cpp
ConfigTransactionInterface.h
ConvertUTF.h
CoordinationInterface.h
DatabaseBackupAgent.actor.cpp
DatabaseConfiguration.cpp
DatabaseConfiguration.h
DatabaseContext.h
EventTypes.actor.h
FDBOptions.h
FDBTypes.cpp
FDBTypes.h
FluentDSampleIngestor.cpp
FileBackupAgent.actor.cpp
GenericManagementAPI.actor.h
GlobalConfig.h
GlobalConfig.actor.h
GlobalConfig.actor.cpp
GrvProxyInterface.h
HTTP.h
HTTP.actor.cpp
HighContentionPrefixAllocator.actor.h
IClientApi.h
IConfigTransaction.cpp
IConfigTransaction.h
ISingleThreadTransaction.cpp
ISingleThreadTransaction.h
JSONDoc.h
JsonBuilder.cpp
JsonBuilder.h
KeyBackedTypes.h
KeyRangeMap.actor.cpp
KeyRangeMap.h
Knobs.h
IKnobCollection.cpp
IKnobCollection.h
LocalClientAPI.cpp
LocalClientAPI.h
ManagementAPI.actor.cpp
ManagementAPI.actor.h
MonitorLeader.actor.cpp
MonitorLeader.h
MultiVersionAssignmentVars.h
MultiVersionTransaction.actor.cpp
MultiVersionTransaction.h
MutationList.h
MutationLogReader.actor.cpp
MutationLogReader.actor.h
NameLineage.h
NameLineage.cpp
NativeAPI.actor.cpp
NativeAPI.actor.h
Notified.h
ParallelStream.actor.cpp
ParallelStream.actor.h
PaxosConfigTransaction.actor.cpp
PaxosConfigTransaction.h
PImpl.h
ProcessInterface.h
SimpleConfigTransaction.actor.cpp
SpecialKeySpace.actor.cpp
SpecialKeySpace.actor.h
RESTClient.h
RESTClient.actor.cpp
RESTUtils.h
RESTUtils.actor.cpp
ReadYourWrites.actor.cpp
ReadYourWrites.h
RestoreInterface.cpp
RestoreInterface.h
RunTransaction.actor.h
RYWIterator.cpp
RYWIterator.h
S3BlobStore.h
S3BlobStore.actor.cpp
Schemas.cpp
Schemas.h
ServerKnobCollection.cpp
ServerKnobCollection.h
ServerKnobs.cpp
ServerKnobs.h
SimpleConfigTransaction.h
SimpleIni.h
SnapshotCache.h
SpecialKeySpace.actor.cpp
SpecialKeySpace.actor.h
Status.h
StatusClient.actor.cpp
StatusClient.h
StorageServerInterface.cpp
StorageServerInterface.h
StorageCheckpoint.h
Subspace.cpp
Subspace.h
StackLineage.h
StackLineage.cpp
SystemData.cpp
SystemData.h
TagThrottle.actor.cpp
TagThrottle.actor.h
TaskBucket.actor.cpp
TaskBucket.h
Tenant.cpp
Tenant.h
TestKnobCollection.cpp
TestKnobCollection.h
ThreadSafeTransaction.cpp
ThreadSafeTransaction.h
Tracing.h
Tracing.actor.cpp
TransactionLineage.h
Tuple.cpp
Tuple.h
VersionedMap.actor.h
VersionedMap.h
VersionedMap.cpp
Versionstamp.cpp
Versionstamp.h
VersionVector.h
VersionVector.cpp
WriteMap.h
WriteMap.cpp
json_spirit/json_spirit_error_position.h
json_spirit/json_spirit_reader_template.h
json_spirit/json_spirit_value.h
json_spirit/json_spirit_writer_options.h
json_spirit/json_spirit_writer_template.h
libb64/encode.h
libb64/decode.h
libb64/cdecode.h
libb64/cdecode.c
libb64/cencode.h
libb64/cencode.c
md5/md5.h
md5/md5.c
rapidxml/rapidxml.hpp
rapidxml/rapidxml_iterators.hpp
rapidxml/rapidxml_print.hpp
rapidxml/rapidxml_utils.hpp
sha1/SHA1.h
fdb_find_sources(FDBCLIENT_SRCS)
list(APPEND FDBCLIENT_SRCS
sha1/SHA1.cpp
zipf.c
zipf.h)
libb64/cdecode.c
libb64/cencode.c)
message(STATUS "FDB version is ${FDB_VERSION}")
message(STATUS "FDB package name is ${FDB_PACKAGE_NAME}")
@ -224,8 +35,7 @@ if(BUILD_AZURE_BACKUP)
add_compile_definitions(BUILD_AZURE_BACKUP)
set(FDBCLIENT_SRCS
${FDBCLIENT_SRCS}
BackupContainerAzureBlobStore.actor.cpp
BackupContainerAzureBlobStore.h)
azure_backup/BackupContainerAzureBlobStore.actor.cpp)
configure_file(azurestorage.cmake azurestorage-download/CMakeLists.txt)
@ -267,6 +77,7 @@ if(WITH_AWS_BACKUP)
endif()
add_flow_target(STATIC_LIBRARY NAME fdbclient SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
target_include_directories(fdbclient PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/versions.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/versions.h)
add_dependencies(fdbclient fdboptions)
target_link_libraries(fdbclient PUBLIC fdbrpc msgpack)
@ -275,6 +86,7 @@ target_link_libraries(fdbclient PUBLIC fdbrpc msgpack)
# fdbserver retain sampling functionality in client code while disabling
# sampling for pure clients.
add_flow_target(STATIC_LIBRARY NAME fdbclient_sampling SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
target_include_directories(fdbclient_sampling PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include")
add_dependencies(fdbclient_sampling fdboptions)
target_link_libraries(fdbclient_sampling PUBLIC fdbrpc_sampling msgpack)
target_compile_definitions(fdbclient_sampling PRIVATE -DENABLE_SAMPLING)

View File

@ -637,8 +637,7 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
parse((&type), value);
storageMigrationType = (StorageMigrationType::MigrationType)type;
} else if (ck == LiteralStringRef("tenant_mode")) {
parse((&type), value);
tenantMode = (TenantMode::Mode)type;
tenantMode = TenantMode::fromValue(value);
} else if (ck == LiteralStringRef("proxies")) {
overwriteProxiesCount();
} else if (ck == LiteralStringRef("blob_granules_enabled")) {

View File

@ -1340,7 +1340,7 @@ bool MultiVersionTransaction::isValid() {
}
// MultiVersionTenant
MultiVersionTenant::MultiVersionTenant(Reference<MultiVersionDatabase> db, StringRef tenantName)
MultiVersionTenant::MultiVersionTenant(Reference<MultiVersionDatabase> db, TenantNameRef tenantName)
: tenantState(makeReference<TenantState>(db, tenantName)) {}
MultiVersionTenant::~MultiVersionTenant() {
@ -1363,7 +1363,7 @@ ThreadFuture<Void> MultiVersionTenant::waitPurgeGranulesComplete(const KeyRef& p
return abortableFuture(f, tenantState->db->dbState->dbVar->get().onChange);
}
MultiVersionTenant::TenantState::TenantState(Reference<MultiVersionDatabase> db, StringRef tenantName)
MultiVersionTenant::TenantState::TenantState(Reference<MultiVersionDatabase> db, TenantNameRef tenantName)
: tenantVar(new ThreadSafeAsyncVar<Reference<ITenant>>(Reference<ITenant>(nullptr))), tenantName(tenantName), db(db),
closed(false) {
updateTenant();

View File

@ -7015,7 +7015,7 @@ ACTOR Future<ProtocolVersion> getClusterProtocolImpl(
} else {
state NetworkAddress coordinatorAddress;
if (coordinator->get().get().hostname.present()) {
Hostname h = coordinator->get().get().hostname.get();
state Hostname h = coordinator->get().get().hostname.get();
wait(store(coordinatorAddress, h.resolveWithRetry()));
} else {
coordinatorAddress = coordinator->get().get().getLeader.getEndpoint().getPrimaryAddress();
@ -9588,3 +9588,27 @@ int64_t getMaxWriteKeySize(KeyRef const& key, bool hasRawAccess) {
int64_t getMaxClearKeySize(KeyRef const& key) {
return getMaxKeySize(key);
}
namespace NativeAPI {
ACTOR Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> getServerListAndProcessClasses(
Transaction* tr) {
state Future<std::vector<ProcessData>> workers = getWorkers(tr);
state Future<RangeResult> serverList = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
wait(success(workers) && success(serverList));
ASSERT(!serverList.get().more && serverList.get().size() < CLIENT_KNOBS->TOO_MANY);
std::map<Optional<Standalone<StringRef>>, ProcessData> id_data;
for (int i = 0; i < workers.get().size(); i++)
id_data[workers.get()[i].locality.processId()] = workers.get()[i];
std::vector<std::pair<StorageServerInterface, ProcessClass>> results;
for (int i = 0; i < serverList.get().size(); i++) {
auto ssi = decodeServerListValue(serverList.get()[i].value);
results.emplace_back(ssi, id_data[ssi.locality.processId()].processClass);
}
return results;
}
} // namespace NativeAPI

View File

@ -801,7 +801,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( FASTRESTORE_NUM_LOADERS, 3 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_LOADERS = deterministicRandom()->random01() * 10 + 1; }
init( FASTRESTORE_NUM_APPLIERS, 3 ); if( randomize && BUGGIFY ) { FASTRESTORE_NUM_APPLIERS = deterministicRandom()->random01() * 10 + 1; }
init( FASTRESTORE_TXN_BATCH_MAX_BYTES, 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_TXN_BATCH_MAX_BYTES = deterministicRandom()->random01() * 1024.0 * 1024.0 + 1.0; }
init( FASTRESTORE_VERSIONBATCH_MAX_BYTES, 10.0 * 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VERSIONBATCH_MAX_BYTES = deterministicRandom()->random01() < 0.2 ? 10 * 1024 : deterministicRandom()->random01() < 0.4 ? 100 * 1024 * 1024 : deterministicRandom()->random01() * 1000.0 * 1024.0 * 1024.0; } // too small value may increase chance of TooManyFile error
init( FASTRESTORE_VERSIONBATCH_MAX_BYTES, 10.0 * 1024.0 * 1024.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VERSIONBATCH_MAX_BYTES = deterministicRandom()->random01() < 0.2 ? 50 * 1024 : deterministicRandom()->random01() < 0.4 ? 100 * 1024 * 1024 : deterministicRandom()->random01() * 1000.0 * 1024.0 * 1024.0; } // too small value may increase chance of TooManyFile error
init( FASTRESTORE_VB_PARALLELISM, 5 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_PARALLELISM = deterministicRandom()->random01() < 0.2 ? 2 : deterministicRandom()->random01() * 10 + 1; }
init( FASTRESTORE_VB_MONITOR_DELAY, 30 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_MONITOR_DELAY = deterministicRandom()->random01() * 20 + 1; }
init( FASTRESTORE_VB_LAUNCH_DELAY, 1.0 ); if( randomize && BUGGIFY ) { FASTRESTORE_VB_LAUNCH_DELAY = deterministicRandom()->random01() < 0.2 ? 0.1 : deterministicRandom()->random01() * 10.0 + 1; }

View File

@ -2730,123 +2730,3 @@ Future<Optional<std::string>> FailedLocalitiesRangeImpl::commit(ReadYourWritesTr
// exclude locality with failed option as true.
return excludeLocalityCommitActor(ryw, true);
}
ACTOR Future<RangeResult> getTenantList(ReadYourWritesTransaction* ryw, KeyRangeRef kr, GetRangeLimits limitsHint) {
state KeyRef managementPrefix =
kr.begin.substr(0,
SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin.size() +
TenantMapRangeImpl::submoduleRange.begin.size());
kr = kr.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
TenantNameRef beginTenant = kr.begin.removePrefix(TenantMapRangeImpl::submoduleRange.begin);
TenantNameRef endTenant = kr.end;
if (endTenant.startsWith(TenantMapRangeImpl::submoduleRange.begin)) {
endTenant = endTenant.removePrefix(TenantMapRangeImpl::submoduleRange.begin);
} else {
endTenant = "\xff"_sr;
}
std::map<TenantName, TenantMapEntry> tenants =
wait(ManagementAPI::listTenantsTransaction(&ryw->getTransaction(), beginTenant, endTenant, limitsHint.rows));
RangeResult results;
for (auto tenant : tenants) {
json_spirit::mObject tenantEntry;
tenantEntry["id"] = tenant.second.id;
tenantEntry["prefix"] = tenant.second.prefix.toString();
std::string tenantEntryString = json_spirit::write_string(json_spirit::mValue(tenantEntry));
ValueRef tenantEntryBytes(results.arena(), tenantEntryString);
results.push_back(results.arena(),
KeyValueRef(tenant.first.withPrefix(managementPrefix, results.arena()), tenantEntryBytes));
}
return results;
}
TenantMapRangeImpl::TenantMapRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
Future<RangeResult> TenantMapRangeImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr,
GetRangeLimits limitsHint) const {
return getTenantList(ryw, kr, limitsHint);
}
ACTOR Future<Void> createTenants(ReadYourWritesTransaction* ryw, std::vector<TenantNameRef> tenants) {
Optional<Value> lastIdVal = wait(ryw->getTransaction().get(tenantLastIdKey));
int64_t previousId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) : -1;
std::vector<Future<Void>> createFutures;
for (auto tenant : tenants) {
createFutures.push_back(
success(ManagementAPI::createTenantTransaction(&ryw->getTransaction(), tenant, ++previousId)));
}
ryw->getTransaction().set(tenantLastIdKey, TenantMapEntry::idToPrefix(previousId));
wait(waitForAll(createFutures));
return Void();
}
ACTOR Future<Void> deleteTenantRange(ReadYourWritesTransaction* ryw,
TenantNameRef beginTenant,
TenantNameRef endTenant) {
std::map<TenantName, TenantMapEntry> tenants = wait(
ManagementAPI::listTenantsTransaction(&ryw->getTransaction(), beginTenant, endTenant, CLIENT_KNOBS->TOO_MANY));
if (tenants.size() == CLIENT_KNOBS->TOO_MANY) {
TraceEvent(SevWarn, "DeleteTenantRangeTooLange")
.detail("BeginTenant", beginTenant)
.detail("EndTenant", endTenant);
ryw->setSpecialKeySpaceErrorMsg("too many tenants to range delete");
throw special_keys_api_failure();
}
std::vector<Future<Void>> deleteFutures;
for (auto tenant : tenants) {
deleteFutures.push_back(ManagementAPI::deleteTenantTransaction(&ryw->getTransaction(), tenant.first));
}
wait(waitForAll(deleteFutures));
return Void();
}
Future<Optional<std::string>> TenantMapRangeImpl::commit(ReadYourWritesTransaction* ryw) {
auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range);
std::vector<TenantNameRef> tenantsToCreate;
std::vector<Future<Void>> tenantManagementFutures;
for (auto range : ranges) {
if (!range.value().first) {
continue;
}
TenantNameRef tenantName =
range.begin()
.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)
.removePrefix(TenantMapRangeImpl::submoduleRange.begin);
if (range.value().second.present()) {
tenantsToCreate.push_back(tenantName);
} else {
// For a single key clear, just issue the delete
if (KeyRangeRef(range.begin(), range.end()).singleKeyRange()) {
tenantManagementFutures.push_back(
ManagementAPI::deleteTenantTransaction(&ryw->getTransaction(), tenantName));
} else {
TenantNameRef endTenant = range.end().removePrefix(
SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
if (endTenant.startsWith(submoduleRange.begin)) {
endTenant = endTenant.removePrefix(submoduleRange.begin);
} else {
endTenant = "\xff"_sr;
}
tenantManagementFutures.push_back(deleteTenantRange(ryw, tenantName, endTenant));
}
}
}
if (tenantsToCreate.size()) {
tenantManagementFutures.push_back(createTenants(ryw, tenantsToCreate));
}
return tag(waitForAll(tenantManagementFutures), Optional<std::string>());
}

View File

@ -21,7 +21,7 @@
// TODO this should really be renamed "TSSComparison.cpp"
#include "fdbclient/StorageServerInterface.h"
#include "fdbclient/BlobWorkerInterface.h"
#include "flow/crc32c.h" // for crc32c_append, to checksum values in tss trace events
#include "crc32/crc32c.h" // for crc32c_append, to checksum values in tss trace events
// Includes template specializations for all tss operations on storage server types.
// New StorageServerInterface reply types must be added here or it won't compile.

View File

@ -22,6 +22,35 @@
#include "fdbclient/Tenant.h"
#include "flow/UnitTest.h"
Key TenantMapEntry::idToPrefix(int64_t id) {
int64_t swapped = bigEndian64(id);
return StringRef(reinterpret_cast<const uint8_t*>(&swapped), 8);
}
int64_t TenantMapEntry::prefixToId(KeyRef prefix) {
ASSERT(prefix.size() == 8);
int64_t id = *reinterpret_cast<const int64_t*>(prefix.begin());
id = bigEndian64(id);
ASSERT(id >= 0);
return id;
}
void TenantMapEntry::initPrefix(KeyRef subspace) {
ASSERT(id >= 0);
prefix = makeString(8 + subspace.size());
uint8_t* data = mutateString(prefix);
if (subspace.size() > 0) {
memcpy(data, subspace.begin(), subspace.size());
}
int64_t swapped = bigEndian64(id);
memcpy(data + subspace.size(), &swapped, 8);
}
TenantMapEntry::TenantMapEntry() : id(-1) {}
TenantMapEntry::TenantMapEntry(int64_t id, KeyRef subspace) : id(id) {
initPrefix(subspace);
}
TEST_CASE("/fdbclient/TenantMapEntry/Serialization") {
TenantMapEntry entry1(1, ""_sr);
ASSERT(entry1.prefix == "\x00\x00\x00\x00\x00\x00\x00\x01"_sr);

View File

@ -0,0 +1,149 @@
/*
* TenantSpecialKeys.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbclient/ActorLineageProfiler.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/Knobs.h"
#include "fdbclient/DatabaseContext.h"
#include "fdbclient/SpecialKeySpace.actor.h"
#include "fdbclient/TenantManagement.actor.h"
#include "flow/Arena.h"
#include "flow/UnitTest.h"
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR Future<RangeResult> getTenantList(ReadYourWritesTransaction* ryw, KeyRangeRef kr, GetRangeLimits limitsHint) {
state KeyRef managementPrefix =
kr.begin.substr(0,
SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin.size() +
TenantMapRangeImpl::submoduleRange.begin.size());
kr = kr.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
TenantNameRef beginTenant = kr.begin.removePrefix(TenantMapRangeImpl::submoduleRange.begin);
TenantNameRef endTenant = kr.end;
if (endTenant.startsWith(TenantMapRangeImpl::submoduleRange.begin)) {
endTenant = endTenant.removePrefix(TenantMapRangeImpl::submoduleRange.begin);
} else {
endTenant = "\xff"_sr;
}
std::map<TenantName, TenantMapEntry> tenants =
wait(TenantAPI::listTenantsTransaction(&ryw->getTransaction(), beginTenant, endTenant, limitsHint.rows));
RangeResult results;
for (auto tenant : tenants) {
json_spirit::mObject tenantEntry;
tenantEntry["id"] = tenant.second.id;
tenantEntry["prefix"] = tenant.second.prefix.toString();
std::string tenantEntryString = json_spirit::write_string(json_spirit::mValue(tenantEntry));
ValueRef tenantEntryBytes(results.arena(), tenantEntryString);
results.push_back(results.arena(),
KeyValueRef(tenant.first.withPrefix(managementPrefix, results.arena()), tenantEntryBytes));
}
return results;
}
TenantMapRangeImpl::TenantMapRangeImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
Future<RangeResult> TenantMapRangeImpl::getRange(ReadYourWritesTransaction* ryw,
KeyRangeRef kr,
GetRangeLimits limitsHint) const {
return getTenantList(ryw, kr, limitsHint);
}
ACTOR Future<Void> createTenants(ReadYourWritesTransaction* ryw, std::vector<TenantNameRef> tenants) {
Optional<Value> lastIdVal = wait(ryw->getTransaction().get(tenantLastIdKey));
int64_t previousId = lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) : -1;
std::vector<Future<Void>> createFutures;
for (auto tenant : tenants) {
createFutures.push_back(
success(TenantAPI::createTenantTransaction(&ryw->getTransaction(), tenant, ++previousId)));
}
ryw->getTransaction().set(tenantLastIdKey, TenantMapEntry::idToPrefix(previousId));
wait(waitForAll(createFutures));
return Void();
}
ACTOR Future<Void> deleteTenantRange(ReadYourWritesTransaction* ryw,
TenantNameRef beginTenant,
TenantNameRef endTenant) {
std::map<TenantName, TenantMapEntry> tenants =
wait(TenantAPI::listTenantsTransaction(&ryw->getTransaction(), beginTenant, endTenant, CLIENT_KNOBS->TOO_MANY));
if (tenants.size() == CLIENT_KNOBS->TOO_MANY) {
TraceEvent(SevWarn, "DeleteTenantRangeTooLange")
.detail("BeginTenant", beginTenant)
.detail("EndTenant", endTenant);
ryw->setSpecialKeySpaceErrorMsg("too many tenants to range delete");
throw special_keys_api_failure();
}
std::vector<Future<Void>> deleteFutures;
for (auto tenant : tenants) {
deleteFutures.push_back(TenantAPI::deleteTenantTransaction(&ryw->getTransaction(), tenant.first));
}
wait(waitForAll(deleteFutures));
return Void();
}
Future<Optional<std::string>> TenantMapRangeImpl::commit(ReadYourWritesTransaction* ryw) {
auto ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(range);
std::vector<TenantNameRef> tenantsToCreate;
std::vector<Future<Void>> tenantManagementFutures;
for (auto range : ranges) {
if (!range.value().first) {
continue;
}
TenantNameRef tenantName =
range.begin()
.removePrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin)
.removePrefix(TenantMapRangeImpl::submoduleRange.begin);
if (range.value().second.present()) {
tenantsToCreate.push_back(tenantName);
} else {
// For a single key clear, just issue the delete
if (KeyRangeRef(range.begin(), range.end()).singleKeyRange()) {
tenantManagementFutures.push_back(
TenantAPI::deleteTenantTransaction(&ryw->getTransaction(), tenantName));
} else {
TenantNameRef endTenant = range.end().removePrefix(
SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin);
if (endTenant.startsWith(submoduleRange.begin)) {
endTenant = endTenant.removePrefix(submoduleRange.begin);
} else {
endTenant = "\xff"_sr;
}
tenantManagementFutures.push_back(deleteTenantRange(ryw, tenantName, endTenant));
}
}
}
if (tenantsToCreate.size()) {
tenantManagementFutures.push_back(createTenants(ryw, tenantsToCreate));
}
return tag(waitForAll(tenantManagementFutures), Optional<std::string>());
}

View File

@ -175,7 +175,7 @@ Reference<ITransaction> ThreadSafeTenant::createTransaction() {
ThreadFuture<Key> ThreadSafeTenant::purgeBlobGranules(const KeyRangeRef& keyRange, Version purgeVersion, bool force) {
DatabaseContext* db = this->db->db;
Standalone<StringRef> tenantName = this->name;
TenantName tenantName = this->name;
KeyRange range = keyRange;
return onMainThread([db, range, purgeVersion, tenantName, force]() -> Future<Key> {
return db->purgeBlobGranules(range, purgeVersion, tenantName, force);

Some files were not shown because too many files have changed in this diff Show More