wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:apple/foundationdb into split-tenant-metadata

This commit is contained in:
Jon Fu 2023-02-21 10:11:35 -08:00
parent 8e6800663b 2497aa5701
commit 428eb07766
107 changed files with 3299 additions and 1413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -97,7 +97,10 @@ else()
set(FDB_VERSION ${PROJECT_VERSION}) set(FDB_VERSION ${PROJECT_VERSION})
endif() endif()
if (NOT FDB_RELEASE) if (NOT FDB_RELEASE)
string(TIMESTAMP FDB_BUILDTIME %Y%m%d%H%M%S) string(TIMESTAMP FDB_BUILD_TIMESTMAP %Y%m%d%H%M%S)
# Adding support to pass custom fdb_build timestamp,
# required to achieve uniform naming across different builds
set(FDB_BUILDTIME "${FDB_BUILD_TIMESTMAP}" CACHE STRING "A timestamp for packages")
set(FDB_BUILDTIME_STRING ".${FDB_BUILDTIME}") set(FDB_BUILDTIME_STRING ".${FDB_BUILDTIME}")
set(PRERELEASE_TAG "prerelease") set(PRERELEASE_TAG "prerelease")
endif() endif()

397
bindings/bindingtester/tests/api.py
View File

@ -33,23 +33,32 @@ fdb.api_version(FDB_API_VERSION)
def matches_op(op, target_op): def matches_op(op, target_op):
return op == target_op or op == target_op + '_SNAPSHOT' or op == target_op + '_DATABASE' or op == target_op + '_TENANT' return (
op == target_op
or op == target_op + "_SNAPSHOT"
or op == target_op + "_DATABASE"
or op == target_op + "_TENANT"
)
def is_non_transaction_op(op): def is_non_transaction_op(op):
return op.endswith('_DATABASE') or op.endswith('_TENANT') return op.endswith("_DATABASE") or op.endswith("_TENANT")
class ApiTest(Test): class ApiTest(Test):
def __init__(self, subspace): def __init__(self, subspace):
super(ApiTest, self).__init__(subspace) super(ApiTest, self).__init__(subspace)
self.workspace = self.subspace['workspace'] # The keys and values here must match between subsequent runs of the same test self.workspace = self.subspace[
self.scratch = self.subspace['scratch'] # The keys and values here can differ between runs "workspace"
self.stack_subspace = self.subspace['stack'] ] # The keys and values here must match between subsequent runs of the same test
self.scratch = self.subspace[
"scratch"
] # The keys and values here can differ between runs
self.stack_subspace = self.subspace["stack"]
self.versionstamped_values = self.scratch['versionstamped_values'] self.versionstamped_values = self.scratch["versionstamped_values"]
self.versionstamped_values_2 = self.scratch['versionstamped_values_2'] self.versionstamped_values_2 = self.scratch["versionstamped_values_2"]
self.versionstamped_keys = self.scratch['versionstamped_keys'] self.versionstamped_keys = self.scratch["versionstamped_keys"]
def setup(self, args): def setup(self, args):
self.stack_size = 0 self.stack_size = 0
@ -64,7 +73,9 @@ class ApiTest(Test):
self.generated_keys = [] self.generated_keys = []
self.outstanding_ops = [] self.outstanding_ops = []
self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version, args.types) self.random = test_util.RandomGenerator(
args.max_int_bits, args.api_version, args.types
)
self.api_version = args.api_version self.api_version = args.api_version
self.allocated_tenants = set() self.allocated_tenants = set()
@ -88,7 +99,9 @@ class ApiTest(Test):
self.string_depth = max(0, self.string_depth - num) self.string_depth = max(0, self.string_depth - num)
self.key_depth = max(0, self.key_depth - num) self.key_depth = max(0, self.key_depth - num)
self.outstanding_ops = [i for i in self.outstanding_ops if i[0] <= self.stack_size] self.outstanding_ops = [
i for i in self.outstanding_ops if i[0] <= self.stack_size
]
def ensure_string(self, instructions, num): def ensure_string(self, instructions, num):
while self.string_depth < num: while self.string_depth < num:
@ -101,7 +114,7 @@ class ApiTest(Test):
if random.random() < float(len(self.generated_keys)) / self.max_keys: if random.random() < float(len(self.generated_keys)) / self.max_keys:
tup = random.choice(self.generated_keys) tup = random.choice(self.generated_keys)
if random.random() < 0.3: if random.random() < 0.3:
return self.workspace.pack(tup[0:random.randint(0, len(tup))]) return self.workspace.pack(tup[0 : random.randint(0, len(tup))])
return self.workspace.pack(tup) return self.workspace.pack(tup)
@ -119,7 +132,9 @@ class ApiTest(Test):
def ensure_key_value(self, instructions): def ensure_key_value(self, instructions):
if self.string_depth == 0: if self.string_depth == 0:
instructions.push_args(self.choose_key(), self.random.random_string(random.randint(0, 100))) instructions.push_args(
self.choose_key(), self.random.random_string(random.randint(0, 100))
)
elif self.string_depth == 1 or self.key_depth == 0: elif self.string_depth == 1 or self.key_depth == 0:
self.ensure_key(instructions, 1) self.ensure_key(instructions, 1)
@ -131,7 +146,7 @@ class ApiTest(Test):
def preload_database(self, instructions, num): def preload_database(self, instructions, num):
for i in range(num): for i in range(num):
self.ensure_key_value(instructions) self.ensure_key_value(instructions)
instructions.append('SET') instructions.append("SET")
if i % 100 == 99: if i % 100 == 99:
test_util.blocking_commit(instructions) test_util.blocking_commit(instructions)
@ -140,42 +155,81 @@ class ApiTest(Test):
self.add_stack_items(1) self.add_stack_items(1)
def wait_for_reads(self, instructions): def wait_for_reads(self, instructions):
while len(self.outstanding_ops) > 0 and self.outstanding_ops[-1][0] <= self.stack_size: while (
len(self.outstanding_ops) > 0
and self.outstanding_ops[-1][0] <= self.stack_size
):
read = self.outstanding_ops.pop() read = self.outstanding_ops.pop()
# print '%d. waiting for read at instruction %r' % (len(instructions), read) # print '%d. waiting for read at instruction %r' % (len(instructions), read)
test_util.to_front(instructions, self.stack_size - read[0]) test_util.to_front(instructions, self.stack_size - read[0])
instructions.append('WAIT_FUTURE') instructions.append("WAIT_FUTURE")
def choose_tenant(self, new_tenant_probability): def choose_tenant(self, new_tenant_probability):
if len(self.allocated_tenants) == 0 or random.random() < new_tenant_probability: if len(self.allocated_tenants) == 0 or random.random() < new_tenant_probability:
return self.random.random_string(random.randint(0, 30)) return self.random.random_string(random.randint(0, 30))
else: else:
return random.choice(list(self.allocated_tenants)) tenant_list = list(self.allocated_tenants)
# sort to ensure deterministic selection of a tenant
tenant_list.sort()
return random.choice(tenant_list)
def generate(self, args, thread_number): def generate(self, args, thread_number):
instructions = InstructionSet() instructions = InstructionSet()
op_choices = ['NEW_TRANSACTION', 'COMMIT'] op_choices = ["NEW_TRANSACTION", "COMMIT"]
reads = ['GET', 'GET_KEY', 'GET_RANGE', 'GET_RANGE_STARTS_WITH', 'GET_RANGE_SELECTOR'] reads = [
mutations = ['SET', 'CLEAR', 'CLEAR_RANGE', 'CLEAR_RANGE_STARTS_WITH', 'ATOMIC_OP'] "GET",
snapshot_reads = [x + '_SNAPSHOT' for x in reads] "GET_KEY",
database_reads = [x + '_DATABASE' for x in reads] "GET_RANGE",
database_mutations = [x + '_DATABASE' for x in mutations] "GET_RANGE_STARTS_WITH",
tenant_reads = [x + '_TENANT' for x in reads] "GET_RANGE_SELECTOR",
tenant_mutations = [x + '_TENANT' for x in mutations] ]
mutations += ['VERSIONSTAMP'] mutations = [
versions = ['GET_READ_VERSION', 'SET_READ_VERSION', 'GET_COMMITTED_VERSION'] "SET",
snapshot_versions = ['GET_READ_VERSION_SNAPSHOT'] "CLEAR",
tuples = ['TUPLE_PACK', 'TUPLE_UNPACK', 'TUPLE_RANGE', 'TUPLE_SORT', 'SUB', 'ENCODE_FLOAT', 'ENCODE_DOUBLE', 'DECODE_DOUBLE', 'DECODE_FLOAT'] "CLEAR_RANGE",
if 'versionstamp' in args.types: "CLEAR_RANGE_STARTS_WITH",
tuples.append('TUPLE_PACK_WITH_VERSIONSTAMP') "ATOMIC_OP",
resets = ['ON_ERROR', 'RESET', 'CANCEL'] ]
read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY'] snapshot_reads = [x + "_SNAPSHOT" for x in reads]
write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT'] database_reads = [x + "_DATABASE" for x in reads]
txn_sizes = ['GET_APPROXIMATE_SIZE'] database_mutations = [x + "_DATABASE" for x in mutations]
storage_metrics = ['GET_ESTIMATED_RANGE_SIZE', 'GET_RANGE_SPLIT_POINTS'] tenant_reads = [x + "_TENANT" for x in reads]
tenants = ['TENANT_CREATE', 'TENANT_DELETE', 'TENANT_SET_ACTIVE', 'TENANT_CLEAR_ACTIVE', 'TENANT_LIST', 'TENANT_GET_ID'] tenant_mutations = [x + "_TENANT" for x in mutations]
mutations += ["VERSIONSTAMP"]
versions = ["GET_READ_VERSION", "SET_READ_VERSION", "GET_COMMITTED_VERSION"]
snapshot_versions = ["GET_READ_VERSION_SNAPSHOT"]
tuples = [
"TUPLE_PACK",
"TUPLE_UNPACK",
"TUPLE_RANGE",
"TUPLE_SORT",
"SUB",
"ENCODE_FLOAT",
"ENCODE_DOUBLE",
"DECODE_DOUBLE",
"DECODE_FLOAT",
]
if "versionstamp" in args.types:
tuples.append("TUPLE_PACK_WITH_VERSIONSTAMP")
resets = ["ON_ERROR", "RESET", "CANCEL"]
read_conflicts = ["READ_CONFLICT_RANGE", "READ_CONFLICT_KEY"]
write_conflicts = [
"WRITE_CONFLICT_RANGE",
"WRITE_CONFLICT_KEY",
"DISABLE_WRITE_CONFLICT",
]
txn_sizes = ["GET_APPROXIMATE_SIZE"]
storage_metrics = ["GET_ESTIMATED_RANGE_SIZE", "GET_RANGE_SPLIT_POINTS"]
tenants = [
"TENANT_CREATE",
"TENANT_DELETE",
"TENANT_SET_ACTIVE",
"TENANT_CLEAR_ACTIVE",
"TENANT_LIST",
"TENANT_GET_ID",
]
op_choices += reads op_choices += reads
op_choices += mutations op_choices += mutations
@ -196,16 +250,23 @@ class ApiTest(Test):
op_choices += tenant_reads op_choices += tenant_reads
op_choices += tenant_mutations op_choices += tenant_mutations
idempotent_atomic_ops = ['BIT_AND', 'BIT_OR', 'MAX', 'MIN', 'BYTE_MIN', 'BYTE_MAX'] idempotent_atomic_ops = [
atomic_ops = idempotent_atomic_ops + ['ADD', 'BIT_XOR', 'APPEND_IF_FITS'] "BIT_AND",
"BIT_OR",
"MAX",
"MIN",
"BYTE_MIN",
"BYTE_MAX",
]
atomic_ops = idempotent_atomic_ops + ["ADD", "BIT_XOR", "APPEND_IF_FITS"]
if args.concurrency > 1: if args.concurrency > 1:
self.max_keys = random.randint(100, 1000) self.max_keys = random.randint(100, 1000)
else: else:
self.max_keys = random.randint(100, 10000) self.max_keys = random.randint(100, 10000)
instructions.append('NEW_TRANSACTION') instructions.append("NEW_TRANSACTION")
instructions.append('GET_READ_VERSION') instructions.append("GET_READ_VERSION")
self.preload_database(instructions, self.max_keys) self.preload_database(instructions, self.max_keys)
@ -218,25 +279,29 @@ class ApiTest(Test):
# print 'Adding instruction %s at %d' % (op, index) # print 'Adding instruction %s at %d' % (op, index)
if args.concurrency == 1 and (op in database_mutations or op in tenant_mutations or op in ['TENANT_CREATE', 'TENANT_DELETE']): if args.concurrency == 1 and (
op in database_mutations
or op in tenant_mutations
or op in ["TENANT_CREATE", "TENANT_DELETE"]
):
self.wait_for_reads(instructions) self.wait_for_reads(instructions)
test_util.blocking_commit(instructions) test_util.blocking_commit(instructions)
self.can_get_commit_version = False self.can_get_commit_version = False
self.add_stack_items(1) self.add_stack_items(1)
if op in resets or op == 'NEW_TRANSACTION': if op in resets or op == "NEW_TRANSACTION":
if args.concurrency == 1: if args.concurrency == 1:
self.wait_for_reads(instructions) self.wait_for_reads(instructions)
self.outstanding_ops = [] self.outstanding_ops = []
if op == 'NEW_TRANSACTION': if op == "NEW_TRANSACTION":
instructions.append(op) instructions.append(op)
self.can_get_commit_version = True self.can_get_commit_version = True
self.can_set_version = True self.can_set_version = True
self.can_use_key_selectors = True self.can_use_key_selectors = True
elif op == 'ON_ERROR': elif op == "ON_ERROR":
instructions.push_args(random.randint(0, 5000)) instructions.push_args(random.randint(0, 5000))
instructions.append(op) instructions.append(op)
@ -244,20 +309,20 @@ class ApiTest(Test):
if args.concurrency == 1: if args.concurrency == 1:
self.wait_for_reads(instructions) self.wait_for_reads(instructions)
instructions.append('NEW_TRANSACTION') instructions.append("NEW_TRANSACTION")
self.can_get_commit_version = True self.can_get_commit_version = True
self.can_set_version = True self.can_set_version = True
self.can_use_key_selectors = True self.can_use_key_selectors = True
self.add_strings(1) self.add_strings(1)
elif matches_op(op, 'GET'): elif matches_op(op, "GET"):
self.ensure_key(instructions, 1) self.ensure_key(instructions, 1)
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
self.can_set_version = False self.can_set_version = False
read_performed = True read_performed = True
elif matches_op(op, 'GET_KEY'): elif matches_op(op, "GET_KEY"):
if is_non_transaction_op(op) or self.can_use_key_selectors: if is_non_transaction_op(op) or self.can_use_key_selectors:
self.ensure_key(instructions, 1) self.ensure_key(instructions, 1)
instructions.push_args(self.workspace.key()) instructions.push_args(self.workspace.key())
@ -270,7 +335,7 @@ class ApiTest(Test):
self.can_set_version = False self.can_set_version = False
read_performed = True read_performed = True
elif matches_op(op, 'GET_RANGE'): elif matches_op(op, "GET_RANGE"):
self.ensure_key(instructions, 2) self.ensure_key(instructions, 2)
range_params = self.random.random_range_params() range_params = self.random.random_range_params()
instructions.push_args(*range_params) instructions.push_args(*range_params)
@ -278,7 +343,9 @@ class ApiTest(Test):
test_util.to_front(instructions, 4) test_util.to_front(instructions, 4)
instructions.append(op) instructions.append(op)
if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large if (
range_params[0] >= 1 and range_params[0] <= 1000
): # avoid adding a string if the limit is large
self.add_strings(1) self.add_strings(1)
else: else:
self.add_stack_items(1) self.add_stack_items(1)
@ -286,7 +353,7 @@ class ApiTest(Test):
self.can_set_version = False self.can_set_version = False
read_performed = True read_performed = True
elif matches_op(op, 'GET_RANGE_STARTS_WITH'): elif matches_op(op, "GET_RANGE_STARTS_WITH"):
# TODO: not tested well # TODO: not tested well
self.ensure_key(instructions, 1) self.ensure_key(instructions, 1)
range_params = self.random.random_range_params() range_params = self.random.random_range_params()
@ -294,7 +361,9 @@ class ApiTest(Test):
test_util.to_front(instructions, 3) test_util.to_front(instructions, 3)
instructions.append(op) instructions.append(op)
if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large if (
range_params[0] >= 1 and range_params[0] <= 1000
): # avoid adding a string if the limit is large
self.add_strings(1) self.add_strings(1)
else: else:
self.add_stack_items(1) self.add_stack_items(1)
@ -302,7 +371,7 @@ class ApiTest(Test):
self.can_set_version = False self.can_set_version = False
read_performed = True read_performed = True
elif matches_op(op, 'GET_RANGE_SELECTOR'): elif matches_op(op, "GET_RANGE_SELECTOR"):
if is_non_transaction_op(op) or self.can_use_key_selectors: if is_non_transaction_op(op) or self.can_use_key_selectors:
self.ensure_key(instructions, 2) self.ensure_key(instructions, 2)
instructions.push_args(self.workspace.key()) instructions.push_args(self.workspace.key())
@ -314,7 +383,9 @@ class ApiTest(Test):
test_util.to_front(instructions, 9) test_util.to_front(instructions, 9)
instructions.append(op) instructions.append(op)
if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large if (
range_params[0] >= 1 and range_params[0] <= 1000
): # avoid adding a string if the limit is large
self.add_strings(1) self.add_strings(1)
else: else:
self.add_stack_items(1) self.add_stack_items(1)
@ -322,29 +393,29 @@ class ApiTest(Test):
self.can_set_version = False self.can_set_version = False
read_performed = True read_performed = True
elif matches_op(op, 'GET_READ_VERSION'): elif matches_op(op, "GET_READ_VERSION"):
instructions.append(op) instructions.append(op)
self.has_version = self.can_set_version self.has_version = self.can_set_version
self.add_strings(1) self.add_strings(1)
elif matches_op(op, 'SET'): elif matches_op(op, "SET"):
self.ensure_key_value(instructions) self.ensure_key_value(instructions)
instructions.append(op) instructions.append(op)
if is_non_transaction_op(op): if is_non_transaction_op(op):
self.add_stack_items(1) self.add_stack_items(1)
elif op == 'SET_READ_VERSION': elif op == "SET_READ_VERSION":
if self.has_version and self.can_set_version: if self.has_version and self.can_set_version:
instructions.append(op) instructions.append(op)
self.can_set_version = False self.can_set_version = False
elif matches_op(op, 'CLEAR'): elif matches_op(op, "CLEAR"):
self.ensure_key(instructions, 1) self.ensure_key(instructions, 1)
instructions.append(op) instructions.append(op)
if is_non_transaction_op(op): if is_non_transaction_op(op):
self.add_stack_items(1) self.add_stack_items(1)
elif matches_op(op, 'CLEAR_RANGE'): elif matches_op(op, "CLEAR_RANGE"):
# Protect against inverted range # Protect against inverted range
key1 = self.workspace.pack(self.random.random_tuple(5)) key1 = self.workspace.pack(self.random.random_tuple(5))
key2 = self.workspace.pack(self.random.random_tuple(5)) key2 = self.workspace.pack(self.random.random_tuple(5))
@ -358,13 +429,13 @@ class ApiTest(Test):
if is_non_transaction_op(op): if is_non_transaction_op(op):
self.add_stack_items(1) self.add_stack_items(1)
elif matches_op(op, 'CLEAR_RANGE_STARTS_WITH'): elif matches_op(op, "CLEAR_RANGE_STARTS_WITH"):
self.ensure_key(instructions, 1) self.ensure_key(instructions, 1)
instructions.append(op) instructions.append(op)
if is_non_transaction_op(op): if is_non_transaction_op(op):
self.add_stack_items(1) self.add_stack_items(1)
elif matches_op(op, 'ATOMIC_OP'): elif matches_op(op, "ATOMIC_OP"):
self.ensure_key_value(instructions) self.ensure_key_value(instructions)
if is_non_transaction_op(op) and args.concurrency == 1: if is_non_transaction_op(op) and args.concurrency == 1:
instructions.push_args(random.choice(idempotent_atomic_ops)) instructions.push_args(random.choice(idempotent_atomic_ops))
@ -375,50 +446,58 @@ class ApiTest(Test):
if is_non_transaction_op(op): if is_non_transaction_op(op):
self.add_stack_items(1) self.add_stack_items(1)
elif op == 'VERSIONSTAMP': elif op == "VERSIONSTAMP":
rand_str1 = self.random.random_string(100) rand_str1 = self.random.random_string(100)
key1 = self.versionstamped_values.pack((rand_str1,)) key1 = self.versionstamped_values.pack((rand_str1,))
key2 = self.versionstamped_values_2.pack((rand_str1,)) key2 = self.versionstamped_values_2.pack((rand_str1,))
split = random.randint(0, 70) split = random.randint(0, 70)
prefix = self.random.random_string(20 + split) prefix = self.random.random_string(20 + split)
if prefix.endswith(b'\xff'): if prefix.endswith(b"\xff"):
# Necessary to make sure that the SET_VERSIONSTAMPED_VALUE check # Necessary to make sure that the SET_VERSIONSTAMPED_VALUE check
# correctly finds where the version is supposed to fit in. # correctly finds where the version is supposed to fit in.
prefix += b'\x00' prefix += b"\x00"
suffix = self.random.random_string(70 - split) suffix = self.random.random_string(70 - split)
rand_str2 = prefix + fdb.tuple.Versionstamp._UNSET_TR_VERSION + suffix rand_str2 = prefix + fdb.tuple.Versionstamp._UNSET_TR_VERSION + suffix
key3 = self.versionstamped_keys.pack() + rand_str2 key3 = self.versionstamped_keys.pack() + rand_str2
index = len(self.versionstamped_keys.pack()) + len(prefix) index = len(self.versionstamped_keys.pack()) + len(prefix)
key3 = self.versionstamp_key(key3, index) key3 = self.versionstamp_key(key3, index)
instructions.push_args('SET_VERSIONSTAMPED_VALUE', instructions.push_args(
key1, "SET_VERSIONSTAMPED_VALUE",
self.versionstamp_value(fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2)) key1,
instructions.append('ATOMIC_OP') self.versionstamp_value(
fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2
),
)
instructions.append("ATOMIC_OP")
if args.api_version >= 520: if args.api_version >= 520:
instructions.push_args('SET_VERSIONSTAMPED_VALUE', key2, self.versionstamp_value(rand_str2, len(prefix))) instructions.push_args(
instructions.append('ATOMIC_OP') "SET_VERSIONSTAMPED_VALUE",
key2,
self.versionstamp_value(rand_str2, len(prefix)),
)
instructions.append("ATOMIC_OP")
instructions.push_args('SET_VERSIONSTAMPED_KEY', key3, rand_str1) instructions.push_args("SET_VERSIONSTAMPED_KEY", key3, rand_str1)
instructions.append('ATOMIC_OP') instructions.append("ATOMIC_OP")
self.can_use_key_selectors = False self.can_use_key_selectors = False
elif op == 'READ_CONFLICT_RANGE' or op == 'WRITE_CONFLICT_RANGE': elif op == "READ_CONFLICT_RANGE" or op == "WRITE_CONFLICT_RANGE":
self.ensure_key(instructions, 2) self.ensure_key(instructions, 2)
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
elif op == 'READ_CONFLICT_KEY' or op == 'WRITE_CONFLICT_KEY': elif op == "READ_CONFLICT_KEY" or op == "WRITE_CONFLICT_KEY":
self.ensure_key(instructions, 1) self.ensure_key(instructions, 1)
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
elif op == 'DISABLE_WRITE_CONFLICT': elif op == "DISABLE_WRITE_CONFLICT":
instructions.append(op) instructions.append(op)
elif op == 'COMMIT': elif op == "COMMIT":
if args.concurrency == 1 or i < self.max_keys or random.random() < 0.9: if args.concurrency == 1 or i < self.max_keys or random.random() < 0.9:
if args.concurrency == 1: if args.concurrency == 1:
self.wait_for_reads(instructions) self.wait_for_reads(instructions)
@ -431,23 +510,23 @@ class ApiTest(Test):
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
elif op == 'RESET': elif op == "RESET":
instructions.append(op) instructions.append(op)
self.can_get_commit_version = False self.can_get_commit_version = False
self.can_set_version = True self.can_set_version = True
self.can_use_key_selectors = True self.can_use_key_selectors = True
elif op == 'CANCEL': elif op == "CANCEL":
instructions.append(op) instructions.append(op)
self.can_set_version = False self.can_set_version = False
elif op == 'GET_COMMITTED_VERSION': elif op == "GET_COMMITTED_VERSION":
if self.can_get_commit_version: if self.can_get_commit_version:
do_commit = random.random() < 0.5 do_commit = random.random() < 0.5
if do_commit: if do_commit:
instructions.append('COMMIT') instructions.append("COMMIT")
instructions.append('WAIT_FUTURE') instructions.append("WAIT_FUTURE")
self.add_stack_items(1) self.add_stack_items(1)
instructions.append(op) instructions.append(op)
@ -456,35 +535,47 @@ class ApiTest(Test):
self.add_strings(1) self.add_strings(1)
if do_commit: if do_commit:
instructions.append('RESET') instructions.append("RESET")
self.can_get_commit_version = False self.can_get_commit_version = False
self.can_set_version = True self.can_set_version = True
self.can_use_key_selectors = True self.can_use_key_selectors = True
elif op == 'GET_APPROXIMATE_SIZE': elif op == "GET_APPROXIMATE_SIZE":
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
elif op == 'TUPLE_PACK' or op == 'TUPLE_RANGE': elif op == "TUPLE_PACK" or op == "TUPLE_RANGE":
tup = self.random.random_tuple(10) tup = self.random.random_tuple(10)
instructions.push_args(len(tup), *tup) instructions.push_args(len(tup), *tup)
instructions.append(op) instructions.append(op)
if op == 'TUPLE_PACK': if op == "TUPLE_PACK":
self.add_strings(1) self.add_strings(1)
else: else:
self.add_strings(2) self.add_strings(2)
elif op == 'TUPLE_PACK_WITH_VERSIONSTAMP': elif op == "TUPLE_PACK_WITH_VERSIONSTAMP":
tup = (self.random.random_string(20),) + self.random.random_tuple(10, incomplete_versionstamps=True) tup = (self.random.random_string(20),) + self.random.random_tuple(
10, incomplete_versionstamps=True
)
prefix = self.versionstamped_keys.pack() prefix = self.versionstamped_keys.pack()
instructions.push_args(prefix, len(tup), *tup) instructions.push_args(prefix, len(tup), *tup)
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
versionstamp_param = prefix + fdb.tuple.pack(tup) versionstamp_param = prefix + fdb.tuple.pack(tup)
first_incomplete = versionstamp_param.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION) first_incomplete = versionstamp_param.find(
second_incomplete = -1 if first_incomplete < 0 else \ fdb.tuple.Versionstamp._UNSET_TR_VERSION
versionstamp_param.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION, first_incomplete + len(fdb.tuple.Versionstamp._UNSET_TR_VERSION) + 1) )
second_incomplete = (
-1
if first_incomplete < 0
else versionstamp_param.find(
fdb.tuple.Versionstamp._UNSET_TR_VERSION,
first_incomplete
+ len(fdb.tuple.Versionstamp._UNSET_TR_VERSION)
+ 1,
)
)
# If there is exactly one incomplete versionstamp, perform the versionstamp operation. # If there is exactly one incomplete versionstamp, perform the versionstamp operation.
if first_incomplete >= 0 and second_incomplete < 0: if first_incomplete >= 0 and second_incomplete < 0:
@ -492,76 +583,90 @@ class ApiTest(Test):
instructions.push_args(rand_str) instructions.push_args(rand_str)
test_util.to_front(instructions, 1) test_util.to_front(instructions, 1)
instructions.push_args('SET_VERSIONSTAMPED_KEY') instructions.push_args("SET_VERSIONSTAMPED_KEY")
instructions.append('ATOMIC_OP') instructions.append("ATOMIC_OP")
if self.api_version >= 520: if self.api_version >= 520:
version_value_key_2 = self.versionstamped_values_2.pack((rand_str,)) version_value_key_2 = self.versionstamped_values_2.pack(
versionstamped_value = self.versionstamp_value(fdb.tuple.pack(tup), first_incomplete - len(prefix)) (rand_str,)
instructions.push_args('SET_VERSIONSTAMPED_VALUE', version_value_key_2, versionstamped_value) )
instructions.append('ATOMIC_OP') versionstamped_value = self.versionstamp_value(
fdb.tuple.pack(tup), first_incomplete - len(prefix)
)
instructions.push_args(
"SET_VERSIONSTAMPED_VALUE",
version_value_key_2,
versionstamped_value,
)
instructions.append("ATOMIC_OP")
version_value_key = self.versionstamped_values.pack((rand_str,)) version_value_key = self.versionstamped_values.pack((rand_str,))
instructions.push_args('SET_VERSIONSTAMPED_VALUE', version_value_key, instructions.push_args(
self.versionstamp_value(fdb.tuple.Versionstamp._UNSET_TR_VERSION + fdb.tuple.pack(tup))) "SET_VERSIONSTAMPED_VALUE",
instructions.append('ATOMIC_OP') version_value_key,
self.versionstamp_value(
fdb.tuple.Versionstamp._UNSET_TR_VERSION
+ fdb.tuple.pack(tup)
),
)
instructions.append("ATOMIC_OP")
self.can_use_key_selectors = False self.can_use_key_selectors = False
elif op == 'TUPLE_UNPACK': elif op == "TUPLE_UNPACK":
tup = self.random.random_tuple(10) tup = self.random.random_tuple(10)
instructions.push_args(len(tup), *tup) instructions.push_args(len(tup), *tup)
instructions.append('TUPLE_PACK') instructions.append("TUPLE_PACK")
instructions.append(op) instructions.append(op)
self.add_strings(len(tup)) self.add_strings(len(tup))
elif op == 'TUPLE_SORT': elif op == "TUPLE_SORT":
tups = self.random.random_tuple_list(10, 30) tups = self.random.random_tuple_list(10, 30)
for tup in tups: for tup in tups:
instructions.push_args(len(tup), *tup) instructions.push_args(len(tup), *tup)
instructions.append('TUPLE_PACK') instructions.append("TUPLE_PACK")
instructions.push_args(len(tups)) instructions.push_args(len(tups))
instructions.append(op) instructions.append(op)
self.add_strings(len(tups)) self.add_strings(len(tups))
# Use SUB to test if integers are correctly unpacked # Use SUB to test if integers are correctly unpacked
elif op == 'SUB': elif op == "SUB":
a = self.random.random_int() // 2 a = self.random.random_int() // 2
b = self.random.random_int() // 2 b = self.random.random_int() // 2
instructions.push_args(0, a, b) instructions.push_args(0, a, b)
instructions.append(op) instructions.append(op)
instructions.push_args(1) instructions.push_args(1)
instructions.append('SWAP') instructions.append("SWAP")
instructions.append(op) instructions.append(op)
instructions.push_args(1) instructions.push_args(1)
instructions.append('TUPLE_PACK') instructions.append("TUPLE_PACK")
self.add_stack_items(1) self.add_stack_items(1)
elif op == 'ENCODE_FLOAT': elif op == "ENCODE_FLOAT":
f = self.random.random_float(8) f = self.random.random_float(8)
f_bytes = struct.pack('>f', f) f_bytes = struct.pack(">f", f)
instructions.push_args(f_bytes) instructions.push_args(f_bytes)
instructions.append(op) instructions.append(op)
self.add_stack_items(1) self.add_stack_items(1)
elif op == 'ENCODE_DOUBLE': elif op == "ENCODE_DOUBLE":
d = self.random.random_float(11) d = self.random.random_float(11)
d_bytes = struct.pack('>d', d) d_bytes = struct.pack(">d", d)
instructions.push_args(d_bytes) instructions.push_args(d_bytes)
instructions.append(op) instructions.append(op)
self.add_stack_items(1) self.add_stack_items(1)
elif op == 'DECODE_FLOAT': elif op == "DECODE_FLOAT":
f = self.random.random_float(8) f = self.random.random_float(8)
instructions.push_args(fdb.tuple.SingleFloat(f)) instructions.push_args(fdb.tuple.SingleFloat(f))
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
elif op == 'DECODE_DOUBLE': elif op == "DECODE_DOUBLE":
d = self.random.random_float(11) d = self.random.random_float(11)
instructions.push_args(d) instructions.push_args(d)
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
elif op == 'GET_ESTIMATED_RANGE_SIZE': elif op == "GET_ESTIMATED_RANGE_SIZE":
# Protect against inverted range and identical keys # Protect against inverted range and identical keys
key1 = self.workspace.pack(self.random.random_tuple(1)) key1 = self.workspace.pack(self.random.random_tuple(1))
key2 = self.workspace.pack(self.random.random_tuple(1)) key2 = self.workspace.pack(self.random.random_tuple(1))
@ -576,7 +681,7 @@ class ApiTest(Test):
instructions.push_args(key1, key2) instructions.push_args(key1, key2)
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
elif op == 'GET_RANGE_SPLIT_POINTS': elif op == "GET_RANGE_SPLIT_POINTS":
# Protect against inverted range and identical keys # Protect against inverted range and identical keys
key1 = self.workspace.pack(self.random.random_tuple(1)) key1 = self.workspace.pack(self.random.random_tuple(1))
key2 = self.workspace.pack(self.random.random_tuple(1)) key2 = self.workspace.pack(self.random.random_tuple(1))
@ -593,27 +698,27 @@ class ApiTest(Test):
instructions.push_args(key1, key2, chunkSize) instructions.push_args(key1, key2, chunkSize)
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
elif op == 'TENANT_CREATE': elif op == "TENANT_CREATE":
tenant_name = self.choose_tenant(0.8) tenant_name = self.choose_tenant(0.8)
self.allocated_tenants.add(tenant_name) self.allocated_tenants.add(tenant_name)
instructions.push_args(tenant_name) instructions.push_args(tenant_name)
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
elif op == 'TENANT_DELETE': elif op == "TENANT_DELETE":
tenant_name = self.choose_tenant(0.2) tenant_name = self.choose_tenant(0.2)
if tenant_name in self.allocated_tenants: if tenant_name in self.allocated_tenants:
self.allocated_tenants.remove(tenant_name) self.allocated_tenants.remove(tenant_name)
instructions.push_args(tenant_name) instructions.push_args(tenant_name)
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
elif op == 'TENANT_SET_ACTIVE': elif op == "TENANT_SET_ACTIVE":
tenant_name = self.choose_tenant(0.8) tenant_name = self.choose_tenant(0.8)
instructions.push_args(tenant_name) instructions.push_args(tenant_name)
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
elif op == 'TENANT_CLEAR_ACTIVE': elif op == "TENANT_CLEAR_ACTIVE":
instructions.append(op) instructions.append(op)
elif op == 'TENANT_LIST': elif op == "TENANT_LIST":
self.ensure_string(instructions, 2) self.ensure_string(instructions, 2)
instructions.push_args(self.random.random_int()) instructions.push_args(self.random.random_int())
test_util.to_front(instructions, 2) test_util.to_front(instructions, 2)
@ -624,27 +729,33 @@ class ApiTest(Test):
instructions.append(op) instructions.append(op)
self.add_strings(1) self.add_strings(1)
else: else:
assert False, 'Unknown operation: ' + op assert False, "Unknown operation: " + op
if read_performed and op not in database_reads and op not in tenant_reads: if read_performed and op not in database_reads and op not in tenant_reads:
self.outstanding_ops.append((self.stack_size, len(instructions) - 1)) self.outstanding_ops.append((self.stack_size, len(instructions) - 1))
if args.concurrency == 1 and (op in database_reads or op in database_mutations or op in tenant_reads or op in tenant_mutations or op in ['TENANT_CREATE', 'TENANT_DELETE']): if args.concurrency == 1 and (
instructions.append('WAIT_FUTURE') op in database_reads
or op in database_mutations
or op in tenant_reads
or op in tenant_mutations
or op in ["TENANT_CREATE", "TENANT_DELETE"]
):
instructions.append("WAIT_FUTURE")
instructions.begin_finalization() instructions.begin_finalization()
if not args.no_tenants: if not args.no_tenants:
instructions.append('TENANT_CLEAR_ACTIVE') instructions.append("TENANT_CLEAR_ACTIVE")
if args.concurrency == 1: if args.concurrency == 1:
self.wait_for_reads(instructions) self.wait_for_reads(instructions)
test_util.blocking_commit(instructions) test_util.blocking_commit(instructions)
self.add_stack_items(1) self.add_stack_items(1)
instructions.append('NEW_TRANSACTION') instructions.append("NEW_TRANSACTION")
instructions.push_args(self.stack_subspace.key()) instructions.push_args(self.stack_subspace.key())
instructions.append('LOG_STACK') instructions.append("LOG_STACK")
test_util.blocking_commit(instructions) test_util.blocking_commit(instructions)
@ -654,22 +765,30 @@ class ApiTest(Test):
def check_versionstamps(self, tr, begin_key, limit): def check_versionstamps(self, tr, begin_key, limit):
next_begin = None next_begin = None
incorrect_versionstamps = 0 incorrect_versionstamps = 0
for k, v in tr.get_range(begin_key, self.versionstamped_values.range().stop, limit=limit): for k, v in tr.get_range(
next_begin = k + b'\x00' begin_key, self.versionstamped_values.range().stop, limit=limit
):
next_begin = k + b"\x00"
random_id = self.versionstamped_values.unpack(k)[0] random_id = self.versionstamped_values.unpack(k)[0]
versioned_value = v[10:].replace(fdb.tuple.Versionstamp._UNSET_TR_VERSION, v[:10], 1) versioned_value = v[10:].replace(
fdb.tuple.Versionstamp._UNSET_TR_VERSION, v[:10], 1
)
versioned_key = self.versionstamped_keys.pack() + versioned_value versioned_key = self.versionstamped_keys.pack() + versioned_value
if tr[versioned_key] != random_id: if tr[versioned_key] != random_id:
util.get_logger().error(' INCORRECT VERSIONSTAMP:') util.get_logger().error(" INCORRECT VERSIONSTAMP:")
util.get_logger().error(' %s != %s', repr(tr[versioned_key]), repr(random_id)) util.get_logger().error(
" %s != %s", repr(tr[versioned_key]), repr(random_id)
)
incorrect_versionstamps += 1 incorrect_versionstamps += 1
if self.api_version >= 520: if self.api_version >= 520:
k2 = self.versionstamped_values_2.pack((random_id,)) k2 = self.versionstamped_values_2.pack((random_id,))
if tr[k2] != versioned_value: if tr[k2] != versioned_value:
util.get_logger().error(' INCORRECT VERSIONSTAMP:') util.get_logger().error(" INCORRECT VERSIONSTAMP:")
util.get_logger().error(' %s != %s', repr(tr[k2]), repr(versioned_value)) util.get_logger().error(
" %s != %s", repr(tr[k2]), repr(versioned_value)
)
incorrect_versionstamps += 1 incorrect_versionstamps += 1
return (next_begin, incorrect_versionstamps) return (next_begin, incorrect_versionstamps)
@ -681,16 +800,26 @@ class ApiTest(Test):
incorrect_versionstamps = 0 incorrect_versionstamps = 0
while begin is not None: while begin is not None:
(begin, current_incorrect_versionstamps) = self.check_versionstamps(db, begin, 100) (begin, current_incorrect_versionstamps) = self.check_versionstamps(
db, begin, 100
)
incorrect_versionstamps += current_incorrect_versionstamps incorrect_versionstamps += current_incorrect_versionstamps
if incorrect_versionstamps > 0: if incorrect_versionstamps > 0:
errors.append('There were %d failed version stamp operations' % incorrect_versionstamps) errors.append(
"There were %d failed version stamp operations"
% incorrect_versionstamps
)
return errors return errors
def get_result_specifications(self): def get_result_specifications(self):
return [ return [
ResultSpecification(self.workspace, global_error_filter=[1007, 1009, 1021]), ResultSpecification(self.workspace, global_error_filter=[1007, 1009, 1021]),
ResultSpecification(self.stack_subspace, key_start_index=1, ordering_index=1, global_error_filter=[1007, 1009, 1021]) ResultSpecification(
self.stack_subspace,
key_start_index=1,
ordering_index=1,
global_error_filter=[1007, 1009, 1021],
),
] ]

95
bindings/bindingtester/tests/test_util.py
View File

@ -33,16 +33,20 @@ from bindingtester.known_testers import COMMON_TYPES
class RandomGenerator(object): class RandomGenerator(object):
def __init__(self, max_int_bits=64, api_version=FDB_API_VERSION, types=COMMON_TYPES): def __init__(
self, max_int_bits=64, api_version=FDB_API_VERSION, types=COMMON_TYPES
):
self.max_int_bits = max_int_bits self.max_int_bits = max_int_bits
self.api_version = api_version self.api_version = api_version
self.types = list(types) self.types = list(types)
def random_unicode_str(self, length): def random_unicode_str(self, length):
return ''.join(self.random_unicode_char() for i in range(0, length)) return "".join(self.random_unicode_char() for i in range(0, length))
def random_int(self): def random_int(self):
num_bits = random.randint(0, self.max_int_bits) # This way, we test small numbers with higher probability num_bits = random.randint(
0, self.max_int_bits
) # This way, we test small numbers with higher probability
max_value = (1 << num_bits) - 1 max_value = (1 << num_bits) - 1
min_value = -max_value - 1 min_value = -max_value - 1
@ -54,11 +58,15 @@ class RandomGenerator(object):
def random_float(self, exp_bits): def random_float(self, exp_bits):
if random.random() < 0.05: if random.random() < 0.05:
# Choose a special value. # Choose a special value.
return random.choice([float('-nan'), float('-inf'), -0.0, 0.0, float('inf'), float('nan')]) return random.choice(
[float("-nan"), float("-inf"), -0.0, 0.0, float("inf"), float("nan")]
)
else: else:
# Choose a value from all over the range of acceptable floats for this precision. # Choose a value from all over the range of acceptable floats for this precision.
sign = -1 if random.random() < 0.5 else 1 sign = -1 if random.random() < 0.5 else 1
exponent = random.randint(-(1 << (exp_bits - 1)) - 10, (1 << (exp_bits - 1) - 1)) exponent = random.randint(
-(1 << (exp_bits - 1)) - 10, (1 << (exp_bits - 1) - 1)
)
mantissa = random.random() mantissa = random.random()
result = sign * math.pow(2, exponent) * mantissa result = sign * math.pow(2, exponent) * mantissa
@ -73,38 +81,38 @@ class RandomGenerator(object):
for i in range(size): for i in range(size):
choice = random.choice(self.types) choice = random.choice(self.types)
if choice == 'int': if choice == "int":
tup.append(self.random_int()) tup.append(self.random_int())
elif choice == 'null': elif choice == "null":
tup.append(None) tup.append(None)
elif choice == 'bytes': elif choice == "bytes":
tup.append(self.random_string(random.randint(0, 100))) tup.append(self.random_string(random.randint(0, 100)))
elif choice == 'string': elif choice == "string":
tup.append(self.random_unicode_str(random.randint(0, 100))) tup.append(self.random_unicode_str(random.randint(0, 100)))
elif choice == 'uuid': elif choice == "uuid":
tup.append(uuid.uuid4()) tup.append(uuid.UUID(int=random.getrandbits(128)))
elif choice == 'bool': elif choice == "bool":
b = random.random() < 0.5 b = random.random() < 0.5
if self.api_version < 500: if self.api_version < 500:
tup.append(int(b)) tup.append(int(b))
else: else:
tup.append(b) tup.append(b)
elif choice == 'float': elif choice == "float":
tup.append(fdb.tuple.SingleFloat(self.random_float(8))) tup.append(fdb.tuple.SingleFloat(self.random_float(8)))
elif choice == 'double': elif choice == "double":
tup.append(self.random_float(11)) tup.append(self.random_float(11))
elif choice == 'tuple': elif choice == "tuple":
length = random.randint(0, max_size - size) length = random.randint(0, max_size - size)
if length == 0: if length == 0:
tup.append(()) tup.append(())
else: else:
tup.append(self.random_tuple(length)) tup.append(self.random_tuple(length))
elif choice == 'versionstamp': elif choice == "versionstamp":
if incomplete_versionstamps and random.random() < 0.5: if incomplete_versionstamps and random.random() < 0.5:
tr_version = fdb.tuple.Versionstamp._UNSET_TR_VERSION tr_version = fdb.tuple.Versionstamp._UNSET_TR_VERSION
else: else:
tr_version = self.random_string(10) tr_version = self.random_string(10)
user_version = random.randint(0, 0xffff) user_version = random.randint(0, 0xFFFF)
tup.append(fdb.tuple.Versionstamp(tr_version, user_version)) tup.append(fdb.tuple.Versionstamp(tr_version, user_version))
else: else:
assert False assert False
@ -123,12 +131,19 @@ class RandomGenerator(object):
smaller_size = random.randint(1, len(to_add)) smaller_size = random.randint(1, len(to_add))
tuples.append(to_add[:smaller_size]) tuples.append(to_add[:smaller_size])
else: else:
non_empty = [x for x in enumerate(to_add) if (isinstance(x[1], list) or isinstance(x[1], tuple)) and len(x[1]) > 0] non_empty = [
x
for x in enumerate(to_add)
if (isinstance(x[1], list) or isinstance(x[1], tuple))
and len(x[1]) > 0
]
if len(non_empty) > 0 and random.random() < 0.25: if len(non_empty) > 0 and random.random() < 0.25:
# Add a smaller list to test prefixes of nested structures. # Add a smaller list to test prefixes of nested structures.
idx, choice = random.choice(non_empty) idx, choice = random.choice(non_empty)
smaller_size = random.randint(0, len(to_add[idx])) smaller_size = random.randint(0, len(to_add[idx]))
tuples.append(to_add[:idx] + (choice[:smaller_size],) + to_add[idx + 1:]) tuples.append(
to_add[:idx] + (choice[:smaller_size],) + to_add[idx + 1 :]
)
random.shuffle(tuples) random.shuffle(tuples)
return tuples return tuples
@ -153,30 +168,40 @@ class RandomGenerator(object):
def random_string(self, length): def random_string(self, length):
if length == 0: if length == 0:
return b'' return b""
return bytes([random.randint(0, 254)] + [random.randint(0, 255) for i in range(0, length - 1)]) return bytes(
[random.randint(0, 254)]
+ [random.randint(0, 255) for i in range(0, length - 1)]
)
def random_unicode_char(self): def random_unicode_char(self):
while True: while True:
if random.random() < 0.05: if random.random() < 0.05:
# Choose one of these special character sequences. # Choose one of these special character sequences.
specials = ['\U0001f4a9', '\U0001f63c', '\U0001f3f3\ufe0f\u200d\U0001f308', '\U0001f1f5\U0001f1f2', '\uf8ff', specials = [
'\U0002a2b2', '\u05e9\u05dc\u05d5\u05dd'] "\U0001f4a9",
"\U0001f63c",
"\U0001f3f3\ufe0f\u200d\U0001f308",
"\U0001f1f5\U0001f1f2",
"\uf8ff",
"\U0002a2b2",
"\u05e9\u05dc\u05d5\u05dd",
]
return random.choice(specials) return random.choice(specials)
c = random.randint(0, 0xffff) c = random.randint(0, 0xFFFF)
if unicodedata.category(chr(c))[0] in 'LMNPSZ': if unicodedata.category(chr(c))[0] in "LMNPSZ":
return chr(c) return chr(c)
def error_string(error_code): def error_string(error_code):
return fdb.tuple.pack((b'ERROR', bytes(str(error_code), 'utf-8'))) return fdb.tuple.pack((b"ERROR", bytes(str(error_code), "utf-8")))
def blocking_commit(instructions): def blocking_commit(instructions):
instructions.append('COMMIT') instructions.append("COMMIT")
instructions.append('WAIT_FUTURE') instructions.append("WAIT_FUTURE")
instructions.append('RESET') instructions.append("RESET")
def to_front(instructions, index): def to_front(instructions, index):
@ -184,19 +209,19 @@ def to_front(instructions, index):
pass pass
elif index == 1: elif index == 1:
instructions.push_args(1) instructions.push_args(1)
instructions.append('SWAP') instructions.append("SWAP")
elif index == 2: elif index == 2:
instructions.push_args(index - 1) instructions.push_args(index - 1)
instructions.append('SWAP') instructions.append("SWAP")
instructions.push_args(index) instructions.push_args(index)
instructions.append('SWAP') instructions.append("SWAP")
else: else:
instructions.push_args(index - 1) instructions.push_args(index - 1)
instructions.append('SWAP') instructions.append("SWAP")
instructions.push_args(index) instructions.push_args(index)
instructions.append('SWAP') instructions.append("SWAP")
instructions.push_args(index - 1) instructions.push_args(index - 1)
instructions.append('SWAP') instructions.append("SWAP")
to_front(instructions, index - 1) to_front(instructions, index - 1)

63
bindings/c/fdb_c.cpp
View File

@ -386,8 +386,7 @@ void setBlobFilePointer(FDBBGFilePointer* dest, const BlobFilePointerRef& source
dest->file_offset = source.offset; dest->file_offset = source.offset;
dest->file_length = source.length; dest->file_length = source.length;
dest->full_file_length = source.fullFileLength; dest->full_file_length = source.fullFileLength;
// FIXME: add version info to each source file pointer dest->file_version = source.fileVersion;
dest->file_version = 0;
// handle encryption // handle encryption
if (source.cipherKeysCtx.present()) { if (source.cipherKeysCtx.present()) {
@ -717,6 +716,17 @@ extern "C" DLLEXPORT FDBFuture* fdb_database_blobbify_range(FDBDatabase* db,
.extractPtr()); .extractPtr());
} }
extern "C" DLLEXPORT FDBFuture* fdb_database_blobbify_range_blocking(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length) {
return (FDBFuture*)(DB(db)
->blobbifyRangeBlocking(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
StringRef(end_key_name, end_key_name_length)))
.extractPtr());
}
extern "C" DLLEXPORT FDBFuture* fdb_database_unblobbify_range(FDBDatabase* db, extern "C" DLLEXPORT FDBFuture* fdb_database_unblobbify_range(FDBDatabase* db,
uint8_t const* begin_key_name, uint8_t const* begin_key_name,
int begin_key_name_length, int begin_key_name_length,
@ -758,6 +768,25 @@ extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_verify_blob_rang
.extractPtr()); .extractPtr());
} }
extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_flush_blob_range(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version) {
Optional<Version> rv;
if (version != latestVersion) {
rv = version;
}
return (FDBFuture*)(DB(db)
->flushBlobRange(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
StringRef(end_key_name, end_key_name_length)),
compact,
rv)
.extractPtr());
}
extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_client_status(FDBDatabase* db) { extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_client_status(FDBDatabase* db) {
return (FDBFuture*)(DB(db)->getClientStatus().extractPtr()); return (FDBFuture*)(DB(db)->getClientStatus().extractPtr());
} }
@ -799,6 +828,17 @@ extern "C" DLLEXPORT FDBFuture* fdb_tenant_blobbify_range(FDBTenant* tenant,
.extractPtr()); .extractPtr());
} }
extern "C" DLLEXPORT FDBFuture* fdb_tenant_blobbify_range_blocking(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length) {
return (FDBFuture*)(TENANT(tenant)
->blobbifyRangeBlocking(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
StringRef(end_key_name, end_key_name_length)))
.extractPtr());
}
extern "C" DLLEXPORT FDBFuture* fdb_tenant_unblobbify_range(FDBTenant* tenant, extern "C" DLLEXPORT FDBFuture* fdb_tenant_unblobbify_range(FDBTenant* tenant,
uint8_t const* begin_key_name, uint8_t const* begin_key_name,
int begin_key_name_length, int begin_key_name_length,
@ -840,6 +880,25 @@ extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_verify_blob_range(
.extractPtr()); .extractPtr());
} }
extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_flush_blob_range(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version) {
Optional<Version> rv;
if (version != latestVersion) {
rv = version;
}
return (FDBFuture*)(TENANT(tenant)
->flushBlobRange(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
StringRef(end_key_name, end_key_name_length)),
compact,
rv)
.extractPtr());
}
extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_get_id(FDBTenant* tenant) { extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_get_id(FDBTenant* tenant) {
return (FDBFuture*)(TENANT(tenant)->getId().extractPtr()); return (FDBFuture*)(TENANT(tenant)->getId().extractPtr());
} }

28
bindings/c/foundationdb/fdb_c.h
View File

@ -417,6 +417,12 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_blobbify_range(FDBDatabase*
uint8_t const* end_key_name, uint8_t const* end_key_name,
int end_key_name_length); int end_key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_blobbify_range_blocking(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_unblobbify_range(FDBDatabase* db, DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_unblobbify_range(FDBDatabase* db,
uint8_t const* begin_key_name, uint8_t const* begin_key_name,
int begin_key_name_length, int begin_key_name_length,
@ -437,6 +443,14 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_verify_blob_range(FDBDataba
int end_key_name_length, int end_key_name_length,
int64_t version); int64_t version);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_flush_blob_range(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_client_status(FDBDatabase* db); DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_client_status(FDBDatabase* db);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_tenant_create_transaction(FDBTenant* tenant, DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_tenant_create_transaction(FDBTenant* tenant,
@ -460,6 +474,12 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_blobbify_range(FDBTenant* ten
uint8_t const* end_key_name, uint8_t const* end_key_name,
int end_key_name_length); int end_key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_blobbify_range_blocking(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_unblobbify_range(FDBTenant* tenant, DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_unblobbify_range(FDBTenant* tenant,
uint8_t const* begin_key_name, uint8_t const* begin_key_name,
int begin_key_name_length, int begin_key_name_length,
@ -487,6 +507,14 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_verify_blob_range(FDBTenant*
int end_key_name_length, int end_key_name_length,
int64_t version); int64_t version);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_flush_blob_range(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_get_id(FDBTenant* tenant); DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_get_id(FDBTenant* tenant);
DLLEXPORT void fdb_tenant_destroy(FDBTenant* tenant); DLLEXPORT void fdb_tenant_destroy(FDBTenant* tenant);

46
bindings/c/test/apitester/TesterApiWorkload.cpp
View File

@ -380,7 +380,6 @@ void ApiWorkload::setupBlobGranules(TTaskFct cont) {
void ApiWorkload::blobbifyTenant(std::optional<int> tenantId, void ApiWorkload::blobbifyTenant(std::optional<int> tenantId,
std::shared_ptr<std::atomic<int>> blobbifiedCount, std::shared_ptr<std::atomic<int>> blobbifiedCount,
TTaskFct cont) { TTaskFct cont) {
auto retBlobbifyRange = std::make_shared<bool>(false);
execOperation( execOperation(
[=](auto ctx) { [=](auto ctx) {
fdb::Key begin(1, '\x00'); fdb::Key begin(1, '\x00');
@ -388,48 +387,17 @@ void ApiWorkload::blobbifyTenant(std::optional<int> tenantId,
info(fmt::format("setup: blobbifying {}: [\\x00 - \\xff)\n", debugTenantStr(tenantId))); info(fmt::format("setup: blobbifying {}: [\\x00 - \\xff)\n", debugTenantStr(tenantId)));
fdb::Future f = ctx->dbOps()->blobbifyRange(begin, end).eraseType(); // wait for blobbification before returning
ctx->continueAfter(f, [ctx, retBlobbifyRange, f]() { fdb::Future f = ctx->dbOps()->blobbifyRangeBlocking(begin, end).eraseType();
*retBlobbifyRange = f.get<fdb::future_var::Bool>(); ctx->continueAfter(f, [ctx, f]() {
bool success = f.get<fdb::future_var::Bool>();
ASSERT(success);
ctx->done(); ctx->done();
}); });
}, },
[=]() { [=]() {
if (!*retBlobbifyRange) { if (blobbifiedCount->fetch_sub(1) == 1) {
schedule([=]() { blobbifyTenant(tenantId, blobbifiedCount, cont); }); schedule(cont);
} else {
schedule([=]() { verifyTenant(tenantId, blobbifiedCount, cont); });
}
},
/*tenant=*/getTenant(tenantId),
/* failOnError = */ false);
}
void ApiWorkload::verifyTenant(std::optional<int> tenantId,
std::shared_ptr<std::atomic<int>> blobbifiedCount,
TTaskFct cont) {
auto retVerifyVersion = std::make_shared<int64_t>(-1);
execOperation(
[=](auto ctx) {
fdb::Key begin(1, '\x00');
fdb::Key end(1, '\xff');
info(fmt::format("setup: verifying {}: [\\x00 - \\xff)\n", debugTenantStr(tenantId)));
fdb::Future f = ctx->dbOps()->verifyBlobRange(begin, end, /*latest_version*/ -2).eraseType();
ctx->continueAfter(f, [ctx, retVerifyVersion, f]() {
*retVerifyVersion = f.get<fdb::future_var::Int64>();
ctx->done();
});
},
[=]() {
if (*retVerifyVersion == -1) {
schedule([=]() { verifyTenant(tenantId, blobbifiedCount, cont); });
} else {
if (blobbifiedCount->fetch_sub(1) == 1) {
schedule(cont);
}
} }
}, },
/*tenant=*/getTenant(tenantId), /*tenant=*/getTenant(tenantId),

1
bindings/c/test/apitester/TesterApiWorkload.h
View File

@ -135,7 +135,6 @@ protected:
// Generic BlobGranules setup. // Generic BlobGranules setup.
void setupBlobGranules(TTaskFct cont); void setupBlobGranules(TTaskFct cont);
void blobbifyTenant(std::optional<int> tenantId, std::shared_ptr<std::atomic<int>> blobbifiedCount, TTaskFct cont); void blobbifyTenant(std::optional<int> tenantId, std::shared_ptr<std::atomic<int>> blobbifiedCount, TTaskFct cont);
void verifyTenant(std::optional<int> tenantId, std::shared_ptr<std::atomic<int>> blobbifiedCount, TTaskFct cont);
private: private:
void populateDataTx(TTaskFct cont, std::optional<int> tenantId); void populateDataTx(TTaskFct cont, std::optional<int> tenantId);

65
bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp
View File

@ -37,6 +37,9 @@ public:
if (Random::get().randomInt(0, 1) == 0) { if (Random::get().randomInt(0, 1) == 0) {
excludedOpTypes.push_back(OP_CLEAR_RANGE); excludedOpTypes.push_back(OP_CLEAR_RANGE);
} }
if (Random::get().randomInt(0, 1) == 0) {
excludedOpTypes.push_back(OP_FLUSH);
}
} }
private: private:
@ -51,7 +54,8 @@ private:
OP_GET_BLOB_RANGES, OP_GET_BLOB_RANGES,
OP_VERIFY, OP_VERIFY,
OP_READ_DESC, OP_READ_DESC,
OP_LAST = OP_READ_DESC OP_FLUSH,
OP_LAST = OP_FLUSH
}; };
std::vector<OpType> excludedOpTypes; std::vector<OpType> excludedOpTypes;
@ -303,7 +307,10 @@ private:
fdb::native::FDBReadBlobGranuleContext& bgCtx, fdb::native::FDBReadBlobGranuleContext& bgCtx,
fdb::GranuleFilePointer snapshotFile, fdb::GranuleFilePointer snapshotFile,
fdb::KeyRange keyRange, fdb::KeyRange keyRange,
fdb::native::FDBBGTenantPrefix const* tenantPrefix) { fdb::native::FDBBGTenantPrefix const* tenantPrefix,
int64_t& prevFileVersion) {
ASSERT(snapshotFile.fileVersion > prevFileVersion);
prevFileVersion = snapshotFile.fileVersion;
if (validatedFiles.contains(snapshotFile.filename)) { if (validatedFiles.contains(snapshotFile.filename)) {
return; return;
} }
@ -339,7 +346,10 @@ private:
fdb::GranuleFilePointer deltaFile, fdb::GranuleFilePointer deltaFile,
fdb::KeyRange keyRange, fdb::KeyRange keyRange,
fdb::native::FDBBGTenantPrefix const* tenantPrefix, fdb::native::FDBBGTenantPrefix const* tenantPrefix,
int64_t& lastDFMaxVersion) { int64_t& lastDFMaxVersion,
int64_t& prevFileVersion) {
ASSERT(deltaFile.fileVersion > prevFileVersion);
prevFileVersion = deltaFile.fileVersion;
if (validatedFiles.contains(deltaFile.filename)) { if (validatedFiles.contains(deltaFile.filename)) {
return; return;
} }
@ -380,6 +390,9 @@ private:
} }
lastDFMaxVersion = std::max(lastDFMaxVersion, thisDFMaxVersion); lastDFMaxVersion = std::max(lastDFMaxVersion, thisDFMaxVersion);
// can be higher due to empty versions but must not be lower
ASSERT(lastDFMaxVersion <= prevFileVersion);
// TODO have delta mutations update map // TODO have delta mutations update map
} }
@ -392,22 +405,28 @@ private:
ASSERT(desc.keyRange.beginKey < desc.keyRange.endKey); ASSERT(desc.keyRange.beginKey < desc.keyRange.endKey);
ASSERT(tenantId.has_value() == desc.tenantPrefix.present); ASSERT(tenantId.has_value() == desc.tenantPrefix.present);
// beginVersion of zero means snapshot present // beginVersion of zero means snapshot present
int64_t prevFileVersion = 0;
// validate snapshot file // validate snapshot file
ASSERT(desc.snapshotFile.has_value()); ASSERT(desc.snapshotFile.has_value());
if (BG_API_DEBUG_VERBOSE) { if (BG_API_DEBUG_VERBOSE) {
info(fmt::format("Loading snapshot file {0}\n", fdb::toCharsRef(desc.snapshotFile->filename))); info(fmt::format("Loading snapshot file {0}\n", fdb::toCharsRef(desc.snapshotFile->filename)));
} }
validateSnapshotData(ctx, bgCtx, *desc.snapshotFile, desc.keyRange, &desc.tenantPrefix); validateSnapshotData(ctx, bgCtx, *desc.snapshotFile, desc.keyRange, &desc.tenantPrefix, prevFileVersion);
// validate delta files // validate delta files
int64_t lastDFMaxVersion = 0; int64_t lastDFMaxVersion = 0;
for (int i = 0; i < desc.deltaFiles.size(); i++) { for (int i = 0; i < desc.deltaFiles.size(); i++) {
validateDeltaData(ctx, bgCtx, desc.deltaFiles[i], desc.keyRange, &desc.tenantPrefix, lastDFMaxVersion); validateDeltaData(
ctx, bgCtx, desc.deltaFiles[i], desc.keyRange, &desc.tenantPrefix, lastDFMaxVersion, prevFileVersion);
} }
// validate memory mutations // validate memory mutations
int64_t lastVersion = 0; if (desc.memoryMutations.size()) {
ASSERT(desc.memoryMutations.front().version > lastDFMaxVersion);
ASSERT(desc.memoryMutations.front().version > prevFileVersion);
}
int64_t lastVersion = prevFileVersion;
for (int i = 0; i < desc.memoryMutations.size(); i++) { for (int i = 0; i < desc.memoryMutations.size(); i++) {
fdb::GranuleMutation& m = desc.memoryMutations[i]; fdb::GranuleMutation& m = desc.memoryMutations[i];
ASSERT(m.type == 0 || m.type == 1); ASSERT(m.type == 0 || m.type == 1);
@ -494,6 +513,33 @@ private:
getTenant(tenantId)); getTenant(tenantId));
} }
void randomFlushOp(TTaskFct cont, std::optional<int> tenantId) {
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::native::fdb_bool_t compact = Random::get().randomBool(0.5);
auto result = std::make_shared<bool>(false);
debugOp(compact ? "Flush" : "Compact", keyRange, tenantId, "starting");
execOperation(
[keyRange, compact, result](auto ctx) {
fdb::Future f =
ctx->dbOps()
->flushBlobRange(keyRange.beginKey, keyRange.endKey, compact, -2 /* latest version*/)
.eraseType();
ctx->continueAfter(f, [ctx, result, f]() {
*result = f.get<fdb::future_var::Bool>();
ctx->done();
});
},
[this, keyRange, compact, result, tenantId, cont]() {
ASSERT(*result);
debugOp(compact ? "Flush " : "Compact ", keyRange, tenantId, "Complete");
schedule(cont);
},
getTenant(tenantId),
/* failOnError = */ false);
}
void randomOperation(TTaskFct cont) override { void randomOperation(TTaskFct cont) override {
std::optional<int> tenantId = randomTenant(); std::optional<int> tenantId = randomTenant();
@ -530,6 +576,13 @@ private:
case OP_READ_DESC: case OP_READ_DESC:
randomReadDescription(cont, tenantId); randomReadDescription(cont, tenantId);
break; break;
case OP_FLUSH:
randomFlushOp(cont, tenantId);
// don't do too many flushes because they're expensive
if (Random::get().randomInt(0, 1) == 0) {
excludedOpTypes.push_back(OP_FLUSH);
}
break;
} }
} }
}; };

143
bindings/c/test/apitester/TesterBlobGranuleErrorsWorkload.cpp
View File

@ -44,7 +44,9 @@ private:
OP_CANCEL_BLOBBIFY, OP_CANCEL_BLOBBIFY,
OP_CANCEL_UNBLOBBIFY, OP_CANCEL_UNBLOBBIFY,
OP_CANCEL_PURGE, OP_CANCEL_PURGE,
OP_LAST = OP_CANCEL_PURGE OP_CANCEL_FLUSH,
OP_FLUSH_TOO_OLD,
OP_LAST = OP_FLUSH_TOO_OLD
}; };
void setup(TTaskFct cont) override { setupBlobGranules(cont); } void setup(TTaskFct cont) override { setupBlobGranules(cont); }
@ -122,14 +124,10 @@ private:
void randomPurgeUnalignedOp(TTaskFct cont) { void randomPurgeUnalignedOp(TTaskFct cont) {
// blobbify/unblobbify need to be aligned to blob range boundaries, so this should always fail // blobbify/unblobbify need to be aligned to blob range boundaries, so this should always fail
fdb::Key begin = randomKeyName(); fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation( execOperation(
[this, begin, end](auto ctx) { [this, keyRange](auto ctx) {
fdb::Future f = ctx->db().purgeBlobGranules(begin, end, -2, false).eraseType(); fdb::Future f = ctx->db().purgeBlobGranules(keyRange.beginKey, keyRange.endKey, -2, false).eraseType();
ctx->continueAfter( ctx->continueAfter(
f, f,
[this, ctx, f]() { [this, ctx, f]() {
@ -144,16 +142,12 @@ private:
void randomBlobbifyUnalignedOp(bool blobbify, TTaskFct cont) { void randomBlobbifyUnalignedOp(bool blobbify, TTaskFct cont) {
// blobbify/unblobbify need to be aligned to blob range boundaries, so this should always return false // blobbify/unblobbify need to be aligned to blob range boundaries, so this should always return false
fdb::Key begin = randomKeyName(); fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
auto success = std::make_shared<bool>(false); auto success = std::make_shared<bool>(false);
execOperation( execOperation(
[begin, end, blobbify, success](auto ctx) { [keyRange, blobbify, success](auto ctx) {
fdb::Future f = blobbify ? ctx->db().blobbifyRange(begin, end).eraseType() fdb::Future f = blobbify ? ctx->db().blobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType()
: ctx->db().unblobbifyRange(begin, end).eraseType(); : ctx->db().unblobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType();
ctx->continueAfter( ctx->continueAfter(
f, f,
[ctx, f, success]() { [ctx, f, success]() {
@ -169,103 +163,116 @@ private:
} }
void randomCancelGetGranulesOp(TTaskFct cont) { void randomCancelGetGranulesOp(TTaskFct cont) {
fdb::Key begin = randomKeyName(); fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execTransaction( execTransaction(
[begin, end](auto ctx) { [keyRange](auto ctx) {
fdb::Future f = ctx->tx().getBlobGranuleRanges(begin, end, 1000).eraseType(); fdb::Future f = ctx->tx().getBlobGranuleRanges(keyRange.beginKey, keyRange.endKey, 1000).eraseType();
ctx->done(); ctx->done();
}, },
[this, cont]() { schedule(cont); }); [this, cont]() { schedule(cont); });
} }
void randomCancelGetRangesOp(TTaskFct cont) { void randomCancelGetRangesOp(TTaskFct cont) {
fdb::Key begin = randomKeyName(); fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation( execOperation(
[begin, end](auto ctx) { [keyRange](auto ctx) {
fdb::Future f = ctx->db().listBlobbifiedRanges(begin, end, 1000).eraseType(); fdb::Future f = ctx->db().listBlobbifiedRanges(keyRange.beginKey, keyRange.endKey, 1000).eraseType();
ctx->done(); ctx->done();
}, },
[this, cont]() { schedule(cont); }); [this, cont]() { schedule(cont); });
} }
void randomCancelVerifyOp(TTaskFct cont) { void randomCancelVerifyOp(TTaskFct cont) {
fdb::Key begin = randomKeyName(); fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation( execOperation(
[begin, end](auto ctx) { [keyRange](auto ctx) {
fdb::Future f = ctx->db().verifyBlobRange(begin, end, -2 /* latest version*/).eraseType(); fdb::Future f =
ctx->db().verifyBlobRange(keyRange.beginKey, keyRange.endKey, -2 /* latest version*/).eraseType();
ctx->done(); ctx->done();
}, },
[this, cont]() { schedule(cont); }); [this, cont]() { schedule(cont); });
} }
void randomCancelSummarizeOp(TTaskFct cont) { void randomCancelSummarizeOp(TTaskFct cont) {
fdb::Key begin = randomKeyName(); fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execTransaction( execTransaction(
[begin, end](auto ctx) { [keyRange](auto ctx) {
fdb::Future f = ctx->tx().summarizeBlobGranules(begin, end, -2, 1000).eraseType(); fdb::Future f =
ctx->tx().summarizeBlobGranules(keyRange.beginKey, keyRange.endKey, -2, 1000).eraseType();
ctx->done(); ctx->done();
}, },
[this, cont]() { schedule(cont); }); [this, cont]() { schedule(cont); });
} }
void randomCancelBlobbifyOp(TTaskFct cont) { void randomCancelBlobbifyOp(TTaskFct cont) {
fdb::Key begin = randomKeyName(); fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation( execOperation(
[begin, end](auto ctx) { [keyRange](auto ctx) {
fdb::Future f = ctx->db().blobbifyRange(begin, end).eraseType(); fdb::Future f;
if (Random::get().randomBool(0.5)) {
f = ctx->db().blobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType();
} else {
f = ctx->db().blobbifyRangeBlocking(keyRange.beginKey, keyRange.endKey).eraseType();
}
ctx->done(); ctx->done();
}, },
[this, cont]() { schedule(cont); }); [this, cont]() { schedule(cont); });
} }
void randomCancelUnblobbifyOp(TTaskFct cont) { void randomCancelUnblobbifyOp(TTaskFct cont) {
fdb::Key begin = randomKeyName(); fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation( execOperation(
[begin, end](auto ctx) { [keyRange](auto ctx) {
fdb::Future f = ctx->db().unblobbifyRange(begin, end).eraseType(); fdb::Future f = ctx->db().unblobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType();
ctx->done(); ctx->done();
}, },
[this, cont]() { schedule(cont); }); [this, cont]() { schedule(cont); });
} }
void randomCancelPurgeOp(TTaskFct cont) { void randomCancelPurgeOp(TTaskFct cont) {
fdb::Key begin = randomKeyName(); fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation( execOperation(
[begin, end](auto ctx) { [keyRange](auto ctx) {
fdb::Future f = ctx->db().purgeBlobGranules(begin, end, -2, false).eraseType(); fdb::Future f = ctx->db().purgeBlobGranules(keyRange.beginKey, keyRange.endKey, -2, false).eraseType();
ctx->done(); ctx->done();
}, },
[this, cont]() { schedule(cont); }); [this, cont]() { schedule(cont); });
} }
void randomCancelFlushOp(TTaskFct cont) {
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::native::fdb_bool_t compact = Random::get().randomBool(0.5);
execOperation(
[keyRange, compact](auto ctx) {
fdb::Future f = ctx->db().flushBlobRange(keyRange.beginKey, keyRange.endKey, compact, -2).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}
void randomFlushTooOldOp(TTaskFct cont) {
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::native::fdb_bool_t compact = Random::get().randomBool(0.5);
auto success = std::make_shared<bool>(false);
execOperation(
[keyRange, compact, success](auto ctx) {
// version of 1 should be too old
fdb::Future f = ctx->db().flushBlobRange(keyRange.beginKey, keyRange.endKey, compact, 1).eraseType();
ctx->continueAfter(
f,
[ctx, f, success]() {
*success = f.get<fdb::future_var::Bool>();
ctx->done();
},
true);
},
[this, cont, success]() {
ASSERT(!(*success));
schedule(cont);
});
}
void randomOperation(TTaskFct cont) override { void randomOperation(TTaskFct cont) override {
OpType txType = (OpType)Random::get().randomInt(0, OP_LAST); OpType txType = (OpType)Random::get().randomInt(0, OP_LAST);
switch (txType) { switch (txType) {
@ -309,6 +316,12 @@ private:
case OP_CANCEL_PURGE: case OP_CANCEL_PURGE:
randomCancelPurgeOp(cont); randomCancelPurgeOp(cont);
break; break;
case OP_CANCEL_FLUSH:
randomCancelPurgeOp(cont);
break;
case OP_FLUSH_TOO_OLD:
randomCancelPurgeOp(cont);
break;
} }
} }
}; };

32
bindings/c/test/fdb_api.hpp
View File

@ -86,6 +86,7 @@ struct GranuleFilePointer {
int64_t offset; int64_t offset;
int64_t length; int64_t length;
int64_t fullFileLength; int64_t fullFileLength;
int64_t fileVersion;
// just keep raw data structures to pass to callbacks // just keep raw data structures to pass to callbacks
native::FDBBGEncryptionCtx encryptionCtx; native::FDBBGEncryptionCtx encryptionCtx;
@ -95,6 +96,7 @@ struct GranuleFilePointer {
offset = nativePointer.file_offset; offset = nativePointer.file_offset;
length = nativePointer.file_length; length = nativePointer.file_length;
fullFileLength = nativePointer.full_file_length; fullFileLength = nativePointer.full_file_length;
fileVersion = nativePointer.file_version;
encryptionCtx = nativePointer.encryption_ctx; encryptionCtx = nativePointer.encryption_ctx;
} }
}; };
@ -829,11 +831,13 @@ public:
virtual Transaction createTransaction() = 0; virtual Transaction createTransaction() = 0;
virtual TypedFuture<future_var::Bool> blobbifyRange(KeyRef begin, KeyRef end) = 0; virtual TypedFuture<future_var::Bool> blobbifyRange(KeyRef begin, KeyRef end) = 0;
virtual TypedFuture<future_var::Bool> blobbifyRangeBlocking(KeyRef begin, KeyRef end) = 0;
virtual TypedFuture<future_var::Bool> unblobbifyRange(KeyRef begin, KeyRef end) = 0; virtual TypedFuture<future_var::Bool> unblobbifyRange(KeyRef begin, KeyRef end) = 0;
virtual TypedFuture<future_var::KeyRangeRefArray> listBlobbifiedRanges(KeyRef begin, virtual TypedFuture<future_var::KeyRangeRefArray> listBlobbifiedRanges(KeyRef begin,
KeyRef end, KeyRef end,
int rangeLimit) = 0; int rangeLimit) = 0;
virtual TypedFuture<future_var::Int64> verifyBlobRange(KeyRef begin, KeyRef end, int64_t version) = 0; virtual TypedFuture<future_var::Int64> verifyBlobRange(KeyRef begin, KeyRef end, int64_t version) = 0;
virtual TypedFuture<future_var::Bool> flushBlobRange(KeyRef begin, KeyRef end, bool compact, int64_t version) = 0;
virtual TypedFuture<future_var::KeyRef> purgeBlobGranules(KeyRef begin, virtual TypedFuture<future_var::KeyRef> purgeBlobGranules(KeyRef begin,
KeyRef end, KeyRef end,
int64_t version, int64_t version,
@ -901,6 +905,13 @@ public:
return native::fdb_tenant_blobbify_range(tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end)); return native::fdb_tenant_blobbify_range(tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end));
} }
TypedFuture<future_var::Bool> blobbifyRangeBlocking(KeyRef begin, KeyRef end) override {
if (!tenant)
throw std::runtime_error("blobbifyRangeBlocking() from null tenant");
return native::fdb_tenant_blobbify_range_blocking(
tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end));
}
TypedFuture<future_var::Bool> unblobbifyRange(KeyRef begin, KeyRef end) override { TypedFuture<future_var::Bool> unblobbifyRange(KeyRef begin, KeyRef end) override {
if (!tenant) if (!tenant)
throw std::runtime_error("unblobbifyRange() from null tenant"); throw std::runtime_error("unblobbifyRange() from null tenant");
@ -922,6 +933,13 @@ public:
tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end), version); tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end), version);
} }
TypedFuture<future_var::Bool> flushBlobRange(KeyRef begin, KeyRef end, bool compact, int64_t version) override {
if (!tenant)
throw std::runtime_error("flushBlobRange() from null tenant");
return native::fdb_tenant_flush_blob_range(
tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end), compact, version);
}
TypedFuture<future_var::Int64> getId() { TypedFuture<future_var::Int64> getId() {
if (!tenant) if (!tenant)
throw std::runtime_error("getId() from null tenant"); throw std::runtime_error("getId() from null tenant");
@ -1026,12 +1044,26 @@ public:
db.get(), begin.data(), intSize(begin), end.data(), intSize(end), version); db.get(), begin.data(), intSize(begin), end.data(), intSize(end), version);
} }
TypedFuture<future_var::Bool> flushBlobRange(KeyRef begin, KeyRef end, bool compact, int64_t version) override {
if (!db)
throw std::runtime_error("flushBlobRange from null database");
return native::fdb_database_flush_blob_range(
db.get(), begin.data(), intSize(begin), end.data(), intSize(end), compact, version);
}
TypedFuture<future_var::Bool> blobbifyRange(KeyRef begin, KeyRef end) override { TypedFuture<future_var::Bool> blobbifyRange(KeyRef begin, KeyRef end) override {
if (!db) if (!db)
throw std::runtime_error("blobbifyRange from null database"); throw std::runtime_error("blobbifyRange from null database");
return native::fdb_database_blobbify_range(db.get(), begin.data(), intSize(begin), end.data(), intSize(end)); return native::fdb_database_blobbify_range(db.get(), begin.data(), intSize(begin), end.data(), intSize(end));
} }
TypedFuture<future_var::Bool> blobbifyRangeBlocking(KeyRef begin, KeyRef end) override {
if (!db)
throw std::runtime_error("blobbifyRangeBlocking from null database");
return native::fdb_database_blobbify_range_blocking(
db.get(), begin.data(), intSize(begin), end.data(), intSize(end));
}
TypedFuture<future_var::Bool> unblobbifyRange(KeyRef begin, KeyRef end) override { TypedFuture<future_var::Bool> unblobbifyRange(KeyRef begin, KeyRef end) override {
if (!db) if (!db)
throw std::runtime_error("unblobbifyRange from null database"); throw std::runtime_error("unblobbifyRange from null database");

148
bindings/java/fdbJNI.cpp
View File

@ -942,6 +942,41 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1blobbi
return (jlong)f; return (jlong)f;
} }
JNIEXPORT jlong JNICALL
Java_com_apple_foundationdb_FDBDatabase_Database_1blobbifyRangeBlocking(JNIEnv* jenv,
jobject,
jlong dbPtr,
jbyteArray beginKeyBytes,
jbyteArray endKeyBytes) {
if (!dbPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv);
return 0;
}
FDBDatabase* database = (FDBDatabase*)dbPtr;
uint8_t* beginKeyArr = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
if (!beginKeyArr) {
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
uint8_t* endKeyArr = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
if (!endKeyArr) {
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
FDBFuture* f = fdb_database_blobbify_range_blocking(
database, beginKeyArr, jenv->GetArrayLength(beginKeyBytes), endKeyArr, jenv->GetArrayLength(endKeyBytes));
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKeyArr, JNI_ABORT);
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1unblobbifyRange(JNIEnv* jenv, JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1unblobbifyRange(JNIEnv* jenv,
jobject, jobject,
jlong dbPtr, jlong dbPtr,
@ -1058,6 +1093,46 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1getCli
return (jlong)f; return (jlong)f;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1flushBlobRange(JNIEnv* jenv,
jobject,
jlong dbPtr,
jbyteArray beginKeyBytes,
jbyteArray endKeyBytes,
jboolean compact,
jlong version) {
if (!dbPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv);
return 0;
}
FDBDatabase* tr = (FDBDatabase*)dbPtr;
uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
if (!startKey) {
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
if (!endKey) {
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
FDBFuture* f = fdb_database_flush_blob_range(tr,
startKey,
jenv->GetArrayLength(beginKeyBytes),
endKey,
jenv->GetArrayLength(endKeyBytes),
(fdb_bool_t)compact,
version);
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT);
return (jlong)f;
}
JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv* jenv, JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv* jenv,
jobject, jobject,
jint predicate, jint predicate,
@ -1212,6 +1287,39 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1blobbifyRa
return (jlong)f; return (jlong)f;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1blobbifyRangeBlocking(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray beginKeyBytes,
jbyteArray endKeyBytes) {
if (!tPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv);
return 0;
}
FDBTenant* tenant = (FDBTenant*)tPtr;
uint8_t* beginKeyArr = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
if (!beginKeyArr) {
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
uint8_t* endKeyArr = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
if (!endKeyArr) {
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
FDBFuture* f = fdb_tenant_blobbify_range_blocking(
tenant, beginKeyArr, jenv->GetArrayLength(beginKeyBytes), endKeyArr, jenv->GetArrayLength(endKeyBytes));
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKeyArr, JNI_ABORT);
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1unblobbifyRange(JNIEnv* jenv, JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1unblobbifyRange(JNIEnv* jenv,
jobject, jobject,
jlong tPtr, jlong tPtr,
@ -1313,6 +1421,46 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1verifyBlob
return (jlong)f; return (jlong)f;
} }
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1flushBlobRange(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray beginKeyBytes,
jbyteArray endKeyBytes,
jboolean compact,
jlong version) {
if (!tPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv);
return 0;
}
FDBTenant* tenant = (FDBTenant*)tPtr;
uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
if (!startKey) {
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
if (!endKey) {
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
FDBFuture* f = fdb_tenant_flush_blob_range(tenant,
startKey,
jenv->GetArrayLength(beginKeyBytes),
endKey,
jenv->GetArrayLength(endKeyBytes),
(fdb_bool_t)compact,
version);
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT);
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1getId(JNIEnv* jenv, jobject, jlong tPtr) { JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1getId(JNIEnv* jenv, jobject, jlong tPtr) {
if (!tPtr) { if (!tPtr) {
throwParamNotNull(jenv); throwParamNotNull(jenv);

47
bindings/java/src/integration/com/apple/foundationdb/BlobGranuleIntegrationTest.java
View File

@ -50,20 +50,6 @@ class BlobGranuleIntegrationTest {
} }
} }
void waitForVerify(Database db, Range blobRange) throws InterruptedException {
System.out.println("Verifying");
while (true) {
Long verifyVersion = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
if (verifyVersion != null && verifyVersion > 0) {
System.out.println("Verify succeeded @ " + verifyVersion);
return;
} else {
System.out.println("Verify failed, sleeping");
Thread.sleep(100);
}
}
}
@Test @Test
void blobManagementFunctionsTest() throws Exception { void blobManagementFunctionsTest() throws Exception {
/* /*
@ -79,9 +65,12 @@ class BlobGranuleIntegrationTest {
Range blobRange = Range.startsWith(key); Range blobRange = Range.startsWith(key);
try (Database db = fdb.open()) { try (Database db = fdb.open()) {
db.blobbifyRange(blobRange.begin, blobRange.end).join(); boolean blobbifySuccess = db.blobbifyRangeBlocking(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(blobbifySuccess);
waitForVerify(db, blobRange); Long verifyVersion = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(verifyVersion >= 0);
// list blob ranges // list blob ranges
KeyRangeArrayResult blobRanges = db.listBlobbifiedRanges(blobRange.begin, blobRange.end, 2).join(); KeyRangeArrayResult blobRanges = db.listBlobbifiedRanges(blobRange.begin, blobRange.end, 2).join();
@ -89,23 +78,41 @@ class BlobGranuleIntegrationTest {
Assertions.assertArrayEquals(blobRange.begin, blobRanges.getKeyRanges().get(0).begin); Assertions.assertArrayEquals(blobRange.begin, blobRanges.getKeyRanges().get(0).begin);
Assertions.assertArrayEquals(blobRange.end, blobRanges.getKeyRanges().get(0).end); Assertions.assertArrayEquals(blobRange.end, blobRanges.getKeyRanges().get(0).end);
boolean flushSuccess = db.flushBlobRange(blobRange.begin, blobRange.end, false).join();
Assertions.assertTrue(flushSuccess);
// verify after flush
Long verifyVersionAfterFlush = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(verifyVersionAfterFlush >= 0);
Assertions.assertTrue(verifyVersionAfterFlush >= verifyVersion);
boolean compactSuccess = db.flushBlobRange(blobRange.begin, blobRange.end, true).join();
Assertions.assertTrue(compactSuccess);
Long verifyVersionAfterCompact = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(verifyVersionAfterCompact >= 0);
Assertions.assertTrue(verifyVersionAfterCompact >= verifyVersionAfterFlush);
// purge/wait // purge/wait
byte[] purgeKey = db.purgeBlobGranules(blobRange.begin, blobRange.end, -2, false).join(); byte[] purgeKey = db.purgeBlobGranules(blobRange.begin, blobRange.end, -2, false).join();
db.waitPurgeGranulesComplete(purgeKey).join(); db.waitPurgeGranulesComplete(purgeKey).join();
// verify again // verify again
waitForVerify(db, blobRange); Long verifyVersionAfterPurge = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(verifyVersionAfterPurge >= 0);
Assertions.assertTrue(verifyVersionAfterPurge >= verifyVersionAfterCompact);
// force purge/wait // force purge/wait
byte[] forcePurgeKey = db.purgeBlobGranules(blobRange.begin, blobRange.end, -2, true).join(); byte[] forcePurgeKey = db.purgeBlobGranules(blobRange.begin, blobRange.end, -2, true).join();
db.waitPurgeGranulesComplete(forcePurgeKey).join(); db.waitPurgeGranulesComplete(forcePurgeKey).join();
// check verify fails // check verify fails
Long verifyVersion = db.verifyBlobRange(blobRange.begin, blobRange.end).join(); Long verifyVersionLast = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
Assertions.assertEquals(-1, verifyVersion); Assertions.assertEquals(-1, verifyVersionLast);
// unblobbify // unblobbify
db.unblobbifyRange(blobRange.begin, blobRange.end).join(); boolean unblobbifySuccess = db.unblobbifyRange(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(unblobbifySuccess);
System.out.println("Blob granule management tests complete!"); System.out.println("Blob granule management tests complete!");
} }

65
bindings/java/src/main/com/apple/foundationdb/Database.java
View File

@ -246,6 +246,31 @@ public interface Database extends AutoCloseable, TransactionContext {
*/ */
CompletableFuture<Boolean> blobbifyRange(byte[] beginKey, byte[] endKey, Executor e); CompletableFuture<Boolean> blobbifyRange(byte[] beginKey, byte[] endKey, Executor e);
/**
* Runs {@link #blobbifyRange(byte[] beginKey, byte[] endKey, boolean wait)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param wait wait for blobbification to complete
* @return if the recording of the range was successful
*/
default CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey) {
return blobbifyRangeBlocking(beginKey, endKey, getExecutor());
}
/**
* Sets a range to be blobbified in the database. Must be a completely unblobbified range.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param wait wait for blobbification to complete
* @param e the {@link Executor} to use for asynchronous callbacks
* @return if the recording of the range was successful
*/
CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e);
/** /**
* Runs {@link #unblobbifyRange(byte[] beginKey, byte[] endKey)} on the default executor. * Runs {@link #unblobbifyRange(byte[] beginKey, byte[] endKey)} on the default executor.
* *
@ -331,6 +356,46 @@ public interface Database extends AutoCloseable, TransactionContext {
*/ */
CompletableFuture<Long> verifyBlobRange(byte[] beginKey, byte[] endKey, long version, Executor e); CompletableFuture<Long> verifyBlobRange(byte[] beginKey, byte[] endKey, long version, Executor e);
/**
* Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
*
* @return a future with a boolean for success or failure
*/
default CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact) {
return flushBlobRange(beginKey, endKey, compact, -2, getExecutor());
}
/**
* Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey, long version)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
* @param version version to flush at
*
* @return a future with a boolean for success or failure
*/
default CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version) {
return flushBlobRange(beginKey, endKey, compact, version, getExecutor());
}
/**
* Checks if a blob range is blobbified.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
* @param version version to flush at
* @param e the {@link Executor} to use for asynchronous callbacks
*
* @return a future with a boolean for success or failure
*/
CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e);
/** /**
* Runs a read-only transactional function against this {@code Database} with retry logic. * Runs a read-only transactional function against this {@code Database} with retry logic.
* {@link Function#apply(Object) apply(ReadTransaction)} will be called on the * {@link Function#apply(Object) apply(ReadTransaction)} will be called on the

22
bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java
View File

@ -229,6 +229,16 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume
} }
} }
@Override
public CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e) {
pointerReadLock.lock();
try {
return new FutureBool(Database_blobbifyRangeBlocking(getPtr(), beginKey, endKey), e);
} finally {
pointerReadLock.unlock();
}
}
@Override @Override
public CompletableFuture<Boolean> unblobbifyRange(byte[] beginKey, byte[] endKey, Executor e) { public CompletableFuture<Boolean> unblobbifyRange(byte[] beginKey, byte[] endKey, Executor e) {
pointerReadLock.lock(); pointerReadLock.lock();
@ -259,6 +269,16 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume
} }
} }
@Override
public CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e) {
pointerReadLock.lock();
try {
return new FutureBool(Database_flushBlobRange(getPtr(), beginKey, endKey, compact, version), e);
} finally {
pointerReadLock.unlock();
}
}
@Override @Override
public Executor getExecutor() { public Executor getExecutor() {
return executor; return executor;
@ -287,8 +307,10 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume
private native long Database_purgeBlobGranules(long cPtr, byte[] beginKey, byte[] endKey, long purgeVersion, boolean force); private native long Database_purgeBlobGranules(long cPtr, byte[] beginKey, byte[] endKey, long purgeVersion, boolean force);
private native long Database_waitPurgeGranulesComplete(long cPtr, byte[] purgeKey); private native long Database_waitPurgeGranulesComplete(long cPtr, byte[] purgeKey);
private native long Database_blobbifyRange(long cPtr, byte[] beginKey, byte[] endKey); private native long Database_blobbifyRange(long cPtr, byte[] beginKey, byte[] endKey);
private native long Database_blobbifyRangeBlocking(long cPtr, byte[] beginKey, byte[] endKey);
private native long Database_unblobbifyRange(long cPtr, byte[] beginKey, byte[] endKey); private native long Database_unblobbifyRange(long cPtr, byte[] beginKey, byte[] endKey);
private native long Database_listBlobbifiedRanges(long cPtr, byte[] beginKey, byte[] endKey, int rangeLimit); private native long Database_listBlobbifiedRanges(long cPtr, byte[] beginKey, byte[] endKey, int rangeLimit);
private native long Database_verifyBlobRange(long cPtr, byte[] beginKey, byte[] endKey, long version); private native long Database_verifyBlobRange(long cPtr, byte[] beginKey, byte[] endKey, long version);
private native long Database_flushBlobRange(long cPtr, byte[] beginKey, byte[] endKey, boolean compact, long version);
private native long Database_getClientStatus(long cPtr); private native long Database_getClientStatus(long cPtr);
} }

22
bindings/java/src/main/com/apple/foundationdb/FDBTenant.java
View File

@ -170,6 +170,16 @@ class FDBTenant extends NativeObjectWrapper implements Tenant {
} }
} }
@Override
public CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e) {
pointerReadLock.lock();
try {
return new FutureBool(Tenant_blobbifyRangeBlocking(getPtr(), beginKey, endKey), e);
} finally {
pointerReadLock.unlock();
}
}
@Override @Override
public CompletableFuture<Boolean> unblobbifyRange(byte[] beginKey, byte[] endKey, Executor e) { public CompletableFuture<Boolean> unblobbifyRange(byte[] beginKey, byte[] endKey, Executor e) {
pointerReadLock.lock(); pointerReadLock.lock();
@ -200,6 +210,16 @@ class FDBTenant extends NativeObjectWrapper implements Tenant {
} }
} }
@Override
public CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e) {
pointerReadLock.lock();
try {
return new FutureBool(Tenant_flushBlobRange(getPtr(), beginKey, endKey, compact, version), e);
} finally {
pointerReadLock.unlock();
}
}
@Override @Override
public CompletableFuture<Long> getId(Executor e) { public CompletableFuture<Long> getId(Executor e) {
pointerReadLock.lock(); pointerReadLock.lock();
@ -233,8 +253,10 @@ class FDBTenant extends NativeObjectWrapper implements Tenant {
private native long Tenant_purgeBlobGranules(long cPtr, byte[] beginKey, byte[] endKey, long purgeVersion, boolean force); private native long Tenant_purgeBlobGranules(long cPtr, byte[] beginKey, byte[] endKey, long purgeVersion, boolean force);
private native long Tenant_waitPurgeGranulesComplete(long cPtr, byte[] purgeKey); private native long Tenant_waitPurgeGranulesComplete(long cPtr, byte[] purgeKey);
private native long Tenant_blobbifyRange(long cPtr, byte[] beginKey, byte[] endKey); private native long Tenant_blobbifyRange(long cPtr, byte[] beginKey, byte[] endKey);
private native long Tenant_blobbifyRangeBlocking(long cPtr, byte[] beginKey, byte[] endKey);
private native long Tenant_unblobbifyRange(long cPtr, byte[] beginKey, byte[] endKey); private native long Tenant_unblobbifyRange(long cPtr, byte[] beginKey, byte[] endKey);
private native long Tenant_listBlobbifiedRanges(long cPtr, byte[] beginKey, byte[] endKey, int rangeLimit); private native long Tenant_listBlobbifiedRanges(long cPtr, byte[] beginKey, byte[] endKey, int rangeLimit);
private native long Tenant_verifyBlobRange(long cPtr, byte[] beginKey, byte[] endKey, long version); private native long Tenant_verifyBlobRange(long cPtr, byte[] beginKey, byte[] endKey, long version);
private native long Tenant_flushBlobRange(long cPtr, byte[] beginKey, byte[] endKey, boolean compact, long version);
private native long Tenant_getId(long cPtr); private native long Tenant_getId(long cPtr);
} }

65
bindings/java/src/main/com/apple/foundationdb/Tenant.java
View File

@ -333,6 +333,31 @@ public interface Tenant extends AutoCloseable, TransactionContext {
*/ */
CompletableFuture<Boolean> blobbifyRange(byte[] beginKey, byte[] endKey, Executor e); CompletableFuture<Boolean> blobbifyRange(byte[] beginKey, byte[] endKey, Executor e);
/**
* Runs {@link #blobbifyRange(byte[] beginKey, byte[] endKey, boolean wait)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param wait wait for blobbification to complete
* @return if the recording of the range was successful
*/
default CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey) {
return blobbifyRangeBlocking(beginKey, endKey, getExecutor());
}
/**
* Sets a range to be blobbified in this tenant. Must be a completely unblobbified range.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param wait wait for blobbification to complete
* @param e the {@link Executor} to use for asynchronous callbacks
* @return if the recording of the range was successful
*/
CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e);
/** /**
* Runs {@link #unblobbifyRange(byte[] beginKey, byte[] endKey)} on the default executor. * Runs {@link #unblobbifyRange(byte[] beginKey, byte[] endKey)} on the default executor.
* *
@ -418,6 +443,46 @@ public interface Tenant extends AutoCloseable, TransactionContext {
*/ */
CompletableFuture<Long> verifyBlobRange(byte[] beginKey, byte[] endKey, long version, Executor e); CompletableFuture<Long> verifyBlobRange(byte[] beginKey, byte[] endKey, long version, Executor e);
/**
* Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
*
* @return a future with a boolean for success or failure
*/
default CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact) {
return flushBlobRange(beginKey, endKey, compact, -2, getExecutor());
}
/**
* Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey, long version)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
* @param version version to flush at
*
* @return a future with a boolean for success or failure
*/
default CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version) {
return flushBlobRange(beginKey, endKey, compact, version, getExecutor());
}
/**
* Checks if a blob range is blobbified.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
* @param version version to flush at
* @param e the {@link Executor} to use for asynchronous callbacks
*
* @return a future with a boolean for success or failure
*/
CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e);
/** /**
* Runs {@link #getId()} on the default executor. * Runs {@link #getId()} on the default executor.
* *

20
fdbcli/BlobRangeCommand.actor.cpp
View File

@ -24,7 +24,6 @@
#include "fdbclient/IClientApi.h" #include "fdbclient/IClientApi.h"
#include "fdbclient/ManagementAPI.actor.h" #include "fdbclient/ManagementAPI.actor.h"
#include "fdbclient/NativeAPI.actor.h" #include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/BlobGranuleRequest.actor.h"
#include "flow/Arena.h" #include "flow/Arena.h"
#include "flow/FastRef.h" #include "flow/FastRef.h"
@ -90,17 +89,16 @@ ACTOR Future<Void> doBlobCheck(Database db, Key startKey, Key endKey, Optional<V
} }
ACTOR Future<Void> doBlobFlush(Database db, Key startKey, Key endKey, Optional<Version> version, bool compact) { ACTOR Future<Void> doBlobFlush(Database db, Key startKey, Key endKey, Optional<Version> version, bool compact) {
// TODO make DB function? state double elapsed = -timer_monotonic();
state Version flushVersion; state KeyRange keyRange(KeyRangeRef(startKey, endKey));
if (version.present()) { bool result = wait(db->flushBlobRange(keyRange, compact, version));
flushVersion = version.get(); elapsed += timer_monotonic();
} else {
wait(store(flushVersion, getLatestReadVersion(db)));
}
KeyRange range(KeyRangeRef(startKey, endKey)); fmt::print("Blob Flush [{0} - {1}) {2} in {3:.6f} seconds\n",
FlushGranuleRequest req(-1, range, flushVersion, compact); startKey.printable(),
wait(success(doBlobGranuleRequests(db, range, req, &BlobWorkerInterface::flushGranuleRequest))); endKey.printable(),
result ? "succeeded" : "failed",
elapsed);
return Void(); return Void();
} }

83
fdbcli/IdempotencyIdsCommand.actor.cpp Normal file
View File

@ -0,0 +1,83 @@
/*
* IdempotencyIdsCommand.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbcli/fdbcli.actor.h"
#include "fdbclient/IdempotencyId.actor.h"
#include "fdbclient/JsonBuilder.h"
#include "fdbclient/json_spirit/json_spirit_reader_template.h"
#include "flow/actorcompiler.h" // This must be the last include
namespace {
Optional<double> parseAgeValue(StringRef token) {
try {
return std::stod(token.toString());
} catch (...) {
return {};
}
}
} // namespace
namespace fdb_cli {
ACTOR Future<bool> idempotencyIdsCommandActor(Database db, std::vector<StringRef> tokens) {
if (tokens.size() < 2 || tokens.size() > 3) {
printUsage(tokens[0]);
return false;
} else {
auto const action = tokens[1];
if (action == "status"_sr) {
if (tokens.size() != 2) {
printUsage(tokens[0]);
return false;
}
JsonBuilderObject status = wait(getIdmpKeyStatus(db));
fmt::print("{}\n", status.getJson());
return true;
} else if (action == "clear"_sr) {
if (tokens.size() != 3) {
printUsage(tokens[0]);
return false;
}
auto const age = parseAgeValue(tokens[2]);
if (!age.present()) {
printUsage(tokens[0]);
return false;
}
wait(cleanIdempotencyIds(db, age.get()));
fmt::print("Successfully cleared idempotency IDs.\n");
return true;
} else {
printUsage(tokens[0]);
return false;
}
}
}
CommandFactory idempotencyIdsCommandFactory(
"idempotencyids",
CommandHelp(
"idempotencyids [status | clear <min_age_seconds>]",
"View status of idempotency ids, or reclaim space used by idempotency ids older than the given age",
"View status of idempotency ids currently in the cluster, or reclaim space by clearing all the idempotency ids "
"older than min_age_seconds (which will expire all transaction versions older than this age)\n"));
} // namespace fdb_cli

8
fdbcli/fdbcli.actor.cpp
View File

@ -2137,6 +2137,14 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
continue; continue;
} }
if (tokencmp(tokens[0], "idempotencyids")) {
bool _result = wait(makeInterruptable(idempotencyIdsCommandActor(localDb, tokens)));
if (!_result) {
is_error = true;
}
continue;
}
fprintf(stderr, "ERROR: Unknown command `%s'. Try `help'?\n", formatStringRef(tokens[0]).c_str()); fprintf(stderr, "ERROR: Unknown command `%s'. Try `help'?\n", formatStringRef(tokens[0]).c_str());
is_error = true; is_error = true;
} }

2
fdbcli/include/fdbcli/fdbcli.actor.h
View File

@ -274,6 +274,8 @@ ACTOR Future<bool> tssqCommandActor(Reference<IDatabase> db, std::vector<StringR
ACTOR Future<bool> versionEpochCommandActor(Reference<IDatabase> db, Database cx, std::vector<StringRef> tokens); ACTOR Future<bool> versionEpochCommandActor(Reference<IDatabase> db, Database cx, std::vector<StringRef> tokens);
// targetversion command // targetversion command
ACTOR Future<bool> targetVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens); ACTOR Future<bool> targetVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// idempotencyids command
ACTOR Future<bool> idempotencyIdsCommandActor(Database cx, std::vector<StringRef> tokens);
} // namespace fdb_cli } // namespace fdb_cli

1297
fdbcli/tests/fdbcli_tests.py
View File

File diff suppressed because it is too large Load Diff

113
fdbcli/tests/metacluster_fdbcli_tests.py
View File

@ -38,29 +38,7 @@ def enable_logging(level=logging.DEBUG):
return func_decorator return func_decorator
def run_command(*args):
commands = ["{}".format(args)]
print(commands)
try:
process = subprocess.run(commands, stdout=subprocess.PIPE, env=fdbcli_env, timeout=20)
return process.stdout.decode('utf-8').strip()
except subprocess.TimeoutExpired:
raise Exception('the command is stuck')
def run_fdbcli_command(cluster_file, *args): def run_fdbcli_command(cluster_file, *args):
command_template = [fdbcli_bin, '-C', "{}".format(cluster_file), '--exec']
commands = command_template + ["{}".format(' '.join(args))]
print(commands)
try:
# if the fdbcli command is stuck for more than 20 seconds, the database is definitely unavailable
process = subprocess.run(commands, stdout=subprocess.PIPE, env=fdbcli_env, timeout=20)
return process.stdout.decode('utf-8').strip()
except subprocess.TimeoutExpired:
raise Exception('The fdbcli command is stuck, database is unavailable')
def run_fdbcli_command_and_get_error(cluster_file, *args):
"""run the fdbcli statement: fdbcli --exec '<arg1> <arg2> ... <argN>'. """run the fdbcli statement: fdbcli --exec '<arg1> <arg2> ... <argN>'.
Returns: Returns:
@ -70,7 +48,10 @@ def run_fdbcli_command_and_get_error(cluster_file, *args):
commands = command_template + ["{}".format(' '.join(args))] commands = command_template + ["{}".format(' '.join(args))]
try: try:
process = subprocess.run(commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env, timeout=20) process = subprocess.run(commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env, timeout=20)
return process.stdout.decode('utf-8').strip(), process.stderr.decode('utf-8').strip() rc = process.returncode
out = process.stdout.decode('utf-8').strip()
err = process.stderr.decode('utf-8').strip()
return rc, out, err
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
raise Exception('The fdbcli command is stuck, database is unavailable') raise Exception('The fdbcli command is stuck, database is unavailable')
@ -81,35 +62,46 @@ def get_cluster_connection_str(cluster_file_path):
return conn_str return conn_str
def metacluster_create(cluster_file, name, tenant_id_prefix): @enable_logging()
return run_fdbcli_command(cluster_file, "metacluster create_experimental", name, str(tenant_id_prefix)) def metacluster_create(logger, cluster_file, name, tenant_id_prefix):
rc, out, err = run_fdbcli_command(cluster_file, "metacluster create_experimental", name, str(tenant_id_prefix))
if rc != 0:
raise Exception(err)
logger.debug(out)
logger.debug('Metacluster {} created'.format(name))
def metacluster_register(management_cluster_file, data_cluster_file, name): def metacluster_register(management_cluster_file, data_cluster_file, name, max_tenant_groups):
conn_str = get_cluster_connection_str(data_cluster_file) conn_str = get_cluster_connection_str(data_cluster_file)
return run_fdbcli_command(management_cluster_file, "metacluster register", name, "connection_string={}".format( rc, out, err = run_fdbcli_command(management_cluster_file,
conn_str), 'max_tenant_groups=6') "metacluster register",
name,
"connection_string={}".format(conn_str),
'max_tenant_groups={}'.format(max_tenant_groups))
if rc != 0:
raise Exception(err)
@enable_logging() @enable_logging()
def setup_metacluster(logger, management_cluster, data_clusters): def setup_metacluster(logger, management_cluster, data_clusters, max_tenant_groups_per_cluster):
management_cluster_file = management_cluster[0] management_cluster_file = management_cluster[0]
management_cluster_name = management_cluster[1] management_cluster_name = management_cluster[1]
tenant_id_prefix = random.randint(0, 32767) tenant_id_prefix = random.randint(0, 32767)
output = metacluster_create(management_cluster_file, management_cluster_name, tenant_id_prefix) logger.debug('management cluster: {}'.format(management_cluster_name))
logger.debug(output) logger.debug('data clusters: {}'.format([name for (cf, name) in data_clusters]))
metacluster_create(management_cluster_file, management_cluster_name, tenant_id_prefix)
for (cf, name) in data_clusters: for (cf, name) in data_clusters:
output = metacluster_register(management_cluster_file, cf, name) metacluster_register(management_cluster_file, cf, name, max_tenant_groups=max_tenant_groups_per_cluster)
logger.debug(output)
def metacluster_status(cluster_file): def metacluster_status(cluster_file):
return run_fdbcli_command(cluster_file, "metacluster status") _, out, _ = run_fdbcli_command(cluster_file, "metacluster status")
return out
def setup_tenants(management_cluster_file, data_cluster_files, tenants): def setup_tenants(management_cluster_file, data_cluster_files, tenants):
for tenant in tenants: for tenant in tenants:
output = run_fdbcli_command(management_cluster_file, 'tenant create', tenant) _, output, _ = run_fdbcli_command(management_cluster_file, 'tenant create', tenant)
expected_output = 'The tenant `{}\' has been created'.format(tenant) expected_output = 'The tenant `{}\' has been created'.format(tenant)
assert output == expected_output assert output == expected_output
@ -121,61 +113,74 @@ def configure_tenant(management_cluster_file, data_cluster_files, tenant, tenant
if assigned_cluster: if assigned_cluster:
command = command + ' assigned_cluster={}'.format(assigned_cluster) command = command + ' assigned_cluster={}'.format(assigned_cluster)
output, err = run_fdbcli_command_and_get_error(management_cluster_file, command) _, output, err = run_fdbcli_command(management_cluster_file, command)
return output, err return output, err
@enable_logging() def clear_database_and_tenants(management_cluster_file, data_cluster_files):
def clear_database_and_tenants(logger, management_cluster_file, data_cluster_files):
subcmd1 = 'writemode on' subcmd1 = 'writemode on'
subcmd2 = 'option on SPECIAL_KEY_SPACE_ENABLE_WRITES' subcmd2 = 'option on SPECIAL_KEY_SPACE_ENABLE_WRITES'
subcmd3 = 'clearrange ' '"" \\xff' subcmd3 = 'clearrange ' '"" \\xff'
subcmd4 = 'clearrange \\xff\\xff/management/tenant/map/ \\xff\\xff/management/tenant/map0' subcmd4 = 'clearrange \\xff\\xff/management/tenant/map/ \\xff\\xff/management/tenant/map0'
output = run_fdbcli_command(management_cluster_file, subcmd1, subcmd2, subcmd3, subcmd4) run_fdbcli_command(management_cluster_file, subcmd1, subcmd2, subcmd3, subcmd4)
logger.debug(output)
def run_tenant_test(management_cluster_file, data_cluster_files, test_func):
test_func(management_cluster_file, data_cluster_files)
clear_database_and_tenants(management_cluster_file, data_cluster_files)
@enable_logging() @enable_logging()
def clusters_status_test(logger, cluster_files): def clusters_status_test(logger, cluster_files, max_tenant_groups_per_cluster):
logger.debug('Verifying no cluster is part of a metacluster')
for cf in cluster_files: for cf in cluster_files:
output = metacluster_status(cf) output = metacluster_status(cf)
assert output == "This cluster is not part of a metacluster" assert output == "This cluster is not part of a metacluster"
logger.debug('Verified')
num_clusters = len(cluster_files) num_clusters = len(cluster_files)
names = ['meta_mgmt'] names = ['meta_mgmt']
names.extend(['data{}'.format(i) for i in range(1, num_clusters)]) names.extend(['data{}'.format(i) for i in range(1, num_clusters)])
setup_metacluster([cluster_files[0], names[0]], zip(cluster_files[1:], names[1:])) logger.debug('Setting up a metacluster')
setup_metacluster([cluster_files[0], names[0]], list(zip(cluster_files[1:], names[1:])),
max_tenant_groups_per_cluster=max_tenant_groups_per_cluster)
expected = """ expected = """
number of data clusters: {} number of data clusters: {}
tenant group capacity: {} tenant group capacity: {}
allocated tenant groups: 0 allocated tenant groups: 0
""" """
expected = expected.format(num_clusters - 1, 12).strip() expected = expected.format(num_clusters - 1, (num_clusters - 1) * max_tenant_groups_per_cluster).strip()
output = metacluster_status(cluster_files[0]) output = metacluster_status(cluster_files[0])
assert expected == output assert expected == output
logger.debug('Metacluster setup correctly')
for (cf, name) in zip(cluster_files[1:], names[1:]): for (cf, name) in zip(cluster_files[1:], names[1:]):
output = metacluster_status(cf) output = metacluster_status(cf)
expected = "This cluster \"{}\" is a data cluster within the metacluster named \"{" \ expected = "This cluster \"{}\" is a data cluster within the metacluster named \"{}\"".format(name, names[0])
"}\"".format(name, names[0])
assert expected == output assert expected == output
@enable_logging() @enable_logging()
def configure_tenants_test_disableClusterAssignment(logger, cluster_files): def configure_tenants_test_disableClusterAssignment(logger, cluster_files):
tenants = ['tenant1', 'tenant2'] tenants = ['tenant1', 'tenant2']
logger.debug('Creating tenants {}'.format(tenants)) logger.debug('Tenants to create: {}'.format(tenants))
setup_tenants(cluster_files[0], cluster_files[1:], tenants) setup_tenants(cluster_files[0], cluster_files[1:], tenants)
# Once we reach here, the tenants have been created successfully
logger.debug('Tenants created: {}'.format(tenants))
for tenant in tenants: for tenant in tenants:
out, err = configure_tenant(cluster_files[0], cluster_files[1:], tenant, assigned_cluster='cluster') out, err = configure_tenant(cluster_files[0], cluster_files[1:], tenant, assigned_cluster='cluster')
assert err == 'ERROR: Tenant configuration is invalid (2140)' assert err == 'ERROR: Tenant configuration is invalid (2140)'
logger.debug('Tenants configured')
clear_database_and_tenants(cluster_files[0], cluster_files[1:]) clear_database_and_tenants(cluster_files[0], cluster_files[1:])
logger.debug('Tenants cleared')
@enable_logging()
def test_main(logger):
logger.debug('Tests start')
# This must be the first test to run, since it sets up the metacluster that
# will be used throughout the test
clusters_status_test(cluster_files, max_tenant_groups_per_cluster=5)
configure_tenants_test_disableClusterAssignment(cluster_files)
logger.debug('Tests complete')
if __name__ == "__main__": if __name__ == "__main__":
@ -197,8 +202,4 @@ if __name__ == "__main__":
fdbcli_bin = args.build_dir + '/bin/fdbcli' fdbcli_bin = args.build_dir + '/bin/fdbcli'
# This must be the first test to run, since it sets up the metacluster that test_main()
# will be used throughout the test
clusters_status_test(cluster_files)
configure_tenants_test_disableClusterAssignment(cluster_files)

16
fdbclient/BackupAgentBase.actor.cpp
View File

@ -262,7 +262,7 @@ bool validTenantAccess(std::map<int64_t, TenantName>* tenantMap,
} }
int64_t tenantId = TenantInfo::INVALID_TENANT; int64_t tenantId = TenantInfo::INVALID_TENANT;
if (m.isEncrypted()) { if (m.isEncrypted()) {
tenantId = m.encryptionHeader()->cipherTextDetails.encryptDomainId; tenantId = m.encryptDomainId();
} else { } else {
tenantId = TenantAPI::extractTenantIdFromMutation(m); tenantId = TenantAPI::extractTenantIdFromMutation(m);
} }
@ -383,12 +383,18 @@ ACTOR static Future<Void> decodeBackupLogValue(Arena* arena,
// Decrypt mutation ref if encrypted // Decrypt mutation ref if encrypted
if (logValue.isEncrypted()) { if (logValue.isEncrypted()) {
encryptedLogValue = logValue; encryptedLogValue = logValue;
state EncryptCipherDomainId domainId = logValue.encryptionHeader()->cipherTextDetails.encryptDomainId; state EncryptCipherDomainId domainId = logValue.encryptDomainId();
Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo; Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
try { try {
TextAndHeaderCipherKeys cipherKeys = if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
wait(getEncryptCipherKeys(dbInfo, *logValue.encryptionHeader(), BlobCipherMetrics::RESTORE)); TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(
logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::BACKUP); dbInfo, logValue.configurableEncryptionHeader(), BlobCipherMetrics::RESTORE));
logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::RESTORE);
} else {
TextAndHeaderCipherKeys cipherKeys = wait(
getEncryptCipherKeys(dbInfo, *logValue.encryptionHeader(), BlobCipherMetrics::RESTORE));
logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::RESTORE);
}
} catch (Error& e) { } catch (Error& e) {
// It's possible a tenant was deleted and the encrypt key fetch failed // It's possible a tenant was deleted and the encrypt key fetch failed
TraceEvent(SevWarnAlways, "MutationLogRestoreEncryptKeyFetchFailed") TraceEvent(SevWarnAlways, "MutationLogRestoreEncryptKeyFetchFailed")

36
fdbclient/BlobCipher.cpp
View File

@ -40,7 +40,6 @@
#include "flow/Trace.h" #include "flow/Trace.h"
#include "flow/UnitTest.h" #include "flow/UnitTest.h"
#include "flow/xxhash.h" #include "flow/xxhash.h"
#include "include/fdbclient/BlobCipher.h"
#include <chrono> #include <chrono>
#include <cstring> #include <cstring>
@ -211,6 +210,10 @@ EncryptAuthTokenMode BlobCipherEncryptHeaderRef::getAuthTokenMode() const {
flags); flags);
} }
EncryptCipherDomainId BlobCipherEncryptHeaderRef::getDomainId() const {
return std::visit([](auto& h) { return h.v1.cipherTextDetails.encryptDomainId; }, algoHeader);
}
void BlobCipherEncryptHeaderRef::validateEncryptionHeaderDetails(const BlobCipherDetails& textCipherDetails, void BlobCipherEncryptHeaderRef::validateEncryptionHeaderDetails(const BlobCipherDetails& textCipherDetails,
const BlobCipherDetails& headerCipherDetails, const BlobCipherDetails& headerCipherDetails,
const StringRef& ivRef) const { const StringRef& ivRef) const {
@ -754,8 +757,7 @@ std::vector<Reference<BlobCipherKey>> BlobCipherKeyCache::getAllCiphers(const En
return keyIdCache->getAllCipherKeys(); return keyIdCache->getAllCipherKeys();
} }
namespace { int getEncryptCurrentAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo) {
int getEncryptAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo) {
if (mode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) { if (mode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
return CLIENT_KNOBS->ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION; return CLIENT_KNOBS->ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION;
} else { } else {
@ -768,7 +770,20 @@ int getEncryptAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAu
} }
} }
} }
} // namespace
void BlobCipherDetails::validateCipherDetailsWithCipherKey(Reference<BlobCipherKey> cipherKey) {
if (!(baseCipherId == cipherKey->getBaseCipherId() && encryptDomainId == cipherKey->getDomainId() &&
salt == cipherKey->getSalt())) {
TraceEvent(SevWarn, "EncryptionHeaderCipherMismatch")
.detail("TextDomainId", cipherKey->getDomainId())
.detail("ExpectedTextDomainId", encryptDomainId)
.detail("TextBaseCipherId", cipherKey->getBaseCipherId())
.detail("ExpectedTextBaseCipherId", baseCipherId)
.detail("TextSalt", cipherKey->getSalt())
.detail("ExpectedTextSalt", salt);
throw encrypt_header_metadata_mismatch();
}
}
// EncryptBlobCipherAes265Ctr class methods // EncryptBlobCipherAes265Ctr class methods
@ -896,8 +911,8 @@ void EncryptBlobCipherAes265Ctr::setCipherAlgoHeaderV1(const uint8_t* ciphertext
const BlobCipherEncryptHeaderFlagsV1& flags, const BlobCipherEncryptHeaderFlagsV1& flags,
BlobCipherEncryptHeaderRef* headerRef) { BlobCipherEncryptHeaderRef* headerRef) {
ASSERT_EQ(1, ASSERT_EQ(1,
getEncryptAlgoHeaderVersion((EncryptAuthTokenMode)flags.authTokenMode, getEncryptCurrentAlgoHeaderVersion((EncryptAuthTokenMode)flags.authTokenMode,
(EncryptAuthTokenAlgo)flags.authTokenAlgo)); (EncryptAuthTokenAlgo)flags.authTokenAlgo));
if (flags.authTokenMode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) { if (flags.authTokenMode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
setCipherAlgoHeaderNoAuthV1(flags, headerRef); setCipherAlgoHeaderNoAuthV1(flags, headerRef);
@ -930,7 +945,7 @@ void EncryptBlobCipherAes265Ctr::updateEncryptHeader(const uint8_t* ciphertext,
updateEncryptHeaderFlagsV1(headerRef, &flags); updateEncryptHeaderFlagsV1(headerRef, &flags);
// update cipher algo header // update cipher algo header
int algoHeaderVersion = getEncryptAlgoHeaderVersion(authTokenMode, authTokenAlgo); int algoHeaderVersion = getEncryptCurrentAlgoHeaderVersion(authTokenMode, authTokenAlgo);
ASSERT_EQ(algoHeaderVersion, 1); ASSERT_EQ(algoHeaderVersion, 1);
setCipherAlgoHeaderV1(ciphertext, ciphertextLen, flags, headerRef); setCipherAlgoHeaderV1(ciphertext, ciphertextLen, flags, headerRef);
} }
@ -1045,9 +1060,10 @@ Reference<EncryptBuf> EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plainte
if (authTokenMode != ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) { if (authTokenMode != ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
header->cipherHeaderDetails = headerCipherKeyOpt.get()->details(); header->cipherHeaderDetails = headerCipherKeyOpt.get()->details();
} else { } else {
header->cipherHeaderDetails.encryptDomainId = INVALID_ENCRYPT_DOMAIN_ID; header->cipherHeaderDetails = BlobCipherDetails();
header->cipherHeaderDetails.baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID; ASSERT_EQ(INVALID_ENCRYPT_DOMAIN_ID, header->cipherHeaderDetails.encryptDomainId);
header->cipherHeaderDetails.salt = INVALID_ENCRYPT_RANDOM_SALT; ASSERT_EQ(INVALID_ENCRYPT_CIPHER_KEY_ID, header->cipherHeaderDetails.baseCipherId);
ASSERT_EQ(INVALID_ENCRYPT_RANDOM_SALT, header->cipherHeaderDetails.salt);
} }
memcpy(&header->iv[0], &iv[0], AES_256_IV_LENGTH); memcpy(&header->iv[0], &iv[0], AES_256_IV_LENGTH);

28
fdbclient/BlobGranuleFiles.cpp
View File

@ -1531,6 +1531,7 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
GranuleMaterializeStats& stats) { GranuleMaterializeStats& stats) {
// TODO REMOVE with early replying // TODO REMOVE with early replying
ASSERT(readVersion == chunk.includedVersion); ASSERT(readVersion == chunk.includedVersion);
Version lastFileVersion = 0;
// Arena to hold all allocations for applying deltas. Most of it, and the arenas produced by reading the files, // Arena to hold all allocations for applying deltas. Most of it, and the arenas produced by reading the files,
// will likely be tossed if there are a significant number of mutations, so we copy at the end instead of doing a // will likely be tossed if there are a significant number of mutations, so we copy at the end instead of doing a
@ -1569,6 +1570,8 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
arena.dependsOn(streams.back().arena()); arena.dependsOn(streams.back().arena());
stats.snapshotRows += snapshotRows.size(); stats.snapshotRows += snapshotRows.size();
} }
ASSERT_WE_THINK(lastFileVersion < chunk.snapshotFile.get().fileVersion);
lastFileVersion = chunk.snapshotFile.get().fileVersion;
} else { } else {
ASSERT(!chunk.snapshotFile.present()); ASSERT(!chunk.snapshotFile.present());
} }
@ -1593,6 +1596,9 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
arena.dependsOn(streams.back().arena()); arena.dependsOn(streams.back().arena());
} }
arena.dependsOn(deltaRows.arena()); arena.dependsOn(deltaRows.arena());
ASSERT_WE_THINK(lastFileVersion < chunk.deltaFiles[deltaIdx].fileVersion);
lastFileVersion = chunk.deltaFiles[deltaIdx].fileVersion;
} }
if (BG_READ_DEBUG) { if (BG_READ_DEBUG) {
fmt::print("Applying {} memory deltas\n", chunk.newDeltas.size()); fmt::print("Applying {} memory deltas\n", chunk.newDeltas.size());
@ -1600,6 +1606,7 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
if (!chunk.newDeltas.empty()) { if (!chunk.newDeltas.empty()) {
stats.inputBytes += chunk.newDeltas.expectedSize(); stats.inputBytes += chunk.newDeltas.expectedSize();
// TODO REMOVE validation // TODO REMOVE validation
ASSERT_WE_THINK(lastFileVersion < chunk.newDeltas.front().version);
ASSERT(beginVersion <= chunk.newDeltas.front().version); ASSERT(beginVersion <= chunk.newDeltas.front().version);
ASSERT(readVersion >= chunk.newDeltas.back().version); ASSERT(readVersion >= chunk.newDeltas.back().version);
auto memoryRows = sortMemoryDeltas(chunk.newDeltas, chunk.keyRange, requestRange, beginVersion, readVersion); auto memoryRows = sortMemoryDeltas(chunk.newDeltas, chunk.keyRange, requestRange, beginVersion, readVersion);
@ -2188,7 +2195,8 @@ struct KeyValueGen {
int targetMutationsPerDelta; int targetMutationsPerDelta;
KeyRange allRange; KeyRange allRange;
Version version = 0; // start at higher version to allow snapshot files to have version 1
Version version = 10;
// encryption/compression settings // encryption/compression settings
// TODO: possibly different cipher keys or meta context per file? // TODO: possibly different cipher keys or meta context per file?
@ -2551,7 +2559,7 @@ void checkDeltaRead(const KeyValueGen& kvGen,
deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), readVersion, ".delta"); deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), readVersion, ".delta");
Standalone<BlobGranuleChunkRef> chunk; Standalone<BlobGranuleChunkRef> chunk;
chunk.deltaFiles.emplace_back_deep( chunk.deltaFiles.emplace_back_deep(
chunk.arena(), filename, 0, serialized[0].size(), serialized[0].size(), kvGen.cipherKeys); chunk.arena(), filename, 0, serialized[0].size(), serialized[0].size(), 1, kvGen.cipherKeys);
chunk.keyRange = kvGen.allRange; chunk.keyRange = kvGen.allRange;
chunk.includedVersion = readVersion; chunk.includedVersion = readVersion;
chunk.snapshotVersion = invalidVersion; chunk.snapshotVersion = invalidVersion;
@ -2672,8 +2680,13 @@ void checkGranuleRead(const KeyValueGen& kvGen,
if (beginVersion == 0) { if (beginVersion == 0) {
std::string snapshotFilename = randomBGFilename( std::string snapshotFilename = randomBGFilename(
deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), 0, ".snapshot"); deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), 0, ".snapshot");
chunk.snapshotFile = BlobFilePointerRef( chunk.snapshotFile = BlobFilePointerRef(chunk.arena(),
chunk.arena(), snapshotFilename, 0, serializedSnapshot.size(), serializedSnapshot.size(), kvGen.cipherKeys); snapshotFilename,
0,
serializedSnapshot.size(),
serializedSnapshot.size(),
1,
kvGen.cipherKeys);
} }
int deltaIdx = 0; int deltaIdx = 0;
while (deltaIdx < serializedDeltas.size() && serializedDeltas[deltaIdx].first < beginVersion) { while (deltaIdx < serializedDeltas.size() && serializedDeltas[deltaIdx].first < beginVersion) {
@ -2684,7 +2697,8 @@ void checkGranuleRead(const KeyValueGen& kvGen,
std::string deltaFilename = randomBGFilename( std::string deltaFilename = randomBGFilename(
deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), readVersion, ".delta"); deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), readVersion, ".delta");
size_t fsize = serializedDeltas[deltaIdx].second.size(); size_t fsize = serializedDeltas[deltaIdx].second.size();
chunk.deltaFiles.emplace_back_deep(chunk.arena(), deltaFilename, 0, fsize, fsize, kvGen.cipherKeys); chunk.deltaFiles.emplace_back_deep(
chunk.arena(), deltaFilename, 0, fsize, fsize, serializedDeltas[deltaIdx].first, kvGen.cipherKeys);
deltaPtrsVector.push_back(serializedDeltas[deltaIdx].second); deltaPtrsVector.push_back(serializedDeltas[deltaIdx].second);
if (serializedDeltas[deltaIdx].first >= readVersion) { if (serializedDeltas[deltaIdx].first >= readVersion) {
@ -3207,12 +3221,12 @@ void chunkFromFileSet(const FileSet& fileSet,
int numDeltaFiles) { int numDeltaFiles) {
size_t snapshotSize = std::get<3>(fileSet.snapshotFile).size(); size_t snapshotSize = std::get<3>(fileSet.snapshotFile).size();
chunk.snapshotFile = chunk.snapshotFile =
BlobFilePointerRef(chunk.arena(), std::get<0>(fileSet.snapshotFile), 0, snapshotSize, snapshotSize, keys); BlobFilePointerRef(chunk.arena(), std::get<0>(fileSet.snapshotFile), 0, snapshotSize, snapshotSize, 1, keys);
for (int i = 0; i < numDeltaFiles; i++) { for (int i = 0; i < numDeltaFiles; i++) {
size_t deltaSize = std::get<3>(fileSet.deltaFiles[i]).size(); size_t deltaSize = std::get<3>(fileSet.deltaFiles[i]).size();
chunk.deltaFiles.emplace_back_deep( chunk.deltaFiles.emplace_back_deep(
chunk.arena(), std::get<0>(fileSet.deltaFiles[i]), 0, deltaSize, deltaSize, keys); chunk.arena(), std::get<0>(fileSet.deltaFiles[i]), 0, deltaSize, deltaSize, 2 + i, keys);
deltaPtrs[i] = std::get<2>(fileSet.deltaFiles[i]); deltaPtrs[i] = std::get<2>(fileSet.deltaFiles[i]);
} }

4
fdbclient/ClientKnobs.cpp
View File

@ -24,6 +24,7 @@
#include "fdbclient/Tenant.h" #include "fdbclient/Tenant.h"
#include "flow/IRandom.h" #include "flow/IRandom.h"
#include "flow/UnitTest.h" #include "flow/UnitTest.h"
#include "flow/flow.h"
#define init(...) KNOB_FN(__VA_ARGS__, INIT_ATOMIC_KNOB, INIT_KNOB)(__VA_ARGS__) #define init(...) KNOB_FN(__VA_ARGS__, INIT_ATOMIC_KNOB, INIT_KNOB)(__VA_ARGS__)
@ -302,8 +303,9 @@ void ClientKnobs::initialize(Randomize randomize) {
init( CLIENT_ENABLE_USING_CLUSTER_ID_KEY, false ); init( CLIENT_ENABLE_USING_CLUSTER_ID_KEY, false );
init( ENABLE_ENCRYPTION_CPU_TIME_LOGGING, false ); init( ENABLE_ENCRYPTION_CPU_TIME_LOGGING, false );
init( SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS, true );
init( SIMULATION_EKP_TENANT_IDS_TO_DROP, "-1" ); init( SIMULATION_EKP_TENANT_IDS_TO_DROP, "-1" );
init( ENABLE_CONFIGURABLE_ENCRYPTION, false ); init( ENABLE_CONFIGURABLE_ENCRYPTION, true );
init( ENCRYPT_HEADER_FLAGS_VERSION, 1 ); init( ENCRYPT_HEADER_FLAGS_VERSION, 1 );
init( ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION, 1 ); init( ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION, 1 );
init( ENCRYPT_HEADER_AES_CTR_AES_CMAC_AUTH_VERSION, 1 ); init( ENCRYPT_HEADER_AES_CTR_AES_CMAC_AUTH_VERSION, 1 );

177
fdbclient/FileBackupAgent.actor.cpp
View File

@ -61,6 +61,7 @@
#include <algorithm> #include <algorithm>
#include <unordered_map> #include <unordered_map>
#include <utility> #include <utility>
#include <variant>
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
@ -551,14 +552,13 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
struct Options { struct Options {
constexpr static FileIdentifier file_identifier = 3152016; constexpr static FileIdentifier file_identifier = 3152016;
// TODO: Compression is not currently supported so this should always be false bool configurableEncryptionEnabled = false;
bool compressionEnabled = false;
Options() {} Options() {}
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, compressionEnabled); serializer(ar, configurableEncryptionEnabled);
} }
}; };
@ -575,54 +575,44 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
wPtr = mutateString(buffer); wPtr = mutateString(buffer);
} }
static void validateEncryptionHeader(Optional<Reference<BlobCipherKey>> headerCipherKey, ACTOR static Future<StringRef> decryptImpl(
Reference<BlobCipherKey> textCipherKey, Database cx,
BlobCipherEncryptHeader& header) { std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> headerVariant,
// Validate encryption header 'cipherHeader' details const uint8_t* dataP,
if (header.cipherHeaderDetails.isValid() && int64_t dataLen,
(!headerCipherKey.present() || header.cipherHeaderDetails != headerCipherKey.get()->details())) { Arena* arena) {
TraceEvent(SevWarn, "EncryptionHeader_CipherHeaderMismatch")
.detail("HeaderDomainId", headerCipherKey.get()->getDomainId())
.detail("ExpectedHeaderDomainId", header.cipherHeaderDetails.encryptDomainId)
.detail("HeaderBaseCipherId", headerCipherKey.get()->getBaseCipherId())
.detail("ExpectedHeaderBaseCipherId", header.cipherHeaderDetails.baseCipherId)
.detail("HeaderSalt", headerCipherKey.get()->getSalt())
.detail("ExpectedHeaderSalt", header.cipherHeaderDetails.salt);
throw encrypt_header_metadata_mismatch();
}
// Validate encryption text 'cipherText' details sanity
if (!header.cipherTextDetails.isValid() || header.cipherTextDetails != textCipherKey->details()) {
TraceEvent(SevWarn, "EncryptionHeader_CipherTextMismatch")
.detail("TextDomainId", textCipherKey->getDomainId())
.detail("ExpectedTextDomainId", header.cipherTextDetails.encryptDomainId)
.detail("TextBaseCipherId", textCipherKey->getBaseCipherId())
.detail("ExpectedTextBaseCipherId", header.cipherTextDetails.baseCipherId)
.detail("TextSalt", textCipherKey->getSalt())
.detail("ExpectedTextSalt", header.cipherTextDetails.salt);
throw encrypt_header_metadata_mismatch();
}
}
ACTOR static Future<StringRef> decryptImpl(Database cx,
BlobCipherEncryptHeader header,
const uint8_t* dataP,
int64_t dataLen,
Arena* arena) {
Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo; Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, header, BlobCipherMetrics::RESTORE)); if (std::holds_alternative<BlobCipherEncryptHeaderRef>(headerVariant)) { // configurable encryption
validateEncryptionHeader(cipherKeys.cipherHeaderKey, cipherKeys.cipherTextKey, header); state BlobCipherEncryptHeaderRef headerRef = std::get<BlobCipherEncryptHeaderRef>(headerVariant);
DecryptBlobCipherAes256Ctr decryptor( TextAndHeaderCipherKeys cipherKeys =
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.iv, BlobCipherMetrics::BACKUP); wait(getEncryptCipherKeys(dbInfo, headerRef, BlobCipherMetrics::RESTORE));
return decryptor.decrypt(dataP, dataLen, header, *arena)->toStringRef(); EncryptHeaderCipherDetails cipherDetails = headerRef.getCipherDetails();
cipherDetails.textCipherDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherTextKey);
if (cipherDetails.headerCipherDetails.present()) {
cipherDetails.headerCipherDetails.get().validateCipherDetailsWithCipherKey(cipherKeys.cipherHeaderKey);
}
DecryptBlobCipherAes256Ctr decryptor(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, headerRef.getIV(), BlobCipherMetrics::RESTORE);
return decryptor.decrypt(dataP, dataLen, headerRef, *arena);
} else {
state BlobCipherEncryptHeader header = std::get<BlobCipherEncryptHeader>(headerVariant);
TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, header, BlobCipherMetrics::RESTORE));
header.cipherTextDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherTextKey);
if (header.cipherHeaderDetails.isValid()) {
header.cipherHeaderDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherHeaderKey);
}
DecryptBlobCipherAes256Ctr decryptor(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.iv, BlobCipherMetrics::RESTORE);
return decryptor.decrypt(dataP, dataLen, header, *arena)->toStringRef();
}
} }
static Future<StringRef> decrypt(Database cx, static Future<StringRef> decrypt(Database cx,
BlobCipherEncryptHeader headerS, std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> header,
const uint8_t* dataP, const uint8_t* dataP,
int64_t dataLen, int64_t dataLen,
Arena* arena) { Arena* arena) {
return decryptImpl(cx, headerS, dataP, dataLen, arena); return decryptImpl(cx, header, dataP, dataLen, arena);
} }
ACTOR static Future<Reference<BlobCipherKey>> refreshKey(EncryptedRangeFileWriter* self, ACTOR static Future<Reference<BlobCipherKey>> refreshKey(EncryptedRangeFileWriter* self,
@ -655,12 +645,26 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
AES_256_IV_LENGTH, AES_256_IV_LENGTH,
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE), getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
BlobCipherMetrics::BACKUP); BlobCipherMetrics::BACKUP);
Arena arena;
int64_t payloadSize = self->wPtr - self->dataPayloadStart; int64_t payloadSize = self->wPtr - self->dataPayloadStart;
auto encryptedData = encryptor.encrypt(self->dataPayloadStart, payloadSize, self->encryptHeader, arena); StringRef encryptedData;
if (self->options.configurableEncryptionEnabled) {
BlobCipherEncryptHeaderRef headerRef;
encryptedData = encryptor.encrypt(self->dataPayloadStart, payloadSize, &headerRef, *self->arena);
Standalone<StringRef> serialized = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
self->arena->dependsOn(serialized.arena());
ASSERT(serialized.size() == self->encryptHeader.size());
std::memcpy(mutateString(self->encryptHeader), serialized.begin(), self->encryptHeader.size());
} else {
BlobCipherEncryptHeader header;
encryptedData =
encryptor.encrypt(self->dataPayloadStart, payloadSize, &header, *self->arena)->toStringRef();
StringRef encryptHeaderStringRef = BlobCipherEncryptHeader::toStringRef(header, *self->arena);
ASSERT(encryptHeaderStringRef.size() == self->encryptHeader.size());
std::memcpy(mutateString(self->encryptHeader), encryptHeaderStringRef.begin(), self->encryptHeader.size());
}
// re-write encrypted data to buffer // re-write encrypted data to buffer
std::memcpy(self->dataPayloadStart, encryptedData->begin(), payloadSize); std::memcpy(self->dataPayloadStart, encryptedData.begin(), payloadSize);
return Void(); return Void();
} }
@ -767,13 +771,32 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
copyToBuffer(self, (uint8_t*)&self->fileVersion, sizeof(self->fileVersion)); copyToBuffer(self, (uint8_t*)&self->fileVersion, sizeof(self->fileVersion));
// write options struct // write options struct
self->options.configurableEncryptionEnabled = CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION;
Value serialized = Value serialized =
ObjectWriter::toValue(self->options, IncludeVersion(ProtocolVersion::withEncryptedSnapshotBackupFile())); ObjectWriter::toValue(self->options, IncludeVersion(ProtocolVersion::withEncryptedSnapshotBackupFile()));
appendStringRefWithLenToBuffer(self, &serialized); appendStringRefWithLenToBuffer(self, &serialized);
// calculate encryption header size
uint32_t headerSize = 0;
if (self->options.configurableEncryptionEnabled) {
EncryptAuthTokenMode authTokenMode =
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
EncryptAuthTokenAlgo authTokenAlgo = getAuthTokenAlgoFromMode(authTokenMode);
headerSize = BlobCipherEncryptHeaderRef::getHeaderSize(
CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION,
getEncryptCurrentAlgoHeaderVersion(authTokenMode, authTokenAlgo),
ENCRYPT_CIPHER_MODE_AES_256_CTR,
authTokenMode,
authTokenAlgo);
} else {
headerSize = BlobCipherEncryptHeader::headerSize;
}
ASSERT(headerSize > 0);
// write header size to buffer
copyToBuffer(self, (uint8_t*)&headerSize, sizeof(headerSize));
// leave space for encryption header // leave space for encryption header
self->encryptHeader = (BlobCipherEncryptHeader*)self->wPtr; self->encryptHeader = StringRef(self->wPtr, headerSize);
self->wPtr += BlobCipherEncryptHeader::headerSize; self->wPtr += headerSize;
self->dataPayloadStart = self->wPtr; self->dataPayloadStart = self->wPtr;
// If this is NOT the first block then write duplicate stuff needed from last block // If this is NOT the first block then write duplicate stuff needed from last block
@ -913,7 +936,7 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
private: private:
Standalone<StringRef> buffer; Standalone<StringRef> buffer;
uint8_t* wPtr; uint8_t* wPtr;
BlobCipherEncryptHeader* encryptHeader; StringRef encryptHeader;
uint8_t* dataPayloadStart; uint8_t* dataPayloadStart;
int64_t blockEnd; int64_t blockEnd;
uint32_t fileVersion; uint32_t fileVersion;
@ -1050,7 +1073,7 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
Standalone<VectorRef<KeyValueRef>>* results, Standalone<VectorRef<KeyValueRef>>* results,
bool encryptedBlock, bool encryptedBlock,
EncryptionAtRestMode encryptMode, EncryptionAtRestMode encryptMode,
Optional<BlobCipherEncryptHeader> encryptHeader, Optional<int64_t> blockDomainId,
Optional<Reference<TenantEntryCache<Void>>> tenantCache) { Optional<Reference<TenantEntryCache<Void>>> tenantCache) {
// Read begin key, if this fails then block was invalid. // Read begin key, if this fails then block was invalid.
state uint32_t kLen = reader->consumeNetworkUInt32(); state uint32_t kLen = reader->consumeNetworkUInt32();
@ -1068,9 +1091,9 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
// make sure that all keys in a block belong to exactly one tenant, // make sure that all keys in a block belong to exactly one tenant,
// unless its the last key in which case it can be a truncated (different) tenant prefix // unless its the last key in which case it can be a truncated (different) tenant prefix
if (encryptedBlock && g_network && g_network->isSimulated() && ASSERT(!encryptedBlock || blockDomainId.present());
!isReservedEncryptDomain(encryptHeader.get().cipherTextDetails.encryptDomainId)) { if (CLIENT_KNOBS->SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS && encryptedBlock && g_network &&
ASSERT(encryptHeader.present()); g_network->isSimulated() && !isReservedEncryptDomain(blockDomainId.get())) {
state KeyRef curKey = KeyRef(k, kLen); state KeyRef curKey = KeyRef(k, kLen);
if (!prevDomainId.present()) { if (!prevDomainId.present()) {
EncryptCipherDomainId domainId = EncryptCipherDomainId domainId =
@ -1088,7 +1111,7 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
} }
// make sure that all keys (except possibly the last key) in a block are encrypted using the correct key // make sure that all keys (except possibly the last key) in a block are encrypted using the correct key
if (!prevKey.empty()) { if (!prevKey.empty()) {
ASSERT(prevDomainId.get() == encryptHeader.get().cipherTextDetails.encryptDomainId); ASSERT(prevDomainId.get() == blockDomainId.get());
} }
prevKey = curKey; prevKey = curKey;
prevDomainId = curDomainId; prevDomainId = curDomainId;
@ -1152,15 +1175,14 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
wait(tenantCache.get()->init()); wait(tenantCache.get()->init());
} }
state EncryptionAtRestMode encryptMode = config.encryptionAtRestMode; state EncryptionAtRestMode encryptMode = config.encryptionAtRestMode;
state int64_t blockTenantId = TenantInfo::INVALID_TENANT; state int64_t blockDomainId = TenantInfo::INVALID_TENANT;
try { try {
// Read header, currently only decoding BACKUP_AGENT_SNAPSHOT_FILE_VERSION or // Read header, currently only decoding BACKUP_AGENT_SNAPSHOT_FILE_VERSION or
// BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION // BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION
int32_t file_version = reader.consume<int32_t>(); int32_t file_version = reader.consume<int32_t>();
if (file_version == BACKUP_AGENT_SNAPSHOT_FILE_VERSION) { if (file_version == BACKUP_AGENT_SNAPSHOT_FILE_VERSION) {
wait( wait(decodeKVPairs(&reader, &results, false, encryptMode, Optional<int64_t>(), tenantCache));
decodeKVPairs(&reader, &results, false, encryptMode, Optional<BlobCipherEncryptHeader>(), tenantCache));
} else if (file_version == BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION) { } else if (file_version == BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION) {
CODE_PROBE(true, "decoding encrypted block"); CODE_PROBE(true, "decoding encrypted block");
// decode options struct // decode options struct
@ -1169,39 +1191,50 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
StringRef optionsStringRef = StringRef(o, optionsLen); StringRef optionsStringRef = StringRef(o, optionsLen);
EncryptedRangeFileWriter::Options options = EncryptedRangeFileWriter::Options options =
ObjectReader::fromStringRef<EncryptedRangeFileWriter::Options>(optionsStringRef, IncludeVersion()); ObjectReader::fromStringRef<EncryptedRangeFileWriter::Options>(optionsStringRef, IncludeVersion());
ASSERT(!options.compressionEnabled); // read header size
state uint32_t headerLen = reader.consume<uint32_t>();
// read the encryption header
state const uint8_t* headerStart = reader.consume(headerLen);
StringRef headerS = StringRef(headerStart, headerLen);
state std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> encryptHeader;
if (options.configurableEncryptionEnabled) {
encryptHeader = BlobCipherEncryptHeaderRef::fromStringRef(headerS);
blockDomainId = std::get<BlobCipherEncryptHeaderRef>(encryptHeader)
.getCipherDetails()
.textCipherDetails.encryptDomainId;
} else {
encryptHeader = BlobCipherEncryptHeader::fromStringRef(headerS);
blockDomainId = std::get<BlobCipherEncryptHeader>(encryptHeader).cipherTextDetails.encryptDomainId;
}
// read encryption header if (config.tenantMode == TenantMode::REQUIRED && !isReservedEncryptDomain(blockDomainId)) {
state const uint8_t* headerStart = reader.consume(BlobCipherEncryptHeader::headerSize);
StringRef headerS = StringRef(headerStart, BlobCipherEncryptHeader::headerSize);
state BlobCipherEncryptHeader header = BlobCipherEncryptHeader::fromStringRef(headerS);
blockTenantId = header.cipherTextDetails.encryptDomainId;
if (config.tenantMode == TenantMode::REQUIRED && !isReservedEncryptDomain(blockTenantId)) {
ASSERT(tenantCache.present()); ASSERT(tenantCache.present());
Optional<TenantEntryCachePayload<Void>> payload = wait(tenantCache.get()->getById(blockTenantId)); Optional<TenantEntryCachePayload<Void>> payload = wait(tenantCache.get()->getById(blockDomainId));
if (!payload.present()) { if (!payload.present()) {
throw tenant_not_found(); throw tenant_not_found();
} }
} }
const uint8_t* dataPayloadStart = headerStart + BlobCipherEncryptHeader::headerSize; const uint8_t* dataPayloadStart = headerStart + headerLen;
// calculate the total bytes read up to (and including) the header // calculate the total bytes read up to (and including) the header
int64_t bytesRead = sizeof(int32_t) + sizeof(uint32_t) + optionsLen + BlobCipherEncryptHeader::headerSize; int64_t bytesRead = sizeof(int32_t) + sizeof(uint32_t) + sizeof(uint32_t) + optionsLen + headerLen;
// get the size of the encrypted payload and decrypt it // get the size of the encrypted payload and decrypt it
int64_t dataLen = len - bytesRead; int64_t dataLen = len - bytesRead;
StringRef decryptedData = StringRef decryptedData =
wait(EncryptedRangeFileWriter::decrypt(cx, header, dataPayloadStart, dataLen, &results.arena())); wait(EncryptedRangeFileWriter::decrypt(cx, encryptHeader, dataPayloadStart, dataLen, &results.arena()));
reader = StringRefReader(decryptedData, restore_corrupted_data()); reader = StringRefReader(decryptedData, restore_corrupted_data());
wait(decodeKVPairs(&reader, &results, true, encryptMode, header, tenantCache)); wait(decodeKVPairs(&reader, &results, true, encryptMode, blockDomainId, tenantCache));
} else { } else {
throw restore_unsupported_file_version(); throw restore_unsupported_file_version();
} }
return results; return results;
} catch (Error& e) { } catch (Error& e) {
if (e.code() == error_code_encrypt_keys_fetch_failed) { if (e.code() == error_code_encrypt_keys_fetch_failed) {
TraceEvent(SevWarnAlways, "SnapshotRestoreEncryptKeyFetchFailed").detail("TenantId", blockTenantId); ASSERT(!isReservedEncryptDomain(blockDomainId));
TraceEvent(SevWarnAlways, "SnapshotRestoreEncryptKeyFetchFailed").detail("TenantId", blockDomainId);
CODE_PROBE(true, "Snapshot restore encrypt keys not found"); CODE_PROBE(true, "Snapshot restore encrypt keys not found");
} else if (e.code() == error_code_tenant_not_found) { } else if (e.code() == error_code_tenant_not_found) {
TraceEvent(SevWarnAlways, "EncryptedSnapshotRestoreTenantNotFound").detail("TenantId", blockTenantId); ASSERT(!isReservedEncryptDomain(blockDomainId));
TraceEvent(SevWarnAlways, "EncryptedSnapshotRestoreTenantNotFound").detail("TenantId", blockDomainId);
CODE_PROBE(true, "Encrypted Snapshot restore tenant not found"); CODE_PROBE(true, "Encrypted Snapshot restore tenant not found");
} }
TraceEvent(SevWarn, "FileRestoreDecodeRangeFileBlockFailed") TraceEvent(SevWarn, "FileRestoreDecodeRangeFileBlockFailed")

116
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -549,6 +549,21 @@ ThreadFuture<bool> DLTenant::blobbifyRange(const KeyRangeRef& keyRange) {
}); });
} }
ThreadFuture<bool> DLTenant::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
if (!api->tenantBlobbifyRangeBlocking) {
return unsupported_operation();
}
FdbCApi::FDBFuture* f = api->tenantBlobbifyRangeBlocking(
tenant, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size());
return toThreadFuture<bool>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
FdbCApi::fdb_bool_t ret = false;
ASSERT(!api->futureGetBool(f, &ret));
return ret;
});
}
ThreadFuture<bool> DLTenant::unblobbifyRange(const KeyRangeRef& keyRange) { ThreadFuture<bool> DLTenant::unblobbifyRange(const KeyRangeRef& keyRange) {
if (!api->tenantUnblobbifyRange) { if (!api->tenantUnblobbifyRange) {
return unsupported_operation(); return unsupported_operation();
@ -601,6 +616,28 @@ ThreadFuture<Version> DLTenant::verifyBlobRange(const KeyRangeRef& keyRange, Opt
}); });
} }
ThreadFuture<bool> DLTenant::flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) {
if (!api->tenantFlushBlobRange) {
return unsupported_operation();
}
Version readVersion = version.present() ? version.get() : latestVersion;
FdbCApi::FDBFuture* f = api->tenantFlushBlobRange(tenant,
keyRange.begin.begin(),
keyRange.begin.size(),
keyRange.end.begin(),
keyRange.end.size(),
compact,
readVersion);
return toThreadFuture<bool>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
FdbCApi::fdb_bool_t ret = false;
ASSERT(!api->futureGetBool(f, &ret));
return ret;
});
}
// DLDatabase // DLDatabase
DLDatabase::DLDatabase(Reference<FdbCApi> api, ThreadFuture<FdbCApi::FDBDatabase*> dbFuture) : api(api), db(nullptr) { DLDatabase::DLDatabase(Reference<FdbCApi> api, ThreadFuture<FdbCApi::FDBDatabase*> dbFuture) : api(api), db(nullptr) {
addref(); addref();
@ -768,6 +805,21 @@ ThreadFuture<bool> DLDatabase::blobbifyRange(const KeyRangeRef& keyRange) {
}); });
} }
ThreadFuture<bool> DLDatabase::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
if (!api->databaseBlobbifyRangeBlocking) {
return unsupported_operation();
}
FdbCApi::FDBFuture* f = api->databaseBlobbifyRangeBlocking(
db, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size());
return toThreadFuture<bool>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
FdbCApi::fdb_bool_t ret = false;
ASSERT(!api->futureGetBool(f, &ret));
return ret;
});
}
ThreadFuture<bool> DLDatabase::unblobbifyRange(const KeyRangeRef& keyRange) { ThreadFuture<bool> DLDatabase::unblobbifyRange(const KeyRangeRef& keyRange) {
if (!api->databaseUnblobbifyRange) { if (!api->databaseUnblobbifyRange) {
return unsupported_operation(); return unsupported_operation();
@ -820,6 +872,28 @@ ThreadFuture<Version> DLDatabase::verifyBlobRange(const KeyRangeRef& keyRange, O
}); });
} }
ThreadFuture<bool> DLDatabase::flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) {
if (!api->databaseFlushBlobRange) {
return unsupported_operation();
}
Version readVersion = version.present() ? version.get() : latestVersion;
FdbCApi::FDBFuture* f = api->databaseFlushBlobRange(db,
keyRange.begin.begin(),
keyRange.begin.size(),
keyRange.end.begin(),
keyRange.end.size(),
compact,
readVersion);
return toThreadFuture<bool>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
FdbCApi::fdb_bool_t ret = false;
ASSERT(!api->futureGetBool(f, &ret));
return ret;
});
}
ThreadFuture<Standalone<StringRef>> DLDatabase::getClientStatus() { ThreadFuture<Standalone<StringRef>> DLDatabase::getClientStatus() {
if (!api->databaseGetClientStatus) { if (!api->databaseGetClientStatus) {
return unsupported_operation(); return unsupported_operation();
@ -931,6 +1005,11 @@ void DLApi::init() {
fdbCPath, fdbCPath,
"fdb_database_blobbify_range", "fdb_database_blobbify_range",
headerVersion >= ApiVersion::withBlobRangeApi().version()); headerVersion >= ApiVersion::withBlobRangeApi().version());
loadClientFunction(&api->databaseBlobbifyRangeBlocking,
lib,
fdbCPath,
"fdb_database_blobbify_range_blocking",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->databaseUnblobbifyRange, loadClientFunction(&api->databaseUnblobbifyRange,
lib, lib,
fdbCPath, fdbCPath,
@ -946,6 +1025,11 @@ void DLApi::init() {
fdbCPath, fdbCPath,
"fdb_database_verify_blob_range", "fdb_database_verify_blob_range",
headerVersion >= ApiVersion::withBlobRangeApi().version()); headerVersion >= ApiVersion::withBlobRangeApi().version());
loadClientFunction(&api->databaseFlushBlobRange,
lib,
fdbCPath,
"fdb_database_flush_blob_range",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->databaseGetClientStatus, loadClientFunction(&api->databaseGetClientStatus,
lib, lib,
fdbCPath, fdbCPath,
@ -968,6 +1052,11 @@ void DLApi::init() {
fdbCPath, fdbCPath,
"fdb_tenant_blobbify_range", "fdb_tenant_blobbify_range",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version()); headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->tenantBlobbifyRangeBlocking,
lib,
fdbCPath,
"fdb_tenant_blobbify_range_blocking",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->tenantUnblobbifyRange, loadClientFunction(&api->tenantUnblobbifyRange,
lib, lib,
fdbCPath, fdbCPath,
@ -983,6 +1072,11 @@ void DLApi::init() {
fdbCPath, fdbCPath,
"fdb_tenant_verify_blob_range", "fdb_tenant_verify_blob_range",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version()); headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->tenantFlushBlobRange,
lib,
fdbCPath,
"fdb_tenant_flush_blob_range",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->tenantGetId, lib, fdbCPath, "fdb_tenant_get_id", headerVersion >= 730); loadClientFunction(&api->tenantGetId, lib, fdbCPath, "fdb_tenant_get_id", headerVersion >= 730);
loadClientFunction(&api->tenantDestroy, lib, fdbCPath, "fdb_tenant_destroy", headerVersion >= 710); loadClientFunction(&api->tenantDestroy, lib, fdbCPath, "fdb_tenant_destroy", headerVersion >= 710);
@ -1837,6 +1931,10 @@ ThreadFuture<bool> MultiVersionTenant::blobbifyRange(const KeyRangeRef& keyRange
return executeOperation(&ITenant::blobbifyRange, keyRange); return executeOperation(&ITenant::blobbifyRange, keyRange);
} }
ThreadFuture<bool> MultiVersionTenant::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
return executeOperation(&ITenant::blobbifyRangeBlocking, keyRange);
}
ThreadFuture<bool> MultiVersionTenant::unblobbifyRange(const KeyRangeRef& keyRange) { ThreadFuture<bool> MultiVersionTenant::unblobbifyRange(const KeyRangeRef& keyRange) {
return executeOperation(&ITenant::unblobbifyRange, keyRange); return executeOperation(&ITenant::unblobbifyRange, keyRange);
} }
@ -1850,6 +1948,13 @@ ThreadFuture<Version> MultiVersionTenant::verifyBlobRange(const KeyRangeRef& key
return executeOperation(&ITenant::verifyBlobRange, keyRange, std::forward<Optional<Version>>(version)); return executeOperation(&ITenant::verifyBlobRange, keyRange, std::forward<Optional<Version>>(version));
} }
ThreadFuture<bool> MultiVersionTenant::flushBlobRange(const KeyRangeRef& keyRange,
bool compact,
Optional<Version> version) {
return executeOperation(
&ITenant::flushBlobRange, keyRange, std::forward<bool>(compact), std::forward<Optional<Version>>(version));
}
MultiVersionTenant::TenantState::TenantState(Reference<MultiVersionDatabase> db, TenantNameRef tenantName) MultiVersionTenant::TenantState::TenantState(Reference<MultiVersionDatabase> db, TenantNameRef tenantName)
: tenantVar(new ThreadSafeAsyncVar<Reference<ITenant>>(Reference<ITenant>(nullptr))), tenantName(tenantName), db(db), : tenantVar(new ThreadSafeAsyncVar<Reference<ITenant>>(Reference<ITenant>(nullptr))), tenantName(tenantName), db(db),
closed(false) { closed(false) {
@ -2071,6 +2176,10 @@ ThreadFuture<bool> MultiVersionDatabase::blobbifyRange(const KeyRangeRef& keyRan
return executeOperation(&IDatabase::blobbifyRange, keyRange); return executeOperation(&IDatabase::blobbifyRange, keyRange);
} }
ThreadFuture<bool> MultiVersionDatabase::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
return executeOperation(&IDatabase::blobbifyRangeBlocking, keyRange);
}
ThreadFuture<bool> MultiVersionDatabase::unblobbifyRange(const KeyRangeRef& keyRange) { ThreadFuture<bool> MultiVersionDatabase::unblobbifyRange(const KeyRangeRef& keyRange) {
return executeOperation(&IDatabase::unblobbifyRange, keyRange); return executeOperation(&IDatabase::unblobbifyRange, keyRange);
} }
@ -2084,6 +2193,13 @@ ThreadFuture<Version> MultiVersionDatabase::verifyBlobRange(const KeyRangeRef& k
return executeOperation(&IDatabase::verifyBlobRange, keyRange, std::forward<Optional<Version>>(version)); return executeOperation(&IDatabase::verifyBlobRange, keyRange, std::forward<Optional<Version>>(version));
} }
ThreadFuture<bool> MultiVersionDatabase::flushBlobRange(const KeyRangeRef& keyRange,
bool compact,
Optional<Version> version) {
return executeOperation(
&IDatabase::flushBlobRange, keyRange, std::forward<bool>(compact), std::forward<Optional<Version>>(version));
}
// Returns the protocol version reported by the coordinator this client is connected to // Returns the protocol version reported by the coordinator this client is connected to
// If an expected version is given, the future won't return until the protocol version is different than expected // If an expected version is given, the future won't return until the protocol version is different than expected
// Note: this will never return if the server is running a protocol from FDB 5.0 or older // Note: this will never return if the server is running a protocol from FDB 5.0 or older

69
fdbclient/NativeAPI.actor.cpp
View File

@ -47,6 +47,7 @@
#include "fdbclient/AnnotateActor.h" #include "fdbclient/AnnotateActor.h"
#include "fdbclient/Atomic.h" #include "fdbclient/Atomic.h"
#include "fdbclient/BlobGranuleCommon.h" #include "fdbclient/BlobGranuleCommon.h"
#include "fdbclient/BlobGranuleRequest.actor.h"
#include "fdbclient/ClusterInterface.h" #include "fdbclient/ClusterInterface.h"
#include "fdbclient/ClusterConnectionFile.h" #include "fdbclient/ClusterConnectionFile.h"
#include "fdbclient/ClusterConnectionMemoryRecord.h" #include "fdbclient/ClusterConnectionMemoryRecord.h"
@ -8526,6 +8527,41 @@ Future<Version> DatabaseContext::verifyBlobRange(const KeyRange& range,
return verifyBlobRangeActor(Reference<DatabaseContext>::addRef(this), range, version, tenant); return verifyBlobRangeActor(Reference<DatabaseContext>::addRef(this), range, version, tenant);
} }
ACTOR Future<bool> flushBlobRangeActor(Reference<DatabaseContext> cx,
KeyRange range,
bool compact,
Optional<Version> version,
Optional<Reference<Tenant>> tenant) {
if (tenant.present()) {
wait(tenant.get()->ready());
range = range.withPrefix(tenant.get()->prefix());
}
state Database db(cx);
if (!version.present()) {
state Transaction tr(db);
Version _v = wait(tr.getReadVersion());
version = _v;
}
FlushGranuleRequest req(-1, range, version.get(), compact);
try {
wait(success(doBlobGranuleRequests(db, range, req, &BlobWorkerInterface::flushGranuleRequest)));
return true;
} catch (Error& e) {
if (e.code() == error_code_blob_granule_transaction_too_old) {
// can't flush data at this version, because no granules
return false;
}
throw e;
}
}
Future<bool> DatabaseContext::flushBlobRange(const KeyRange& range,
bool compact,
Optional<Version> version,
Optional<Reference<Tenant>> tenant) {
return flushBlobRangeActor(Reference<DatabaseContext>::addRef(this), range, compact, version, tenant);
}
ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx, ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
bool primary, bool primary,
bool use_system_priority) { bool use_system_priority) {
@ -10928,8 +10964,39 @@ ACTOR Future<bool> setBlobRangeActor(Reference<DatabaseContext> cx,
} }
} }
ACTOR Future<bool> blobbifyRangeActor(Reference<DatabaseContext> cx,
KeyRange range,
bool doWait,
Optional<Reference<Tenant>> tenant) {
if (BG_REQUEST_DEBUG) {
fmt::print("BlobbifyRange [{0} - {1}) ({2})\n", range.begin.printable(), range.end.printable(), doWait);
}
state bool result = wait(setBlobRangeActor(cx, range, true, tenant));
if (!doWait || !result) {
return result;
}
// FIXME: add blob worker verifyRange rpc call that just waits for granule to become readable at any version
loop {
Version verifyVersion = wait(cx->verifyBlobRange(range, latestVersion, tenant));
if (verifyVersion != invalidVersion) {
if (BG_REQUEST_DEBUG) {
fmt::print("BlobbifyRange [{0} - {1}) got complete @ {2}\n",
range.begin.printable(),
range.end.printable(),
verifyVersion);
}
return result;
}
wait(delay(0.1));
}
}
Future<bool> DatabaseContext::blobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant) { Future<bool> DatabaseContext::blobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant) {
return setBlobRangeActor(Reference<DatabaseContext>::addRef(this), range, true, tenant); return blobbifyRangeActor(Reference<DatabaseContext>::addRef(this), range, false, tenant);
}
Future<bool> DatabaseContext::blobbifyRangeBlocking(KeyRange range, Optional<Reference<Tenant>> tenant) {
return blobbifyRangeActor(Reference<DatabaseContext>::addRef(this), range, true, tenant);
} }
Future<bool> DatabaseContext::unblobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant) { Future<bool> DatabaseContext::unblobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant) {

27
fdbclient/RESTClient.actor.cpp
View File

@ -40,6 +40,16 @@
#include "flow/actorcompiler.h" // always the last include #include "flow/actorcompiler.h" // always the last include
#define ENABLE_VERBOSE_DEBUG true
#define TRACE_REST_OP(opName, url, secure) \
do { \
if (ENABLE_VERBOSE_DEBUG) { \
const std::string urlStr = url.toString(); \
TraceEvent("RESTClientOp").detail("Op", #opName).detail("Url", urlStr).detail("IsSecure", secure); \
} \
} while (0);
json_spirit::mObject RESTClient::Stats::getJSON() { json_spirit::mObject RESTClient::Stats::getJSON() {
json_spirit::mObject o; json_spirit::mObject o;
@ -61,10 +71,13 @@ RESTClient::Stats RESTClient::Stats::operator-(const Stats& rhs) {
return r; return r;
} }
RESTClient::RESTClient() {} RESTClient::RESTClient() {
conectionPool = makeReference<RESTConnectionPool>(knobs.connection_pool_size);
}
RESTClient::RESTClient(std::unordered_map<std::string, int>& knobSettings) { RESTClient::RESTClient(std::unordered_map<std::string, int>& knobSettings) {
knobs.set(knobSettings); knobs.set(knobSettings);
conectionPool = makeReference<RESTConnectionPool>(knobs.connection_pool_size);
} }
void RESTClient::setKnobs(const std::unordered_map<std::string, int>& knobSettings) { void RESTClient::setKnobs(const std::unordered_map<std::string, int>& knobSettings) {
@ -83,6 +96,10 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<RESTClient> cli
state UnsentPacketQueue content; state UnsentPacketQueue content;
state int contentLen = url->body.size(); state int contentLen = url->body.size();
if (ENABLE_VERBOSE_DEBUG) {
TraceEvent(SevDebug, "DoRequestImpl").detail("Url", url->toString());
}
if (url->body.size() > 0) { if (url->body.size() > 0) {
PacketWriter pw(content.getWriteBuffer(url->body.size()), nullptr, Unversioned()); PacketWriter pw(content.getWriteBuffer(url->body.size()), nullptr, Unversioned());
pw.serializeBytes(url->body); pw.serializeBytes(url->body);
@ -166,7 +183,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<RESTClient> cli
// But only if our previous attempt was not the last allowable try. // But only if our previous attempt was not the last allowable try.
retryable = retryable && (thisTry < maxTries); retryable = retryable && (thisTry < maxTries);
TraceEvent event(SevWarn, retryable ? "RESTClient_FailedRetryable" : "RESTClient_RequestFailed"); TraceEvent event(SevWarn, retryable ? "RESTClientFailedRetryable" : "RESTClientRequestFailed");
// Attach err to trace event if present, otherwise extract some stuff from the response // Attach err to trace event if present, otherwise extract some stuff from the response
if (err.present()) { if (err.present()) {
@ -269,6 +286,7 @@ Future<Reference<HTTP::Response>> RESTClient::doPost(const std::string& fullUrl,
const std::string& requestBody, const std::string& requestBody,
Optional<HTTP::Headers> optHeaders) { Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, requestBody, knobs.secure_connection); RESTUrl url(fullUrl, requestBody, knobs.secure_connection);
TRACE_REST_OP("DoPost", url, knobs.secure_connection);
return doPutOrPost(HTTP::HTTP_VERB_POST, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK }); return doPutOrPost(HTTP::HTTP_VERB_POST, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK });
} }
@ -276,6 +294,7 @@ Future<Reference<HTTP::Response>> RESTClient::doPut(const std::string& fullUrl,
const std::string& requestBody, const std::string& requestBody,
Optional<HTTP::Headers> optHeaders) { Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, requestBody, knobs.secure_connection); RESTUrl url(fullUrl, requestBody, knobs.secure_connection);
TRACE_REST_OP("DoPut", url, knobs.secure_connection);
return doPutOrPost( return doPutOrPost(
HTTP::HTTP_VERB_PUT, HTTP::HTTP_VERB_PUT,
optHeaders, optHeaders,
@ -299,16 +318,19 @@ Future<Reference<HTTP::Response>> RESTClient::doGetHeadDeleteOrTrace(const std::
Future<Reference<HTTP::Response>> RESTClient::doGet(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) { Future<Reference<HTTP::Response>> RESTClient::doGet(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, knobs.secure_connection); RESTUrl url(fullUrl, knobs.secure_connection);
TRACE_REST_OP("DoGet", url, knobs.secure_connection);
return doGetHeadDeleteOrTrace(HTTP::HTTP_VERB_GET, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK }); return doGetHeadDeleteOrTrace(HTTP::HTTP_VERB_GET, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK });
} }
Future<Reference<HTTP::Response>> RESTClient::doHead(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) { Future<Reference<HTTP::Response>> RESTClient::doHead(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, knobs.secure_connection); RESTUrl url(fullUrl, knobs.secure_connection);
TRACE_REST_OP("DoHead", url, knobs.secure_connection);
return doGetHeadDeleteOrTrace(HTTP::HTTP_VERB_HEAD, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK }); return doGetHeadDeleteOrTrace(HTTP::HTTP_VERB_HEAD, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK });
} }
Future<Reference<HTTP::Response>> RESTClient::doDelete(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) { Future<Reference<HTTP::Response>> RESTClient::doDelete(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, knobs.secure_connection); RESTUrl url(fullUrl, knobs.secure_connection);
TRACE_REST_OP("DoDelete", url, knobs.secure_connection);
return doGetHeadDeleteOrTrace( return doGetHeadDeleteOrTrace(
HTTP::HTTP_VERB_DELETE, HTTP::HTTP_VERB_DELETE,
optHeaders, optHeaders,
@ -321,6 +343,7 @@ Future<Reference<HTTP::Response>> RESTClient::doDelete(const std::string& fullUr
Future<Reference<HTTP::Response>> RESTClient::doTrace(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) { Future<Reference<HTTP::Response>> RESTClient::doTrace(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, knobs.secure_connection); RESTUrl url(fullUrl, knobs.secure_connection);
TRACE_REST_OP("DoTrace", url, knobs.secure_connection);
return doGetHeadDeleteOrTrace( return doGetHeadDeleteOrTrace(
HTTP::HTTP_VERB_TRACE, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK }); HTTP::HTTP_VERB_TRACE, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK });
} }

32
fdbclient/RESTUtils.actor.cpp
View File

@ -25,6 +25,7 @@
#include "flow/IConnection.h" #include "flow/IConnection.h"
#include <boost/algorithm/string.hpp> #include <boost/algorithm/string.hpp>
#include <queue>
#include "flow/actorcompiler.h" // always the last include #include "flow/actorcompiler.h" // always the last include
@ -66,12 +67,12 @@ RESTClientKnobs::RESTClientKnobs() {
} }
void RESTClientKnobs::set(const std::unordered_map<std::string, int>& knobSettings) { void RESTClientKnobs::set(const std::unordered_map<std::string, int>& knobSettings) {
TraceEvent trace = TraceEvent("RESTClient_SetKnobs"); TraceEvent trace = TraceEvent("RESTClientSetKnobs");
for (const auto& itr : knobSettings) { for (const auto& itr : knobSettings) {
const auto& kItr = RESTClientKnobs::knobMap.find(itr.first); const auto& kItr = RESTClientKnobs::knobMap.find(itr.first);
if (kItr == RESTClientKnobs::knobMap.end()) { if (kItr == RESTClientKnobs::knobMap.end()) {
trace.detail("RESTClient_InvalidKnobName", itr.first); trace.detail("RESTClientInvalidKnobName", itr.first);
throw rest_invalid_rest_client_knob(); throw rest_invalid_rest_client_knob();
} }
*(kItr->second) = itr.second; *(kItr->second) = itr.second;
@ -98,28 +99,37 @@ ACTOR Future<RESTConnectionPool::ReusableConnection> connect_impl(Reference<REST
bool isSecure, bool isSecure,
int maxConnLife) { int maxConnLife) {
auto poolItr = connectionPool->connectionPoolMap.find(connectKey); auto poolItr = connectionPool->connectionPoolMap.find(connectKey);
if (poolItr == connectionPool->connectionPoolMap.end()) { while (poolItr != connectionPool->connectionPoolMap.end() && !poolItr->second.empty()) {
throw rest_connectpool_key_not_found();
}
while (!poolItr->second.empty()) {
RESTConnectionPool::ReusableConnection rconn = poolItr->second.front(); RESTConnectionPool::ReusableConnection rconn = poolItr->second.front();
poolItr->second.pop(); poolItr->second.pop();
if (rconn.expirationTime > now()) { if (rconn.expirationTime > now()) {
TraceEvent("RESTClient_ReusableConnection") TraceEvent("RESTClientReuseConn")
.suppressFor(60) .suppressFor(60)
.detail("Host", connectKey.first)
.detail("Service", connectKey.second)
.detail("RemoteEndpoint", rconn.conn->getPeerAddress()) .detail("RemoteEndpoint", rconn.conn->getPeerAddress())
.detail("ExpireIn", rconn.expirationTime - now()); .detail("ExpireIn", rconn.expirationTime - now());
return rconn; return rconn;
} }
} }
// No valid connection exists, create a new one
state Reference<IConnection> conn = state Reference<IConnection> conn =
wait(INetworkConnections::net()->connect(connectKey.first, connectKey.second, isSecure)); wait(INetworkConnections::net()->connect(connectKey.first, connectKey.second, isSecure));
wait(conn->connectHandshake()); wait(conn->connectHandshake());
return RESTConnectionPool::ReusableConnection({ conn, now() + maxConnLife }); RESTConnectionPool::ReusableConnection reusableConn =
RESTConnectionPool::ReusableConnection({ conn, now() + maxConnLife });
connectionPool->connectionPoolMap.insert(
{ connectKey, std::queue<RESTConnectionPool::ReusableConnection>({ reusableConn }) });
TraceEvent("RESTClientCreateNewConn")
.suppressFor(60)
.detail("Host", connectKey.first)
.detail("Service", connectKey.second)
.detail("RemoteEndpoint", conn->getPeerAddress());
return reusableConn;
} }
Future<RESTConnectionPool::ReusableConnection> RESTConnectionPool::connect(RESTConnectionPoolKey connectKey, Future<RESTConnectionPool::ReusableConnection> RESTConnectionPool::connect(RESTConnectionPoolKey connectKey,
@ -188,14 +198,14 @@ void RESTUrl::parseUrl(const std::string& fullUrl, const bool isSecure) {
host = h.toString(); host = h.toString();
service = hRef.eat().toString(); service = hRef.eat().toString();
TraceEvent("RESTClient_ParseURI") TraceEvent("RESTClientParseURI")
.detail("URI", fullUrl) .detail("URI", fullUrl)
.detail("Host", host) .detail("Host", host)
.detail("Service", service) .detail("Service", service)
.detail("Resource", resource) .detail("Resource", resource)
.detail("ReqParameters", reqParameters); .detail("ReqParameters", reqParameters);
} catch (std::string& err) { } catch (std::string& err) {
TraceEvent("RESTClient_ParseError").detail("URI", fullUrl).detail("Error", err); TraceEvent("RESTClientParseError").detail("URI", fullUrl).detail("Error", err);
throw rest_invalid_uri(); throw rest_invalid_uri();
} }
} }

8
fdbclient/ServerKnobs.cpp
View File

@ -302,7 +302,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD, 20 ); init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD, 20 );
init( DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC, isSimulated ? 2 : 21 * 60 * 60 * 24 ); if(randomize && BUGGIFY) DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC = isSimulated ? 0: 120; init( DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC, isSimulated ? 2 : 21 * 60 * 60 * 24 ); if(randomize && BUGGIFY) DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC = isSimulated ? 0: 120;
init( DD_TENANT_AWARENESS_ENABLED, false ); init( DD_TENANT_AWARENESS_ENABLED, false );
init( STORAGE_QUOTA_ENABLED, false ); if(isSimulated) STORAGE_QUOTA_ENABLED = deterministicRandom()->coinflip(); init( STORAGE_QUOTA_ENABLED, true ); if(isSimulated) STORAGE_QUOTA_ENABLED = deterministicRandom()->coinflip();
init( TENANT_CACHE_LIST_REFRESH_INTERVAL, 2 ); if( randomize && BUGGIFY ) TENANT_CACHE_LIST_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10); init( TENANT_CACHE_LIST_REFRESH_INTERVAL, 2 ); if( randomize && BUGGIFY ) TENANT_CACHE_LIST_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10);
init( TENANT_CACHE_STORAGE_USAGE_REFRESH_INTERVAL, 2 ); if( randomize && BUGGIFY ) TENANT_CACHE_STORAGE_USAGE_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10); init( TENANT_CACHE_STORAGE_USAGE_REFRESH_INTERVAL, 2 ); if( randomize && BUGGIFY ) TENANT_CACHE_STORAGE_USAGE_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10);
init( TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL, 10 ); if( randomize && BUGGIFY ) TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10); init( TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL, 10 ); if( randomize && BUGGIFY ) TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10);
@ -987,13 +987,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init ( CLUSTER_RECOVERY_EVENT_NAME_PREFIX, "Master" ); init ( CLUSTER_RECOVERY_EVENT_NAME_PREFIX, "Master" );
// Encryption // Encryption
init( ENABLE_ENCRYPTION, false ); if ( randomize && BUGGIFY ) ENABLE_ENCRYPTION = !ENABLE_ENCRYPTION;
init( ENCRYPTION_MODE, "AES-256-CTR" );
init( SIM_KMS_MAX_KEYS, 4096 ); init( SIM_KMS_MAX_KEYS, 4096 );
init( ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH, 100000 ); init( ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH, 100000 );
init( ENABLE_TLOG_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY && ENABLE_ENCRYPTION ) ENABLE_TLOG_ENCRYPTION = false;
init( ENABLE_STORAGE_SERVER_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY && ENABLE_ENCRYPTION) ENABLE_STORAGE_SERVER_ENCRYPTION = false;
init( ENABLE_BLOB_GRANULE_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY && ENABLE_ENCRYPTION) ENABLE_BLOB_GRANULE_ENCRYPTION = false;
// encrypt key proxy // encrypt key proxy
init( ENABLE_BLOB_GRANULE_COMPRESSION, false ); if ( randomize && BUGGIFY ) { ENABLE_BLOB_GRANULE_COMPRESSION = deterministicRandom()->coinflip(); } init( ENABLE_BLOB_GRANULE_COMPRESSION, false ); if ( randomize && BUGGIFY ) { ENABLE_BLOB_GRANULE_COMPRESSION = deterministicRandom()->coinflip(); }
@ -1060,6 +1055,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( BLOB_RESTORE_MANIFEST_URL, isSimulated ? "file://simfdb/fdbblob/manifest" : "" ); init( BLOB_RESTORE_MANIFEST_URL, isSimulated ? "file://simfdb/fdbblob/manifest" : "" );
init( BLOB_RESTORE_MANIFEST_FILE_MAX_SIZE, isSimulated ? 10000 : 10000000 ); init( BLOB_RESTORE_MANIFEST_FILE_MAX_SIZE, isSimulated ? 10000 : 10000000 );
init( BLOB_RESTORE_MANIFEST_RETENTION_MAX, 10 ); init( BLOB_RESTORE_MANIFEST_RETENTION_MAX, 10 );
init( BLOB_RESTORE_MLOGS_RETENTION_SECS, isSimulated ? 120 : 3600 * 24 * 14 );
init( BGCC_TIMEOUT, isSimulated ? 10.0 : 120.0 ); init( BGCC_TIMEOUT, isSimulated ? 10.0 : 120.0 );
init( BGCC_MIN_INTERVAL, isSimulated ? 1.0 : 10.0 ); init( BGCC_MIN_INTERVAL, isSimulated ? 1.0 : 10.0 );

1
fdbclient/SystemData.cpp
View File

@ -1742,6 +1742,7 @@ Standalone<BlobRestoreArg> decodeBlobRestoreArg(ValueRef const& value) {
} }
const Key blobManifestVersionKey = "\xff\x02/blobManifestVersion"_sr; const Key blobManifestVersionKey = "\xff\x02/blobManifestVersion"_sr;
const Key blobGranulesLastFlushKey = "\xff\x02/blobGranulesLastFlushTs"_sr;
const KeyRangeRef idempotencyIdKeys("\xff\x02/idmp/"_sr, "\xff\x02/idmp0"_sr); const KeyRangeRef idempotencyIdKeys("\xff\x02/idmp/"_sr, "\xff\x02/idmp0"_sr);
const KeyRef idempotencyIdsExpiredVersion("\xff\x02/idmpExpiredVersion"_sr); const KeyRef idempotencyIdsExpiredVersion("\xff\x02/idmpExpiredVersion"_sr);

13
fdbclient/Tenant.cpp
View File

@ -55,6 +55,17 @@ int64_t prefixToId(KeyRef prefix, EnforceValidTenantId enforceValidTenantId) {
return id; return id;
} }
KeyRangeRef clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena) {
if (tenantInfo.hasTenant()) {
return KeyRangeRef(range.begin.startsWith(tenantInfo.prefix.get()) ? range.begin : tenantInfo.prefix.get(),
range.end.startsWith(tenantInfo.prefix.get())
? range.end
: allKeys.end.withPrefix(tenantInfo.prefix.get(), arena));
} else {
return range;
}
}
bool withinSingleTenant(KeyRangeRef const& range) { bool withinSingleTenant(KeyRangeRef const& range) {
if (range.begin >= "\x80"_sr || range.begin.size() < TenantAPI::PREFIX_SIZE) { if (range.begin >= "\x80"_sr || range.begin.size() < TenantAPI::PREFIX_SIZE) {
return false; return false;
@ -63,7 +74,7 @@ bool withinSingleTenant(KeyRangeRef const& range) {
return tRange.contains(range); return tRange.contains(range);
} }
}; // namespace TenantAPI } // namespace TenantAPI
std::string TenantAPI::tenantStateToString(TenantState tenantState) { std::string TenantAPI::tenantStateToString(TenantState tenantState) {
switch (tenantState) { switch (tenantState) {

42
fdbclient/ThreadSafeTransaction.cpp
View File

@ -167,6 +167,15 @@ ThreadFuture<bool> ThreadSafeDatabase::blobbifyRange(const KeyRangeRef& keyRange
}); });
} }
ThreadFuture<bool> ThreadSafeDatabase::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
DatabaseContext* db = this->db;
KeyRange range = keyRange;
return onMainThread([=]() -> Future<bool> {
db->checkDeferredError();
return db->blobbifyRangeBlocking(range);
});
}
ThreadFuture<bool> ThreadSafeDatabase::unblobbifyRange(const KeyRangeRef& keyRange) { ThreadFuture<bool> ThreadSafeDatabase::unblobbifyRange(const KeyRangeRef& keyRange) {
DatabaseContext* db = this->db; DatabaseContext* db = this->db;
KeyRange range = keyRange; KeyRange range = keyRange;
@ -195,6 +204,17 @@ ThreadFuture<Version> ThreadSafeDatabase::verifyBlobRange(const KeyRangeRef& key
}); });
} }
ThreadFuture<bool> ThreadSafeDatabase::flushBlobRange(const KeyRangeRef& keyRange,
bool compact,
Optional<Version> version) {
DatabaseContext* db = this->db;
KeyRange range = keyRange;
return onMainThread([=]() -> Future<bool> {
db->checkDeferredError();
return db->flushBlobRange(range, compact, version);
});
}
ThreadSafeDatabase::ThreadSafeDatabase(ConnectionRecordType connectionRecordType, ThreadSafeDatabase::ThreadSafeDatabase(ConnectionRecordType connectionRecordType,
std::string connectionRecordString, std::string connectionRecordString,
int apiVersion) { int apiVersion) {
@ -277,6 +297,16 @@ ThreadFuture<bool> ThreadSafeTenant::blobbifyRange(const KeyRangeRef& keyRange)
}); });
} }
ThreadFuture<bool> ThreadSafeTenant::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
DatabaseContext* db = this->db->db;
KeyRange range = keyRange;
return onMainThread([=]() -> Future<bool> {
db->checkDeferredError();
db->addref();
return db->blobbifyRangeBlocking(range, Reference<Tenant>::addRef(tenant));
});
}
ThreadFuture<bool> ThreadSafeTenant::unblobbifyRange(const KeyRangeRef& keyRange) { ThreadFuture<bool> ThreadSafeTenant::unblobbifyRange(const KeyRangeRef& keyRange) {
DatabaseContext* db = this->db->db; DatabaseContext* db = this->db->db;
KeyRange range = keyRange; KeyRange range = keyRange;
@ -308,6 +338,18 @@ ThreadFuture<Version> ThreadSafeTenant::verifyBlobRange(const KeyRangeRef& keyRa
}); });
} }
ThreadFuture<bool> ThreadSafeTenant::flushBlobRange(const KeyRangeRef& keyRange,
bool compact,
Optional<Version> version) {
DatabaseContext* db = this->db->db;
KeyRange range = keyRange;
return onMainThread([=]() -> Future<bool> {
db->checkDeferredError();
db->addref();
return db->flushBlobRange(range, compact, version, Reference<Tenant>::addRef(tenant));
});
}
ThreadSafeTenant::~ThreadSafeTenant() { ThreadSafeTenant::~ThreadSafeTenant() {
Tenant* t = this->tenant; Tenant* t = this->tenant;
if (t) if (t)

11
fdbclient/include/fdbclient/BlobCipher.h
View File

@ -154,6 +154,8 @@ private:
uint8_t* buffer; uint8_t* buffer;
}; };
class BlobCipherKey;
#pragma pack(push, 1) // exact fit - no padding #pragma pack(push, 1) // exact fit - no padding
struct BlobCipherDetails { struct BlobCipherDetails {
constexpr static FileIdentifier file_identifier = 1945731; constexpr static FileIdentifier file_identifier = 1945731;
@ -163,7 +165,7 @@ struct BlobCipherDetails {
// BaseCipher encryption key identifier // BaseCipher encryption key identifier
EncryptCipherBaseKeyId baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID; EncryptCipherBaseKeyId baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID;
// Random salt // Random salt
EncryptCipherRandomSalt salt{}; EncryptCipherRandomSalt salt = INVALID_ENCRYPT_RANDOM_SALT;
static uint32_t getSize() { static uint32_t getSize() {
return sizeof(EncryptCipherDomainId) + sizeof(EncryptCipherBaseKeyId) + sizeof(EncryptCipherRandomSalt); return sizeof(EncryptCipherDomainId) + sizeof(EncryptCipherBaseKeyId) + sizeof(EncryptCipherRandomSalt);
@ -175,6 +177,8 @@ struct BlobCipherDetails {
const EncryptCipherRandomSalt& random) const EncryptCipherRandomSalt& random)
: encryptDomainId(dId), baseCipherId(bId), salt(random) {} : encryptDomainId(dId), baseCipherId(bId), salt(random) {}
void validateCipherDetailsWithCipherKey(Reference<BlobCipherKey> headerCipherKey);
bool operator==(const BlobCipherDetails& o) const { bool operator==(const BlobCipherDetails& o) const {
return encryptDomainId == o.encryptDomainId && baseCipherId == o.baseCipherId && salt == o.salt; return encryptDomainId == o.encryptDomainId && baseCipherId == o.baseCipherId && salt == o.salt;
} }
@ -209,6 +213,7 @@ struct EncryptHeaderCipherDetails {
BlobCipherDetails textCipherDetails; BlobCipherDetails textCipherDetails;
Optional<BlobCipherDetails> headerCipherDetails; Optional<BlobCipherDetails> headerCipherDetails;
EncryptHeaderCipherDetails() = default;
EncryptHeaderCipherDetails(const BlobCipherDetails& tCipherDetails) : textCipherDetails(tCipherDetails) {} EncryptHeaderCipherDetails(const BlobCipherDetails& tCipherDetails) : textCipherDetails(tCipherDetails) {}
EncryptHeaderCipherDetails(const BlobCipherDetails& tCipherDetails, const BlobCipherDetails& hCipherDetails) EncryptHeaderCipherDetails(const BlobCipherDetails& tCipherDetails, const BlobCipherDetails& hCipherDetails)
: textCipherDetails(tCipherDetails), headerCipherDetails(hCipherDetails) {} : textCipherDetails(tCipherDetails), headerCipherDetails(hCipherDetails) {}
@ -489,6 +494,7 @@ struct BlobCipherEncryptHeaderRef {
const uint8_t* getIV() const; const uint8_t* getIV() const;
const EncryptHeaderCipherDetails getCipherDetails() const; const EncryptHeaderCipherDetails getCipherDetails() const;
EncryptAuthTokenMode getAuthTokenMode() const; EncryptAuthTokenMode getAuthTokenMode() const;
EncryptCipherDomainId getDomainId() const;
void validateEncryptionHeaderDetails(const BlobCipherDetails& textCipherDetails, void validateEncryptionHeaderDetails(const BlobCipherDetails& textCipherDetails,
const BlobCipherDetails& headerCipherDetails, const BlobCipherDetails& headerCipherDetails,
@ -1005,5 +1011,6 @@ void computeAuthToken(const std::vector<std::pair<const uint8_t*, size_t>>& payl
unsigned int digestMaxBufSz); unsigned int digestMaxBufSz);
EncryptAuthTokenMode getEncryptAuthTokenMode(const EncryptAuthTokenMode mode); EncryptAuthTokenMode getEncryptAuthTokenMode(const EncryptAuthTokenMode mode);
int getEncryptCurrentAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo);
#endif // FDBCLIENT_BLOB_CIPHER_H #endif // FDBCLIENT_BLOB_CIPHER_H

23
fdbclient/include/fdbclient/BlobGranuleCommon.h
View File

@ -197,6 +197,7 @@ struct BlobFilePointerRef {
int64_t offset; int64_t offset;
int64_t length; int64_t length;
int64_t fullFileLength; int64_t fullFileLength;
Version fileVersion;
Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx; Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx;
// Non-serializable fields // Non-serializable fields
@ -205,25 +206,34 @@ struct BlobFilePointerRef {
BlobFilePointerRef() {} BlobFilePointerRef() {}
BlobFilePointerRef(Arena& to, const std::string& filename, int64_t offset, int64_t length, int64_t fullFileLength) BlobFilePointerRef(Arena& to,
: filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength) {} const std::string& filename,
int64_t offset,
int64_t length,
int64_t fullFileLength,
Version fileVersion)
: filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength),
fileVersion(fileVersion) {}
BlobFilePointerRef(Arena& to, BlobFilePointerRef(Arena& to,
const std::string& filename, const std::string& filename,
int64_t offset, int64_t offset,
int64_t length, int64_t length,
int64_t fullFileLength, int64_t fullFileLength,
Version fileVersion,
Optional<BlobGranuleCipherKeysCtx> ciphKeysCtx) Optional<BlobGranuleCipherKeysCtx> ciphKeysCtx)
: filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength), : filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength),
cipherKeysCtx(ciphKeysCtx) {} fileVersion(fileVersion), cipherKeysCtx(ciphKeysCtx) {}
BlobFilePointerRef(Arena& to, BlobFilePointerRef(Arena& to,
const std::string& filename, const std::string& filename,
int64_t offset, int64_t offset,
int64_t length, int64_t length,
int64_t fullFileLength, int64_t fullFileLength,
Version fileVersion,
Optional<BlobGranuleCipherKeysMeta> ciphKeysMeta) Optional<BlobGranuleCipherKeysMeta> ciphKeysMeta)
: filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength) { : filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength),
fileVersion(fileVersion) {
if (ciphKeysMeta.present()) { if (ciphKeysMeta.present()) {
cipherKeysMetaRef = BlobGranuleCipherKeysMetaRef(to, ciphKeysMeta.get()); cipherKeysMetaRef = BlobGranuleCipherKeysMetaRef(to, ciphKeysMeta.get());
} }
@ -231,12 +241,12 @@ struct BlobFilePointerRef {
template <class Ar> template <class Ar>
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, filename, offset, length, fullFileLength, cipherKeysCtx); serializer(ar, filename, offset, length, fullFileLength, fileVersion, cipherKeysCtx);
} }
std::string toString() const { std::string toString() const {
std::stringstream ss; std::stringstream ss;
ss << filename.toString() << ":" << offset << ":" << length << ":" << fullFileLength; ss << filename.toString() << ":" << offset << ":" << length << ":" << fullFileLength << "@" << fileVersion;
if (cipherKeysCtx.present()) { if (cipherKeysCtx.present()) {
ss << ":CipherKeysCtx:TextCipher:" << cipherKeysCtx.get().textCipherKey.encryptDomainId << ":" ss << ":CipherKeysCtx:TextCipher:" << cipherKeysCtx.get().textCipherKey.encryptDomainId << ":"
<< cipherKeysCtx.get().textCipherKey.baseCipherId << ":" << cipherKeysCtx.get().textCipherKey.salt << cipherKeysCtx.get().textCipherKey.baseCipherId << ":" << cipherKeysCtx.get().textCipherKey.salt
@ -257,6 +267,7 @@ struct BlobGranuleChunkRef {
constexpr static FileIdentifier file_identifier = 865198; constexpr static FileIdentifier file_identifier = 865198;
KeyRangeRef keyRange; KeyRangeRef keyRange;
Version includedVersion; Version includedVersion;
// FIXME: remove snapshotVersion, it is deprecated with fileVersion in BlobFilePointerRef
Version snapshotVersion; Version snapshotVersion;
Optional<BlobFilePointerRef> snapshotFile; // not set if it's an incremental read Optional<BlobFilePointerRef> snapshotFile; // not set if it's an incremental read
VectorRef<BlobFilePointerRef> deltaFiles; VectorRef<BlobFilePointerRef> deltaFiles;

1
fdbclient/include/fdbclient/ClientKnobs.h
View File

@ -302,6 +302,7 @@ public:
// key_not_found errors for. If TenantInfo::INVALID_TENANT is contained within the list then no tenants will be // key_not_found errors for. If TenantInfo::INVALID_TENANT is contained within the list then no tenants will be
// dropped. This Knob should ONLY be used in simulation for testing purposes // dropped. This Knob should ONLY be used in simulation for testing purposes
std::string SIMULATION_EKP_TENANT_IDS_TO_DROP; std::string SIMULATION_EKP_TENANT_IDS_TO_DROP;
bool SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS;
bool ENABLE_CONFIGURABLE_ENCRYPTION; bool ENABLE_CONFIGURABLE_ENCRYPTION;
int ENCRYPT_HEADER_FLAGS_VERSION; int ENCRYPT_HEADER_FLAGS_VERSION;
int ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION; int ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION;

112
fdbclient/include/fdbclient/CommitTransaction.h
View File

@ -30,6 +30,8 @@
#include "flow/EncryptUtils.h" #include "flow/EncryptUtils.h"
#include "flow/Knobs.h" #include "flow/Knobs.h"
#include <unordered_set>
// The versioned message has wire format : -1, version, messages // The versioned message has wire format : -1, version, messages
static const int32_t VERSION_HEADER = -1; static const int32_t VERSION_HEADER = -1;
@ -143,6 +145,38 @@ struct MutationRef {
return reinterpret_cast<const BlobCipherEncryptHeader*>(param1.begin()); return reinterpret_cast<const BlobCipherEncryptHeader*>(param1.begin());
} }
const BlobCipherEncryptHeaderRef configurableEncryptionHeader() const {
ASSERT(isEncrypted());
return BlobCipherEncryptHeaderRef::fromStringRef(param1);
}
EncryptCipherDomainId encryptDomainId() const {
ASSERT(isEncrypted());
return CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION ? configurableEncryptionHeader().getDomainId()
: encryptionHeader()->cipherTextDetails.encryptDomainId;
}
void updateEncryptCipherDetails(std::unordered_set<BlobCipherDetails>& cipherDetails) {
ASSERT(isEncrypted());
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
EncryptHeaderCipherDetails details = header.getCipherDetails();
ASSERT(details.textCipherDetails.isValid());
cipherDetails.insert(details.textCipherDetails);
if (details.headerCipherDetails.present()) {
ASSERT(details.headerCipherDetails.get().isValid());
cipherDetails.insert(details.headerCipherDetails.get());
}
} else {
const BlobCipherEncryptHeader* header = encryptionHeader();
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
}
}
MutationRef encrypt(TextAndHeaderCipherKeys cipherKeys, MutationRef encrypt(TextAndHeaderCipherKeys cipherKeys,
Arena& arena, Arena& arena,
BlobCipherMetrics::UsageType usageType) const { BlobCipherMetrics::UsageType usageType) const {
@ -150,6 +184,7 @@ struct MutationRef {
deterministicRandom()->randomBytes(iv, AES_256_IV_LENGTH); deterministicRandom()->randomBytes(iv, AES_256_IV_LENGTH);
BinaryWriter bw(AssumeVersion(ProtocolVersion::withEncryptionAtRest())); BinaryWriter bw(AssumeVersion(ProtocolVersion::withEncryptionAtRest()));
bw << *this; bw << *this;
EncryptBlobCipherAes265Ctr cipher( EncryptBlobCipherAes265Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherTextKey,
cipherKeys.cipherHeaderKey, cipherKeys.cipherHeaderKey,
@ -157,11 +192,22 @@ struct MutationRef {
AES_256_IV_LENGTH, AES_256_IV_LENGTH,
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE), getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
usageType); usageType);
BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader;
StringRef headerRef(reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader)); StringRef serializedHeader;
StringRef payload = StringRef payload;
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), header, arena)->toStringRef(); if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
return MutationRef(Encrypted, headerRef, payload); BlobCipherEncryptHeaderRef header;
payload = cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), &header, arena);
Standalone<StringRef> headerStr = BlobCipherEncryptHeaderRef::toStringRef(header);
arena.dependsOn(headerStr.arena());
serializedHeader = headerStr;
} else {
BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader;
serializedHeader = StringRef(reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader));
payload =
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), header, arena)->toStringRef();
}
return MutationRef(Encrypted, serializedHeader, payload);
} }
MutationRef encrypt(const std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>& cipherKeys, MutationRef encrypt(const std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>& cipherKeys,
@ -183,6 +229,7 @@ struct MutationRef {
deterministicRandom()->randomBytes(iv, AES_256_IV_LENGTH); deterministicRandom()->randomBytes(iv, AES_256_IV_LENGTH);
BinaryWriter bw(AssumeVersion(ProtocolVersion::withEncryptionAtRest())); BinaryWriter bw(AssumeVersion(ProtocolVersion::withEncryptionAtRest()));
bw << *this; bw << *this;
EncryptBlobCipherAes265Ctr cipher( EncryptBlobCipherAes265Ctr cipher(
textCipherKey, textCipherKey,
headerCipherKey, headerCipherKey,
@ -190,11 +237,22 @@ struct MutationRef {
AES_256_IV_LENGTH, AES_256_IV_LENGTH,
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE), getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
usageType); usageType);
BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader;
StringRef headerRef(reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader)); if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
StringRef payload = BlobCipherEncryptHeaderRef header;
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), header, arena)->toStringRef(); StringRef payload =
return MutationRef(Encrypted, headerRef, payload); cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), &header, arena);
Standalone<StringRef> serializedHeader = BlobCipherEncryptHeaderRef::toStringRef(header);
arena.dependsOn(serializedHeader.arena());
return MutationRef(Encrypted, serializedHeader, payload);
} else {
BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader;
StringRef serializedHeader =
StringRef(reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader));
StringRef payload =
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), header, arena)->toStringRef();
return MutationRef(Encrypted, serializedHeader, payload);
}
} }
MutationRef encryptMetadata(const std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>& cipherKeys, MutationRef encryptMetadata(const std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>& cipherKeys,
@ -207,9 +265,18 @@ struct MutationRef {
Arena& arena, Arena& arena,
BlobCipherMetrics::UsageType usageType, BlobCipherMetrics::UsageType usageType,
StringRef* buf = nullptr) const { StringRef* buf = nullptr) const {
const BlobCipherEncryptHeader* header = encryptionHeader(); StringRef plaintext;
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header->iv, usageType); if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
StringRef plaintext = cipher.decrypt(param2.begin(), param2.size(), *header, arena)->toStringRef(); const BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.getIV(), usageType);
plaintext = cipher.decrypt(param2.begin(), param2.size(), header, arena);
} else {
const BlobCipherEncryptHeader* header = encryptionHeader();
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header->iv, usageType);
plaintext = cipher.decrypt(param2.begin(), param2.size(), *header, arena)->toStringRef();
}
if (buf != nullptr) { if (buf != nullptr) {
*buf = plaintext; *buf = plaintext;
} }
@ -229,7 +296,6 @@ struct MutationRef {
TextAndHeaderCipherKeys getCipherKeys( TextAndHeaderCipherKeys getCipherKeys(
const std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>& cipherKeys) const { const std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>& cipherKeys) const {
const BlobCipherEncryptHeader* header = encryptionHeader();
auto getCipherKey = [&](const BlobCipherDetails& details) -> Reference<BlobCipherKey> { auto getCipherKey = [&](const BlobCipherDetails& details) -> Reference<BlobCipherKey> {
if (!details.isValid()) { if (!details.isValid()) {
return {}; return {};
@ -239,8 +305,22 @@ struct MutationRef {
return iter->second; return iter->second;
}; };
TextAndHeaderCipherKeys textAndHeaderKeys; TextAndHeaderCipherKeys textAndHeaderKeys;
textAndHeaderKeys.cipherHeaderKey = getCipherKey(header->cipherHeaderDetails); if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
textAndHeaderKeys.cipherTextKey = getCipherKey(header->cipherTextDetails); const BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
EncryptHeaderCipherDetails cipherDetails = header.getCipherDetails();
ASSERT(cipherDetails.textCipherDetails.isValid());
textAndHeaderKeys.cipherTextKey = getCipherKey(cipherDetails.textCipherDetails);
if (cipherDetails.headerCipherDetails.present()) {
ASSERT(cipherDetails.headerCipherDetails.get().isValid());
textAndHeaderKeys.cipherHeaderKey = getCipherKey(cipherDetails.headerCipherDetails.get());
} else {
ASSERT(!FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED);
}
} else {
const BlobCipherEncryptHeader* header = encryptionHeader();
textAndHeaderKeys.cipherHeaderKey = getCipherKey(header->cipherHeaderDetails);
textAndHeaderKeys.cipherTextKey = getCipherKey(header->cipherTextDetails);
}
return textAndHeaderKeys; return textAndHeaderKeys;
} }

5
fdbclient/include/fdbclient/DatabaseContext.h
View File

@ -406,6 +406,7 @@ public:
Future<Void> waitPurgeGranulesComplete(Key purgeKey); Future<Void> waitPurgeGranulesComplete(Key purgeKey);
Future<bool> blobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant = {}); Future<bool> blobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant = {});
Future<bool> blobbifyRangeBlocking(KeyRange range, Optional<Reference<Tenant>> tenant = {});
Future<bool> unblobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant = {}); Future<bool> unblobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant = {});
Future<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(KeyRange range, Future<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(KeyRange range,
int rangeLimit, int rangeLimit,
@ -413,6 +414,10 @@ public:
Future<Version> verifyBlobRange(const KeyRange& range, Future<Version> verifyBlobRange(const KeyRange& range,
Optional<Version> version, Optional<Version> version,
Optional<Reference<Tenant>> tenant = {}); Optional<Reference<Tenant>> tenant = {});
Future<bool> flushBlobRange(const KeyRange& range,
bool compact,
Optional<Version> version,
Optional<Reference<Tenant>> tenant = {});
Future<bool> blobRestore(const KeyRange range, Optional<Version> version); Future<bool> blobRestore(const KeyRange range, Optional<Version> version);
// private: // private:

4
fdbclient/include/fdbclient/FDBTypes.h
View File

@ -1063,9 +1063,9 @@ struct StorageBytes {
constexpr static FileIdentifier file_identifier = 3928581; constexpr static FileIdentifier file_identifier = 3928581;
// Free space on the filesystem // Free space on the filesystem
int64_t free; int64_t free;
// Total space on the filesystem // Total capacity on the filesystem usable by non-privileged users.
int64_t total; int64_t total;
// Used by *this* store, not total - free // Total size of all files owned by *this* storage instance, not total - free
int64_t used; int64_t used;
// Amount of space available for use by the store, which includes free space on the filesystem // Amount of space available for use by the store, which includes free space on the filesystem
// and internal free space within the store data that is immediately reusable. // and internal free space within the store data that is immediately reusable.

24
fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h
View File

@ -56,7 +56,7 @@ Future<Void> onEncryptKeyProxyChange(Reference<AsyncVar<T> const> db) {
break; break;
} }
} }
TraceEvent("GetEncryptCipherKeys_EncryptKeyProxyChanged") TraceEvent("GetEncryptCipherKeysEncryptKeyProxyChanged")
.detail("PreviousProxyId", previousProxyId.orDefault(UID())) .detail("PreviousProxyId", previousProxyId.orDefault(UID()))
.detail("CurrentProxyId", currentProxyId.orDefault(UID())); .detail("CurrentProxyId", currentProxyId.orDefault(UID()));
return Void(); return Void();
@ -69,19 +69,19 @@ Future<EKPGetLatestBaseCipherKeysReply> getUncachedLatestEncryptCipherKeys(Refer
Optional<EncryptKeyProxyInterface> proxy = db->get().encryptKeyProxy; Optional<EncryptKeyProxyInterface> proxy = db->get().encryptKeyProxy;
if (!proxy.present()) { if (!proxy.present()) {
// Wait for onEncryptKeyProxyChange. // Wait for onEncryptKeyProxyChange.
TraceEvent("GetLatestEncryptCipherKeys_EncryptKeyProxyNotPresent").detail("UsageType", toString(usageType)); TraceEvent("GetLatestEncryptCipherKeysEncryptKeyProxyNotPresent").detail("UsageType", toString(usageType));
return Never(); return Never();
} }
request.reply.reset(); request.reply.reset();
try { try {
EKPGetLatestBaseCipherKeysReply reply = wait(proxy.get().getLatestBaseCipherKeys.getReply(request)); EKPGetLatestBaseCipherKeysReply reply = wait(proxy.get().getLatestBaseCipherKeys.getReply(request));
if (reply.error.present()) { if (reply.error.present()) {
TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_RequestFailed").error(reply.error.get()); TraceEvent(SevWarn, "GetLatestEncryptCipherKeysRequestFailed").error(reply.error.get());
throw encrypt_keys_fetch_failed(); throw encrypt_keys_fetch_failed();
} }
return reply; return reply;
} catch (Error& e) { } catch (Error& e) {
TraceEvent("GetLatestEncryptCipherKeys_CaughtError").error(e); TraceEvent("GetLatestEncryptCipherKeysCaughtError").error(e);
if (e.code() == error_code_broken_promise) { if (e.code() == error_code_broken_promise) {
// Wait for onEncryptKeyProxyChange. // Wait for onEncryptKeyProxyChange.
return Never(); return Never();
@ -103,7 +103,7 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
state EKPGetLatestBaseCipherKeysRequest request; state EKPGetLatestBaseCipherKeysRequest request;
if (!db.isValid()) { if (!db.isValid()) {
TraceEvent(SevError, "GetLatestEncryptCipherKeys_ServerDBInfoNotAvailable"); TraceEvent(SevError, "GetLatestEncryptCipherKeysServerDBInfoNotAvailable");
throw encrypt_ops_error(); throw encrypt_ops_error();
} }
@ -140,7 +140,7 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
// Check for any missing cipher keys. // Check for any missing cipher keys.
for (auto domainId : request.encryptDomainIds) { for (auto domainId : request.encryptDomainIds) {
if (cipherKeys.count(domainId) == 0) { if (cipherKeys.count(domainId) == 0) {
TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_KeyMissing").detail("DomainId", domainId); TraceEvent(SevWarn, "GetLatestEncryptCipherKeysKeyMissing").detail("DomainId", domainId);
throw encrypt_key_not_found(); throw encrypt_key_not_found();
} }
} }
@ -176,14 +176,14 @@ Future<EKPGetBaseCipherKeysByIdsReply> getUncachedEncryptCipherKeys(Reference<As
Optional<EncryptKeyProxyInterface> proxy = db->get().encryptKeyProxy; Optional<EncryptKeyProxyInterface> proxy = db->get().encryptKeyProxy;
if (!proxy.present()) { if (!proxy.present()) {
// Wait for onEncryptKeyProxyChange. // Wait for onEncryptKeyProxyChange.
TraceEvent("GetEncryptCipherKeys_EncryptKeyProxyNotPresent").detail("UsageType", toString(usageType)); TraceEvent("GetEncryptCipherKeysEncryptKeyProxyNotPresent").detail("UsageType", toString(usageType));
return Never(); return Never();
} }
request.reply.reset(); request.reply.reset();
try { try {
EKPGetBaseCipherKeysByIdsReply reply = wait(proxy.get().getBaseCipherKeysByIds.getReply(request)); EKPGetBaseCipherKeysByIdsReply reply = wait(proxy.get().getBaseCipherKeysByIds.getReply(request));
if (reply.error.present()) { if (reply.error.present()) {
TraceEvent(SevWarn, "GetEncryptCipherKeys_RequestFailed").error(reply.error.get()); TraceEvent(SevWarn, "GetEncryptCipherKeysRequestFailed").error(reply.error.get());
throw encrypt_keys_fetch_failed(); throw encrypt_keys_fetch_failed();
} }
if (g_network && g_network->isSimulated() && usageType == BlobCipherMetrics::RESTORE) { if (g_network && g_network->isSimulated() && usageType == BlobCipherMetrics::RESTORE) {
@ -192,7 +192,7 @@ Future<EKPGetBaseCipherKeysByIdsReply> getUncachedEncryptCipherKeys(Reference<As
if (!tenantIdsToDrop.count(TenantInfo::INVALID_TENANT)) { if (!tenantIdsToDrop.count(TenantInfo::INVALID_TENANT)) {
for (auto& baseCipherInfo : request.baseCipherInfos) { for (auto& baseCipherInfo : request.baseCipherInfos) {
if (tenantIdsToDrop.count(baseCipherInfo.domainId)) { if (tenantIdsToDrop.count(baseCipherInfo.domainId)) {
TraceEvent("GetEncryptCipherKeys_SimulatedError").detail("DomainId", baseCipherInfo.domainId); TraceEvent("GetEncryptCipherKeysSimulatedError").detail("DomainId", baseCipherInfo.domainId);
throw encrypt_keys_fetch_failed(); throw encrypt_keys_fetch_failed();
} }
} }
@ -200,7 +200,7 @@ Future<EKPGetBaseCipherKeysByIdsReply> getUncachedEncryptCipherKeys(Reference<As
} }
return reply; return reply;
} catch (Error& e) { } catch (Error& e) {
TraceEvent("GetEncryptCipherKeys_CaughtError").error(e); TraceEvent("GetEncryptCipherKeysCaughtError").error(e);
if (e.code() == error_code_broken_promise) { if (e.code() == error_code_broken_promise) {
// Wait for onEncryptKeyProxyChange. // Wait for onEncryptKeyProxyChange.
return Never(); return Never();
@ -225,7 +225,7 @@ Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> getEncry
state EKPGetBaseCipherKeysByIdsRequest request; state EKPGetBaseCipherKeysByIdsRequest request;
if (!db.isValid()) { if (!db.isValid()) {
TraceEvent(SevError, "GetEncryptCipherKeys_ServerDBInfoNotAvailable"); TraceEvent(SevError, "GetEncryptCipherKeysServerDBInfoNotAvailable");
throw encrypt_ops_error(); throw encrypt_ops_error();
} }
@ -262,7 +262,7 @@ Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> getEncry
BaseCipherIndex baseIdx = std::make_pair(details.encryptDomainId, details.baseCipherId); BaseCipherIndex baseIdx = std::make_pair(details.encryptDomainId, details.baseCipherId);
const auto& itr = baseCipherKeys.find(baseIdx); const auto& itr = baseCipherKeys.find(baseIdx);
if (itr == baseCipherKeys.end()) { if (itr == baseCipherKeys.end()) {
TraceEvent(SevError, "GetEncryptCipherKeys_KeyMissing") TraceEvent(SevError, "GetEncryptCipherKeysKeyMissing")
.detail("DomainId", details.encryptDomainId) .detail("DomainId", details.encryptDomainId)
.detail("BaseCipherId", details.baseCipherId); .detail("BaseCipherId", details.baseCipherId);
throw encrypt_key_not_found(); throw encrypt_key_not_found();

4
fdbclient/include/fdbclient/IClientApi.h
View File

@ -153,11 +153,13 @@ public:
virtual ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) = 0; virtual ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) = 0;
virtual ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) = 0; virtual ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) = 0; virtual ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange, virtual ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) = 0; int rangeLimit) = 0;
virtual ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) = 0; virtual ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) = 0;
virtual ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) = 0;
virtual void addref() = 0; virtual void addref() = 0;
virtual void delref() = 0; virtual void delref() = 0;
@ -200,11 +202,13 @@ public:
virtual ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) = 0; virtual ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) = 0;
virtual ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) = 0; virtual ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) = 0; virtual ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange, virtual ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) = 0; int rangeLimit) = 0;
virtual ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) = 0; virtual ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) = 0;
virtual ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) = 0;
// Interface to manage shared state across multiple connections to the same Database // Interface to manage shared state across multiple connections to the same Database
virtual ThreadFuture<DatabaseSharedState*> createSharedState() = 0; virtual ThreadFuture<DatabaseSharedState*> createSharedState() = 0;

36
fdbclient/include/fdbclient/MultiVersionTransaction.h
View File

@ -187,6 +187,12 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
uint8_t const* end_key_name, uint8_t const* end_key_name,
int end_key_name_length); int end_key_name_length);
FDBFuture* (*databaseBlobbifyRangeBlocking)(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
FDBFuture* (*databaseUnblobbifyRange)(FDBDatabase* db, FDBFuture* (*databaseUnblobbifyRange)(FDBDatabase* db,
uint8_t const* begin_key_name, uint8_t const* begin_key_name,
int begin_key_name_length, int begin_key_name_length,
@ -207,6 +213,14 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
int end_key_name_length, int end_key_name_length,
int64_t version); int64_t version);
FDBFuture* (*databaseFlushBlobRange)(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version);
FDBFuture* (*databaseGetClientStatus)(FDBDatabase* db); FDBFuture* (*databaseGetClientStatus)(FDBDatabase* db);
// Tenant // Tenant
@ -230,6 +244,12 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
uint8_t const* end_key_name, uint8_t const* end_key_name,
int end_key_name_length); int end_key_name_length);
FDBFuture* (*tenantBlobbifyRangeBlocking)(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
FDBFuture* (*tenantUnblobbifyRange)(FDBTenant* tenant, FDBFuture* (*tenantUnblobbifyRange)(FDBTenant* tenant,
uint8_t const* begin_key_name, uint8_t const* begin_key_name,
int begin_key_name_length, int begin_key_name_length,
@ -250,6 +270,14 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
int end_key_name_length, int end_key_name_length,
int64_t version); int64_t version);
FDBFuture* (*tenantFlushBlobRange)(FDBTenant* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version);
FDBFuture* (*tenantGetId)(FDBTenant* tenant); FDBFuture* (*tenantGetId)(FDBTenant* tenant);
void (*tenantDestroy)(FDBTenant* tenant); void (*tenantDestroy)(FDBTenant* tenant);
@ -547,11 +575,13 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange, ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override; int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override; ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
void addref() override { ThreadSafeReferenceCounted<DLTenant>::addref(); } void addref() override { ThreadSafeReferenceCounted<DLTenant>::addref(); }
void delref() override { ThreadSafeReferenceCounted<DLTenant>::delref(); } void delref() override { ThreadSafeReferenceCounted<DLTenant>::delref(); }
@ -597,11 +627,13 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange, ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override; int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override; ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
ThreadFuture<DatabaseSharedState*> createSharedState() override; ThreadFuture<DatabaseSharedState*> createSharedState() override;
void setSharedState(DatabaseSharedState* p) override; void setSharedState(DatabaseSharedState* p) override;
@ -866,10 +898,12 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange, ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override; int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override; ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
void addref() override { ThreadSafeReferenceCounted<MultiVersionTenant>::addref(); } void addref() override { ThreadSafeReferenceCounted<MultiVersionTenant>::addref(); }
void delref() override { ThreadSafeReferenceCounted<MultiVersionTenant>::delref(); } void delref() override { ThreadSafeReferenceCounted<MultiVersionTenant>::delref(); }
@ -994,10 +1028,12 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange, ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override; int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override; ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
ThreadFuture<DatabaseSharedState*> createSharedState() override; ThreadFuture<DatabaseSharedState*> createSharedState() override;
void setSharedState(DatabaseSharedState* p) override; void setSharedState(DatabaseSharedState* p) override;

6
fdbclient/include/fdbclient/RESTUtils.h
View File

@ -27,6 +27,7 @@
#include "flow/FastRef.h" #include "flow/FastRef.h"
#include "flow/Net2Packet.h" #include "flow/Net2Packet.h"
#include <fmt/format.h>
#include <unordered_map> #include <unordered_map>
#include <utility> #include <utility>
@ -108,6 +109,11 @@ public:
explicit RESTUrl(const std::string& fullUrl, const bool isSecure); explicit RESTUrl(const std::string& fullUrl, const bool isSecure);
explicit RESTUrl(const std::string& fullUrl, const std::string& body, const bool isSecure); explicit RESTUrl(const std::string& fullUrl, const std::string& body, const bool isSecure);
std::string toString() const {
return fmt::format(
"Host {} Service {} Resource {} ReqParams {} Body {}", host, service, resource, reqParameters, body);
}
private: private:
void parseUrl(const std::string& fullUrl, bool isSecure); void parseUrl(const std::string& fullUrl, bool isSecure);
}; };

6
fdbclient/include/fdbclient/ServerKnobs.h
View File

@ -962,13 +962,8 @@ public:
std::string CLUSTER_RECOVERY_EVENT_NAME_PREFIX; std::string CLUSTER_RECOVERY_EVENT_NAME_PREFIX;
// Encryption // Encryption
bool ENABLE_ENCRYPTION;
std::string ENCRYPTION_MODE;
int SIM_KMS_MAX_KEYS; int SIM_KMS_MAX_KEYS;
int ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH; int ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH;
bool ENABLE_TLOG_ENCRYPTION;
bool ENABLE_STORAGE_SERVER_ENCRYPTION; // Currently only Redwood engine supports encryption
bool ENABLE_BLOB_GRANULE_ENCRYPTION;
// Compression // Compression
bool ENABLE_BLOB_GRANULE_COMPRESSION; bool ENABLE_BLOB_GRANULE_COMPRESSION;
@ -1039,6 +1034,7 @@ public:
std::string BLOB_RESTORE_MANIFEST_URL; std::string BLOB_RESTORE_MANIFEST_URL;
int BLOB_RESTORE_MANIFEST_FILE_MAX_SIZE; int BLOB_RESTORE_MANIFEST_FILE_MAX_SIZE;
int BLOB_RESTORE_MANIFEST_RETENTION_MAX; int BLOB_RESTORE_MANIFEST_RETENTION_MAX;
int BLOB_RESTORE_MLOGS_RETENTION_SECS;
// Blob metadata // Blob metadata
int64_t BLOB_METADATA_CACHE_TTL; int64_t BLOB_METADATA_CACHE_TTL;

1
fdbclient/include/fdbclient/SystemData.h
View File

@ -733,6 +733,7 @@ const KeyRange decodeBlobRestoreArgKeyFor(const KeyRef key);
const Value blobRestoreArgValueFor(BlobRestoreArg args); const Value blobRestoreArgValueFor(BlobRestoreArg args);
Standalone<BlobRestoreArg> decodeBlobRestoreArg(ValueRef const& value); Standalone<BlobRestoreArg> decodeBlobRestoreArg(ValueRef const& value);
extern const Key blobManifestVersionKey; extern const Key blobManifestVersionKey;
extern const Key blobGranulesLastFlushKey;
extern const KeyRangeRef idempotencyIdKeys; extern const KeyRangeRef idempotencyIdKeys;
extern const KeyRef idempotencyIdsExpiredVersion; extern const KeyRef idempotencyIdsExpiredVersion;

1
fdbclient/include/fdbclient/Tenant.h
View File

@ -36,6 +36,7 @@ namespace TenantAPI {
KeyRef idToPrefix(Arena& p, int64_t id); KeyRef idToPrefix(Arena& p, int64_t id);
Key idToPrefix(int64_t id); Key idToPrefix(int64_t id);
int64_t prefixToId(KeyRef prefix, EnforceValidTenantId = EnforceValidTenantId::True); int64_t prefixToId(KeyRef prefix, EnforceValidTenantId = EnforceValidTenantId::True);
KeyRangeRef clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena);
// return true if begin and end has the same non-negative prefix id // return true if begin and end has the same non-negative prefix id
bool withinSingleTenant(KeyRangeRef const&); bool withinSingleTenant(KeyRangeRef const&);

4
fdbclient/include/fdbclient/ThreadSafeTransaction.h
View File

@ -64,11 +64,13 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange, ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override; int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override; ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
ThreadFuture<DatabaseSharedState*> createSharedState() override; ThreadFuture<DatabaseSharedState*> createSharedState() override;
void setSharedState(DatabaseSharedState* p) override; void setSharedState(DatabaseSharedState* p) override;
@ -101,11 +103,13 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange, ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override; int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override; ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
void addref() override { ThreadSafeReferenceCounted<ThreadSafeTenant>::addref(); } void addref() override { ThreadSafeReferenceCounted<ThreadSafeTenant>::addref(); }
void delref() override { ThreadSafeReferenceCounted<ThreadSafeTenant>::delref(); } void delref() override { ThreadSafeReferenceCounted<ThreadSafeTenant>::delref(); }

2
fdbrpc/include/fdbrpc/TenantInfo.h
View File

@ -47,7 +47,7 @@ struct TenantInfo {
// Helper function for most endpoints that read/write data. This returns true iff // Helper function for most endpoints that read/write data. This returns true iff
// the client is either a) a trusted peer or b) is accessing keyspace belonging to a tenant, // the client is either a) a trusted peer or b) is accessing keyspace belonging to a tenant,
// for which it has a valid authorization token. // for which it has a valid authorization token.
// NOTE: In a cluster where TenantMode is OPTIONAL or DISABLED, tenant name may be unset. // NOTE: In a cluster where TenantMode is OPTIONAL or DISABLED, tenant ID may be invalid.
// In such case, the request containing such TenantInfo is valid iff the requesting peer is trusted. // In such case, the request containing such TenantInfo is valid iff the requesting peer is trusted.
bool isAuthorized() const { return trusted || tenantAuthorized; } bool isAuthorized() const { return trusted || tenantAuthorized; }
bool hasTenant() const { return tenantId != INVALID_TENANT; } bool hasTenant() const { return tenantId != INVALID_TENANT; }

1
fdbserver/ApplyMetadataMutation.cpp
View File

@ -25,7 +25,6 @@
#include "fdbclient/Notified.h" #include "fdbclient/Notified.h"
#include "fdbclient/SystemData.h" #include "fdbclient/SystemData.h"
#include "fdbserver/ApplyMetadataMutation.h" #include "fdbserver/ApplyMetadataMutation.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/IKeyValueStore.h" #include "fdbserver/IKeyValueStore.h"
#include "fdbserver/Knobs.h" #include "fdbserver/Knobs.h"
#include "fdbserver/LogProtocolMessage.h" #include "fdbserver/LogProtocolMessage.h"

6
fdbserver/BackupWorker.actor.cpp
View File

@ -105,11 +105,7 @@ struct VersionedMessage {
ArenaReader reader(arena, message, AssumeVersion(ProtocolVersion::withEncryptionAtRest())); ArenaReader reader(arena, message, AssumeVersion(ProtocolVersion::withEncryptionAtRest()));
MutationRef m; MutationRef m;
reader >> m; reader >> m;
const BlobCipherEncryptHeader* header = m.encryptionHeader(); m.updateEncryptCipherDetails(cipherDetails);
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
} }
} }
}; };

3
fdbserver/BlobGranuleServerCommon.actor.cpp
View File

@ -209,6 +209,7 @@ void GranuleFiles::getFiles(Version beginVersion,
snapshotF->offset, snapshotF->offset,
snapshotF->length, snapshotF->length,
snapshotF->fullFileLength, snapshotF->fullFileLength,
snapshotF->version,
summarize ? Optional<BlobGranuleCipherKeysMeta>() : snapshotF->cipherKeysMeta); summarize ? Optional<BlobGranuleCipherKeysMeta>() : snapshotF->cipherKeysMeta);
lastIncluded = chunk.snapshotVersion; lastIncluded = chunk.snapshotVersion;
} else { } else {
@ -221,6 +222,7 @@ void GranuleFiles::getFiles(Version beginVersion,
deltaF->offset, deltaF->offset,
deltaF->length, deltaF->length,
deltaF->fullFileLength, deltaF->fullFileLength,
deltaF->version,
summarize ? Optional<BlobGranuleCipherKeysMeta>() : deltaF->cipherKeysMeta); summarize ? Optional<BlobGranuleCipherKeysMeta>() : deltaF->cipherKeysMeta);
deltaBytesCounter += deltaF->length; deltaBytesCounter += deltaF->length;
ASSERT(lastIncluded < deltaF->version); ASSERT(lastIncluded < deltaF->version);
@ -235,6 +237,7 @@ void GranuleFiles::getFiles(Version beginVersion,
deltaF->offset, deltaF->offset,
deltaF->length, deltaF->length,
deltaF->fullFileLength, deltaF->fullFileLength,
deltaF->version,
deltaF->cipherKeysMeta); deltaF->cipherKeysMeta);
deltaBytesCounter += deltaF->length; deltaBytesCounter += deltaF->length;
lastIncluded = deltaF->version; lastIncluded = deltaF->version;

95
fdbserver/BlobManager.actor.cpp
View File

@ -19,14 +19,20 @@
*/ */
#include <algorithm> #include <algorithm>
#include <ctime>
#include <limits> #include <limits>
#include <sstream> #include <sstream>
#include <queue> #include <queue>
#include <vector> #include <vector>
#include <unordered_map> #include <unordered_map>
#include "fdbclient/BackupContainer.h"
#include "fdbclient/ClientBooleanParams.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/ServerKnobs.h" #include "fdbclient/ServerKnobs.h"
#include "fdbrpc/simulator.h" #include "fdbrpc/simulator.h"
#include "flow/CodeProbe.h"
#include "flow/serialize.h"
#include "fmt/format.h" #include "fmt/format.h"
#include "fdbclient/BackupContainerFileSystem.h" #include "fdbclient/BackupContainerFileSystem.h"
#include "fdbclient/BlobGranuleCommon.h" #include "fdbclient/BlobGranuleCommon.h"
@ -38,6 +44,7 @@
#include "fdbclient/ReadYourWrites.h" #include "fdbclient/ReadYourWrites.h"
#include "fdbclient/SystemData.h" #include "fdbclient/SystemData.h"
#include "fdbclient/Tuple.h" #include "fdbclient/Tuple.h"
#include "fdbclient/BackupAgent.actor.h"
#include "fdbserver/BlobManagerInterface.h" #include "fdbserver/BlobManagerInterface.h"
#include "fdbserver/Knobs.h" #include "fdbserver/Knobs.h"
#include "fdbserver/BlobGranuleValidation.actor.h" #include "fdbserver/BlobGranuleValidation.actor.h"
@ -5243,6 +5250,91 @@ ACTOR Future<Void> bgConsistencyCheck(Reference<BlobManagerData> bmData) {
} }
} }
ACTOR Future<Void> updateLastFlushTs(Database db) {
state Transaction tr(db);
loop {
try {
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
KeyBackedProperty<int64_t> lastFlushTs(blobGranulesLastFlushKey);
int64_t epochs = (int64_t)now();
lastFlushTs.set(&tr, epochs);
wait(tr.commit());
return Void();
} catch (Error& e) {
fmt::print("updateLastFlushTs {} \n", e.what());
wait(tr.onError(e));
}
}
}
ACTOR Future<int64_t> getLastFlushTs(Database db) {
state Transaction tr(db);
loop {
try {
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
KeyBackedProperty<int64_t> lastFlushTs(blobGranulesLastFlushKey);
state int64_t ret;
wait(store(ret, lastFlushTs.getD(&tr, Snapshot::False, 0)));
return ret;
} catch (Error& e) {
wait(tr.onError(e));
}
}
}
ACTOR Future<Void> maybeFlushAndTruncateMutationLogs(Reference<BlobManagerData> bmData) {
try {
int64_t lastFlushTs = wait(getLastFlushTs(bmData->db));
bool shouldFlush = (now() - lastFlushTs) > SERVER_KNOBS->BLOB_RESTORE_MLOGS_RETENTION_SECS;
if (!shouldFlush) {
TraceEvent("SkipBlobGranulesFlush").detail("LastFlushTs", lastFlushTs);
return Void();
}
state std::string mlogsUrl = wait(getMutationLogUrl());
state Reference<IBackupContainer> bc = IBackupContainer::openContainer(mlogsUrl, {}, {});
state BackupDescription desc = wait(bc->describeBackup(true));
if (!desc.contiguousLogEnd.present()) {
TraceEvent("SkipBlobGranulesFlush").detail("LogUrl", mlogsUrl);
return Void(); // skip truncation if no valid backup for mutation logs
}
state Version logEndVersion = desc.contiguousLogEnd.get();
// Force flush in-memory data of all blob workers until end of log
FlushGranuleRequest req(bmData->epoch, normalKeys, logEndVersion, false);
wait(success(doBlobGranuleRequests(bmData->db, normalKeys, req, &BlobWorkerInterface::flushGranuleRequest)));
wait(updateLastFlushTs(bmData->db));
// Truncate mutation logs to max retention period
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(bmData->db));
Optional<int64_t> logEndEpochs = wait(timeKeeperEpochsFromVersion(logEndVersion, tr));
if (!logEndEpochs.present()) {
TraceEvent("SkipMutationLogTruncation").detail("LogEndVersion", logEndVersion);
return Void(); // skip truncation if no timestamp about log end
}
// Find timestamp and version to truncate
int64_t epochs = logEndEpochs.get() - SERVER_KNOBS->BLOB_RESTORE_MLOGS_RETENTION_SECS;
if (epochs <= 0) {
TraceEvent("SkipMutationLogTruncation").detail("Epochs", epochs);
return Void();
}
state std::string timestamp = BackupAgentBase::formatTime(epochs);
state Version truncVersion = wait(timeKeeperVersionFromDatetime(timestamp, bmData->db));
wait(bc->expireData(truncVersion, true));
TraceEvent("TruncateMutationLogs").detail("Version", truncVersion).detail("Timestamp", timestamp);
CODE_PROBE(true, "Flush blob granules and truncate mutation logs");
} catch (Error& e) {
TraceEvent("TruncateMutationLogsError").error(e); // skip and retry next time
}
return Void();
}
ACTOR Future<Void> backupManifest(Reference<BlobManagerData> bmData) { ACTOR Future<Void> backupManifest(Reference<BlobManagerData> bmData) {
bmData->initBStore(); bmData->initBStore();
@ -5256,8 +5348,9 @@ ACTOR Future<Void> backupManifest(Reference<BlobManagerData> bmData) {
break; break;
} }
} }
if (activeRanges) { if (activeRanges && SERVER_KNOBS->BLOB_MANIFEST_BACKUP) {
if (bmData->manifestStore.isValid()) { if (bmData->manifestStore.isValid()) {
wait(maybeFlushAndTruncateMutationLogs(bmData));
wait(dumpManifest(bmData->db, bmData->manifestStore, bmData->epoch, bmData->manifestDumperSeqNo)); wait(dumpManifest(bmData->db, bmData->manifestStore, bmData->epoch, bmData->manifestDumperSeqNo));
bmData->manifestDumperSeqNo++; bmData->manifestDumperSeqNo++;
} else { } else {

14
fdbserver/BlobManifest.actor.cpp
View File

@ -260,6 +260,7 @@ public:
self->segmentNo_ = 1; self->segmentNo_ = 1;
self->totalRows_ = 0; self->totalRows_ = 0;
self->logicalSize_ = 0; self->logicalSize_ = 0;
self->totalBytes_ = 0;
wait(waitForAll(self->pendingFutures_)); wait(waitForAll(self->pendingFutures_));
self->pendingFutures_.clear(); self->pendingFutures_.clear();
self->deleteSegmentFiles(); self->deleteSegmentFiles();
@ -999,3 +1000,16 @@ ACTOR Future<Version> getManifestVersion(Database db) {
} }
} }
} }
ACTOR Future<std::string> getMutationLogUrl() {
state std::string baseUrl = SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL;
if (baseUrl.starts_with("file://")) {
state std::vector<std::string> containers = wait(IBackupContainer::listContainers(baseUrl, {}));
if (containers.size() == 0) {
throw blob_restore_missing_logs();
}
return containers.back();
} else {
return baseUrl;
}
}

18
fdbserver/BlobMigrator.actor.cpp
View File

@ -329,27 +329,15 @@ private:
// Apply mutation logs to blob granules so that they reach to a consistent version for all blob granules // Apply mutation logs to blob granules so that they reach to a consistent version for all blob granules
ACTOR static Future<Void> applyMutationLogs(Reference<BlobMigrator> self) { ACTOR static Future<Void> applyMutationLogs(Reference<BlobMigrator> self) {
// check last version in mutation logs // check last version in mutation logs
state std::string mlogsUrl = wait(getMutationLogUrl());
state std::string baseUrl = SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL;
state std::string mlogsUrl;
if (baseUrl.starts_with("file://")) {
state std::vector<std::string> containers = wait(IBackupContainer::listContainers(baseUrl, {}));
if (containers.size() == 0) {
TraceEvent("MissingMutationLogs", self->interf_.id()).detail("Url", baseUrl);
throw restore_missing_data();
}
mlogsUrl = containers.front();
} else {
mlogsUrl = baseUrl;
}
state Reference<IBackupContainer> bc = IBackupContainer::openContainer(mlogsUrl, {}, {}); state Reference<IBackupContainer> bc = IBackupContainer::openContainer(mlogsUrl, {}, {});
BackupDescription desc = wait(bc->describeBackup()); BackupDescription desc = wait(bc->describeBackup());
if (!desc.contiguousLogEnd.present()) { if (!desc.contiguousLogEnd.present()) {
TraceEvent("InvalidMutationLogs").detail("Url", baseUrl); TraceEvent("InvalidMutationLogs").detail("Url", SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL);
throw blob_restore_missing_logs(); throw blob_restore_missing_logs();
} }
if (!desc.minLogBegin.present()) { if (!desc.minLogBegin.present()) {
TraceEvent("InvalidMutationLogs").detail("Url", baseUrl); TraceEvent("InvalidMutationLogs").detail("Url", SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL);
throw blob_restore_missing_logs(); throw blob_restore_missing_logs();
} }
state Version minLogVersion = desc.minLogBegin.get(); state Version minLogVersion = desc.minLogBegin.get();

3
fdbserver/BlobWorker.actor.cpp
View File

@ -37,7 +37,6 @@
#include "fdbclient/Notified.h" #include "fdbclient/Notified.h"
#include "fdbserver/BlobGranuleServerCommon.actor.h" #include "fdbserver/BlobGranuleServerCommon.actor.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbclient/GetEncryptCipherKeys.actor.h" #include "fdbclient/GetEncryptCipherKeys.actor.h"
#include "fdbserver/Knobs.h" #include "fdbserver/Knobs.h"
#include "fdbserver/MutationTracking.h" #include "fdbserver/MutationTracking.h"
@ -1401,6 +1400,7 @@ ACTOR Future<BlobFileIndex> compactFromBlob(Reference<BlobWorkerData> bwData,
snapshotF.offset, snapshotF.offset,
snapshotF.length, snapshotF.length,
snapshotF.fullFileLength, snapshotF.fullFileLength,
snapshotF.version,
snapCipherKeysCtx); snapCipherKeysCtx);
compactBytesRead += snapshotF.length; compactBytesRead += snapshotF.length;
@ -1432,6 +1432,7 @@ ACTOR Future<BlobFileIndex> compactFromBlob(Reference<BlobWorkerData> bwData,
deltaF.offset, deltaF.offset,
deltaF.length, deltaF.length,
deltaF.fullFileLength, deltaF.fullFileLength,
deltaF.version,
deltaCipherKeysCtx); deltaCipherKeysCtx);
compactBytesRead += deltaF.length; compactBytesRead += deltaF.length;
lastDeltaVersion = files.deltaFiles[deltaIdx].version; lastDeltaVersion = files.deltaFiles[deltaIdx].version;

1
fdbserver/ClusterRecovery.actor.cpp
View File

@ -24,7 +24,6 @@
#include "fdbserver/ApplyMetadataMutation.h" #include "fdbserver/ApplyMetadataMutation.h"
#include "fdbserver/BackupProgress.actor.h" #include "fdbserver/BackupProgress.actor.h"
#include "fdbserver/ClusterRecovery.actor.h" #include "fdbserver/ClusterRecovery.actor.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/Knobs.h" #include "fdbserver/Knobs.h"
#include "fdbserver/MasterInterface.h" #include "fdbserver/MasterInterface.h"
#include "fdbserver/WaitFailure.h" #include "fdbserver/WaitFailure.h"

21
fdbserver/CommitProxyServer.actor.cpp
View File

@ -42,7 +42,6 @@
#include "fdbserver/ApplyMetadataMutation.h" #include "fdbserver/ApplyMetadataMutation.h"
#include "fdbserver/ConflictSet.h" #include "fdbserver/ConflictSet.h"
#include "fdbserver/DataDistributorInterface.h" #include "fdbserver/DataDistributorInterface.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/FDBExecHelper.actor.h" #include "fdbserver/FDBExecHelper.actor.h"
#include "fdbclient/GetEncryptCipherKeys.actor.h" #include "fdbclient/GetEncryptCipherKeys.actor.h"
#include "fdbserver/IKeyValueStore.h" #include "fdbserver/IKeyValueStore.h"
@ -1691,6 +1690,8 @@ ACTOR Future<WriteMutationRefVar> writeMutationEncryptedMutation(CommitBatchCont
Arena* arena) { Arena* arena) {
state MutationRef encryptedMutation = encryptedMutationOpt->get(); state MutationRef encryptedMutation = encryptedMutationOpt->get();
state const BlobCipherEncryptHeader* header; state const BlobCipherEncryptHeader* header;
state BlobCipherEncryptHeaderRef headerRef;
state MutationRef decryptedMutation;
static_assert(TenantInfo::INVALID_TENANT == INVALID_ENCRYPT_DOMAIN_ID); static_assert(TenantInfo::INVALID_TENANT == INVALID_ENCRYPT_DOMAIN_ID);
ASSERT(self->pProxyCommitData->encryptMode.isEncryptionEnabled()); ASSERT(self->pProxyCommitData->encryptMode.isEncryptionEnabled());
@ -1698,9 +1699,15 @@ ACTOR Future<WriteMutationRefVar> writeMutationEncryptedMutation(CommitBatchCont
ASSERT(encryptedMutation.isEncrypted()); ASSERT(encryptedMutation.isEncrypted());
Reference<AsyncVar<ServerDBInfo> const> dbInfo = self->pProxyCommitData->db; Reference<AsyncVar<ServerDBInfo> const> dbInfo = self->pProxyCommitData->db;
header = encryptedMutation.encryptionHeader(); if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, *header, BlobCipherMetrics::TLOG)); headerRef = encryptedMutation.configurableEncryptionHeader();
MutationRef decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG); TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, headerRef, BlobCipherMetrics::TLOG));
decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG);
} else {
header = encryptedMutation.encryptionHeader();
TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, *header, BlobCipherMetrics::TLOG));
decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG);
}
ASSERT(decryptedMutation.type == mutation->type); ASSERT(decryptedMutation.type == mutation->type);
ASSERT(decryptedMutation.param1 == mutation->param1); ASSERT(decryptedMutation.param1 == mutation->param1);
@ -2668,7 +2675,7 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
ssis.push_back(it->interf); ssis.push_back(it->interf);
maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id());
} }
rep.results.emplace_back(r.range(), ssis); rep.results.emplace_back(TenantAPI::clampRangeToTenant(r.range(), req.tenant, req.arena), ssis);
} else if (!req.reverse) { } else if (!req.reverse) {
int count = 0; int count = 0;
for (auto r = commitData->keyInfo.rangeContaining(req.begin); for (auto r = commitData->keyInfo.rangeContaining(req.begin);
@ -2680,7 +2687,7 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
ssis.push_back(it->interf); ssis.push_back(it->interf);
maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id());
} }
rep.results.emplace_back(r.range(), ssis); rep.results.emplace_back(TenantAPI::clampRangeToTenant(r.range(), req.tenant, req.arena), ssis);
count++; count++;
} }
} else { } else {
@ -2693,7 +2700,7 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
ssis.push_back(it->interf); ssis.push_back(it->interf);
maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id());
} }
rep.results.emplace_back(r.range(), ssis); rep.results.emplace_back(TenantAPI::clampRangeToTenant(r.range(), req.tenant, req.arena), ssis);
if (r == commitData->keyInfo.ranges().begin()) { if (r == commitData->keyInfo.ranges().begin()) {
break; break;
} }

172
fdbserver/KeyValueStoreMemory.actor.cpp
View File

@ -35,6 +35,8 @@
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
#define OP_DISK_OVERHEAD (sizeof(OpHeader) + 1) #define OP_DISK_OVERHEAD (sizeof(OpHeader) + 1)
#define ENCRYPTION_ENABLED_BIT 31
static_assert(sizeof(uint32_t) == 4);
template <typename Container> template <typename Container>
class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable { class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
@ -308,7 +310,7 @@ private:
OpCommit, // only in log, not in queue OpCommit, // only in log, not in queue
OpRollback, // only in log, not in queue OpRollback, // only in log, not in queue
OpSnapshotItemDelta, OpSnapshotItemDelta,
OpEncrypted OpEncrypted_Deprecated // deprecated since we now store the encryption status in the first bit of the opType
}; };
struct OpRef { struct OpRef {
@ -319,7 +321,7 @@ private:
size_t expectedSize() const { return p1.expectedSize() + p2.expectedSize(); } size_t expectedSize() const { return p1.expectedSize() + p2.expectedSize(); }
}; };
struct OpHeader { struct OpHeader {
int op; uint32_t op;
int len1, len2; int len1, len2;
}; };
@ -462,39 +464,56 @@ private:
return total; return total;
} }
// Data format for normal operation: static bool isOpEncrypted(OpHeader* header) { return header->op >> ENCRYPTION_ENABLED_BIT == 1; }
// +-------------+-------------+-------------+--------+--------+
// | opType | len1 | len2 | param2 | param2 | static void setEncryptFlag(OpHeader* header, bool set) {
// | sizeof(int) | sizeof(int) | sizeof(int) | len1 | len2 | if (set) {
// +-------------+-------------+-------------+--------+--------+ header->op |= (1UL << ENCRYPTION_ENABLED_BIT);
} else {
header->op &= ~(1UL << ENCRYPTION_ENABLED_BIT);
}
}
// NOTE: The first bit of opType indicates whether the entry is encrypted or not. This is fine for backwards
// compatability since the first bit was never used previously
// //
// However, if the operation is encrypted: // Unencrypted data format:
// +-------------+-------------+-------------+---------------------------------+-------------+--------+--------+ // +-------------+-------------+-------------+--------+--------+-----------+
// | OpEncrypted | len1 | len2 | BlobCipherEncryptHeader | opType | param1 | param2 | // | opType | len1 | len2 | param2 | param2 | \x01 |
// | sizeof(int) | sizeof(int) | sizeof(int) | sizeof(BlobCipherEncryptHeader) | sizeof(int) | len1 | len2 | // | sizeof(int) | sizeof(int) | sizeof(int) | len1 | len2 | 1 byte |
// +-------------+-------------+-------------+---------------------------------+-------------+--------+--------+ // +-------------+-------------+-------------+--------+--------+-----------+
// | plaintext | encrypted | //
// +-----------------------------------------------------------------------------------------------------------+ // Encrypted data format:
// +-------------+-------+---------+------------+-----------------------------+--------+--------+------------+
// | opType |len1 | len2 | headerSize | BlobCipherEncryptHeader | param1 | param2 | \x01 |
// | s(uint32) | s(int)| s(int) | s(uint16) | s(BlobCipherEncryptHeader) | len1 | len2 | 1 byte |
// +-------------+-------+---------+------------+-----------------------------+--------+--------+------------+
// | plaintext | encrypted | |
// +--------------------------------------------------------------------------+-----------------+------------+
// //
IDiskQueue::location log_op(OpType op, StringRef v1, StringRef v2) { IDiskQueue::location log_op(OpType op, StringRef v1, StringRef v2) {
// Metadata op types to be excluded from encryption. // Metadata op types to be excluded from encryption.
static std::unordered_set<OpType> metaOps = { OpSnapshotEnd, OpSnapshotAbort, OpCommit, OpRollback }; static std::unordered_set<OpType> metaOps = { OpSnapshotEnd, OpSnapshotAbort, OpCommit, OpRollback };
uint32_t opType = (uint32_t)op;
// Make sure the first bit of the optype is empty
ASSERT(opType >> ENCRYPTION_ENABLED_BIT == 0);
if (!enableEncryption || metaOps.count(op) > 0) { if (!enableEncryption || metaOps.count(op) > 0) {
OpHeader h = { (int)op, v1.size(), v2.size() }; OpHeader h = { opType, v1.size(), v2.size() };
log->push(StringRef((const uint8_t*)&h, sizeof(h))); log->push(StringRef((const uint8_t*)&h, sizeof(h)));
log->push(v1); log->push(v1);
log->push(v2); log->push(v2);
} else { } else {
OpHeader h = { (int)OpEncrypted, v1.size(), v2.size() }; OpHeader h = { opType, v1.size(), v2.size() };
// Set the first bit of the header to 1 to indicate that the log entry is encrypted
setEncryptFlag(&h, true);
log->push(StringRef((const uint8_t*)&h, sizeof(h))); log->push(StringRef((const uint8_t*)&h, sizeof(h)));
uint8_t* plaintext = new uint8_t[sizeof(int) + v1.size() + v2.size()]; uint8_t* plaintext = new uint8_t[v1.size() + v2.size()];
*(int*)plaintext = op;
if (v1.size()) { if (v1.size()) {
memcpy(plaintext + sizeof(int), v1.begin(), v1.size()); memcpy(plaintext, v1.begin(), v1.size());
} }
if (v2.size()) { if (v2.size()) {
memcpy(plaintext + sizeof(int) + v1.size(), v2.begin(), v2.size()); memcpy(plaintext + v1.size(), v2.begin(), v2.size());
} }
ASSERT(cipherKeys.cipherTextKey.isValid()); ASSERT(cipherKeys.cipherTextKey.isValid());
@ -504,12 +523,28 @@ private:
cipherKeys.cipherHeaderKey, cipherKeys.cipherHeaderKey,
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE), getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
BlobCipherMetrics::KV_MEMORY); BlobCipherMetrics::KV_MEMORY);
BlobCipherEncryptHeader cipherHeader; uint16_t encryptHeaderSize;
// TODO: If possible we want to avoid memcpy to the disk log by using the same arena used by IDiskQueue
Arena arena; Arena arena;
StringRef ciphertext = if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
cipher.encrypt(plaintext, sizeof(int) + v1.size() + v2.size(), &cipherHeader, arena)->toStringRef(); BlobCipherEncryptHeaderRef headerRef;
log->push(StringRef((const uint8_t*)&cipherHeader, BlobCipherEncryptHeader::headerSize)); StringRef cipherText = cipher.encrypt(plaintext, v1.size() + v2.size(), &headerRef, arena);
log->push(ciphertext); Standalone<StringRef> headerRefStr = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
encryptHeaderSize = headerRefStr.size();
ASSERT(encryptHeaderSize > 0);
log->push(StringRef((const uint8_t*)&encryptHeaderSize, sizeof(encryptHeaderSize)));
log->push(headerRefStr);
log->push(cipherText);
} else {
BlobCipherEncryptHeader cipherHeader;
StringRef ciphertext =
cipher.encrypt(plaintext, v1.size() + v2.size(), &cipherHeader, arena)->toStringRef();
encryptHeaderSize = BlobCipherEncryptHeader::headerSize;
ASSERT(encryptHeaderSize > 0);
log->push(StringRef((const uint8_t*)&encryptHeaderSize, sizeof(encryptHeaderSize)));
log->push(StringRef((const uint8_t*)&cipherHeader, encryptHeaderSize));
log->push(ciphertext);
}
} }
return log->push("\x01"_sr); // Changes here should be reflected in OP_DISK_OVERHEAD return log->push("\x01"_sr); // Changes here should be reflected in OP_DISK_OVERHEAD
} }
@ -517,19 +552,38 @@ private:
// In case the op data is not encrypted, simply read the operands and the zero fill flag. // In case the op data is not encrypted, simply read the operands and the zero fill flag.
// Otherwise, decrypt the op type and data. // Otherwise, decrypt the op type and data.
ACTOR static Future<Standalone<StringRef>> readOpData(KeyValueStoreMemory* self, ACTOR static Future<Standalone<StringRef>> readOpData(KeyValueStoreMemory* self,
OpHeader* h, OpHeader h,
bool* isZeroFilled, bool* isZeroFilled,
int* zeroFillSize) { int* zeroFillSize,
bool encryptedOp) {
ASSERT(!isOpEncrypted(&h));
// Metadata op types to be excluded from encryption. // Metadata op types to be excluded from encryption.
static std::unordered_set<OpType> metaOps = { OpSnapshotEnd, OpSnapshotAbort, OpCommit, OpRollback }; static std::unordered_set<OpType> metaOps = { OpSnapshotEnd, OpSnapshotAbort, OpCommit, OpRollback };
if (metaOps.count((OpType)h->op) == 0) { if (metaOps.count((OpType)h.op) == 0) {
// It is not supported to open an encrypted store as unencrypted, or vice-versa. // It is not supported to open an encrypted store as unencrypted, or vice-versa.
ASSERT_EQ(h->op == OpEncrypted, self->enableEncryption); ASSERT_EQ(encryptedOp, self->enableEncryption);
} }
state int remainingBytes = h->len1 + h->len2 + 1; // if encrypted op read the header size
if (h->op == OpEncrypted) { state uint16_t encryptHeaderSize = 0;
if (encryptedOp) {
state Standalone<StringRef> headerSizeStr = wait(self->log->readNext(sizeof(encryptHeaderSize)));
ASSERT(headerSizeStr.size() <= sizeof(encryptHeaderSize));
// Partial read on the header size
memset(&encryptHeaderSize, 0, sizeof(encryptHeaderSize));
memcpy(&encryptHeaderSize, headerSizeStr.begin(), headerSizeStr.size());
if (headerSizeStr.size() < sizeof(encryptHeaderSize)) {
CODE_PROBE(true, "zero fill partial encryption header size", probe::decoration::rare);
*zeroFillSize =
(sizeof(encryptHeaderSize) - headerSizeStr.size()) + encryptHeaderSize + h.len1 + h.len2 + 1;
}
if (*zeroFillSize > 0) {
return headerSizeStr;
}
}
state int remainingBytes = h.len1 + h.len2 + 1;
if (encryptedOp) {
// encryption header, plus the real (encrypted) op type // encryption header, plus the real (encrypted) op type
remainingBytes += BlobCipherEncryptHeader::headerSize + sizeof(int); remainingBytes += encryptHeaderSize;
} }
state Standalone<StringRef> data = wait(self->log->readNext(remainingBytes)); state Standalone<StringRef> data = wait(self->log->readNext(remainingBytes));
ASSERT(data.size() <= remainingBytes); ASSERT(data.size() <= remainingBytes);
@ -537,23 +591,31 @@ private:
if (*zeroFillSize == 0) { if (*zeroFillSize == 0) {
*isZeroFilled = (data[data.size() - 1] == 0); *isZeroFilled = (data[data.size() - 1] == 0);
} }
if (h->op != OpEncrypted || *zeroFillSize > 0 || *isZeroFilled) { if (!encryptedOp || *zeroFillSize > 0 || *isZeroFilled) {
return data; return data;
} }
state BlobCipherEncryptHeader cipherHeader = *(BlobCipherEncryptHeader*)data.begin(); state Arena arena;
TextAndHeaderCipherKeys cipherKeys = state StringRef plaintext;
wait(getEncryptCipherKeys(self->db, cipherHeader, BlobCipherMetrics::KV_MEMORY)); if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
DecryptBlobCipherAes256Ctr cipher( state BlobCipherEncryptHeaderRef cipherHeaderRef =
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, cipherHeader.iv, BlobCipherMetrics::KV_MEMORY); BlobCipherEncryptHeaderRef::fromStringRef(StringRef(data.begin(), encryptHeaderSize));
Arena arena; TextAndHeaderCipherKeys cipherKeys =
StringRef plaintext = cipher wait(getEncryptCipherKeys(self->db, cipherHeaderRef, BlobCipherMetrics::KV_MEMORY));
.decrypt(data.begin() + BlobCipherEncryptHeader::headerSize, DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey,
sizeof(int) + h->len1 + h->len2, cipherKeys.cipherHeaderKey,
cipherHeader, cipherHeaderRef.getIV(),
arena) BlobCipherMetrics::KV_MEMORY);
->toStringRef(); plaintext = cipher.decrypt(data.begin() + encryptHeaderSize, h.len1 + h.len2, cipherHeaderRef, arena);
h->op = *(int*)plaintext.begin(); } else {
return Standalone<StringRef>(plaintext.substr(sizeof(int)), arena); state BlobCipherEncryptHeader cipherHeader = *(BlobCipherEncryptHeader*)data.begin();
TextAndHeaderCipherKeys cipherKeys =
wait(getEncryptCipherKeys(self->db, cipherHeader, BlobCipherMetrics::KV_MEMORY));
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, cipherHeader.iv, BlobCipherMetrics::KV_MEMORY);
plaintext =
cipher.decrypt(data.begin() + encryptHeaderSize, h.len1 + h.len2, cipherHeader, arena)->toStringRef();
}
return Standalone<StringRef>(plaintext, arena);
} }
ACTOR static Future<Void> recover(KeyValueStoreMemory* self, bool exactRecovery) { ACTOR static Future<Void> recover(KeyValueStoreMemory* self, bool exactRecovery) {
@ -586,6 +648,7 @@ private:
try { try {
loop { loop {
state bool encryptedOp = false;
{ {
Standalone<StringRef> data = wait(self->log->readNext(sizeof(OpHeader))); Standalone<StringRef> data = wait(self->log->readNext(sizeof(OpHeader)));
if (data.size() != sizeof(OpHeader)) { if (data.size() != sizeof(OpHeader)) {
@ -595,9 +658,11 @@ private:
memset(&h, 0, sizeof(OpHeader)); memset(&h, 0, sizeof(OpHeader));
memcpy(&h, data.begin(), data.size()); memcpy(&h, data.begin(), data.size());
zeroFillSize = sizeof(OpHeader) - data.size() + h.len1 + h.len2 + 1; zeroFillSize = sizeof(OpHeader) - data.size() + h.len1 + h.len2 + 1;
if (h.op == OpEncrypted) { if (isOpEncrypted(&h)) {
// encryption header, plus the real (encrypted) op type // encrypt header size + encryption header
zeroFillSize += BlobCipherEncryptHeader::headerSize + sizeof(int); // If it's a partial header we assume the header size is 0 (this is fine since we
// don't read the header in this case)
zeroFillSize += 0 + sizeof(uint16_t);
} }
} }
TraceEvent("KVSMemRecoveryComplete", self->id) TraceEvent("KVSMemRecoveryComplete", self->id)
@ -609,8 +674,13 @@ private:
break; break;
} }
h = *(OpHeader*)data.begin(); h = *(OpHeader*)data.begin();
encryptedOp = isOpEncrypted(&h);
// Reset the first bit to 0 so the op can be read properly
setEncryptFlag(&h, false);
ASSERT(h.op != OpEncrypted_Deprecated);
} }
state Standalone<StringRef> data = wait(readOpData(self, &h, &isZeroFilled, &zeroFillSize)); state Standalone<StringRef> data =
wait(readOpData(self, h, &isZeroFilled, &zeroFillSize, encryptedOp));
if (zeroFillSize > 0) { if (zeroFillSize > 0) {
TraceEvent("KVSMemRecoveryComplete", self->id) TraceEvent("KVSMemRecoveryComplete", self->id)
.detail("Reason", "data specified by header does not exist") .detail("Reason", "data specified by header does not exist")

4
fdbserver/RESTKmsConnector.actor.cpp
View File

@ -778,10 +778,10 @@ Future<T> kmsRequestImpl(Reference<RESTKmsConnectorCtx> ctx,
ACTOR Future<Void> fetchEncryptionKeysByKeyIds(Reference<RESTKmsConnectorCtx> ctx, KmsConnLookupEKsByKeyIdsReq req) { ACTOR Future<Void> fetchEncryptionKeysByKeyIds(Reference<RESTKmsConnectorCtx> ctx, KmsConnLookupEKsByKeyIdsReq req) {
state KmsConnLookupEKsByKeyIdsRep reply; state KmsConnLookupEKsByKeyIdsRep reply;
try { try {
bool refreshKmsUrls = shouldRefreshKmsUrls(ctx); bool refreshKmsUrls = shouldRefreshKmsUrls(ctx);
StringRef requestBodyRef = getEncryptKeysByKeyIdsRequestBody(ctx, req, refreshKmsUrls, req.arena); StringRef requestBodyRef = getEncryptKeysByKeyIdsRequestBody(ctx, req, refreshKmsUrls, req.arena);
std::function<Standalone<VectorRef<EncryptCipherKeyDetailsRef>>(Reference<RESTKmsConnectorCtx>, std::function<Standalone<VectorRef<EncryptCipherKeyDetailsRef>>(Reference<RESTKmsConnectorCtx>,
Reference<HTTP::Response>)> Reference<HTTP::Response>)>
f = &parseEncryptCipherResponse; f = &parseEncryptCipherResponse;
@ -1060,11 +1060,13 @@ ACTOR Future<Void> procureValidationTokensFromFiles(Reference<RESTKmsConnectorCt
ACTOR Future<Void> procureValidationTokens(Reference<RESTKmsConnectorCtx> ctx) { ACTOR Future<Void> procureValidationTokens(Reference<RESTKmsConnectorCtx> ctx) {
std::string_view mode{ SERVER_KNOBS->REST_KMS_CONNECTOR_VALIDATION_TOKEN_MODE }; std::string_view mode{ SERVER_KNOBS->REST_KMS_CONNECTOR_VALIDATION_TOKEN_MODE };
TraceEvent("ProcureValidationTokensStart", ctx->uid);
if (mode.compare("file") == 0) { if (mode.compare("file") == 0) {
wait(procureValidationTokensFromFiles(ctx, SERVER_KNOBS->REST_KMS_CONNECTOR_VALIDATION_TOKEN_DETAILS)); wait(procureValidationTokensFromFiles(ctx, SERVER_KNOBS->REST_KMS_CONNECTOR_VALIDATION_TOKEN_DETAILS));
} else { } else {
throw not_implemented(); throw not_implemented();
} }
TraceEvent("ProcureValidationTokensDone", ctx->uid);
return Void(); return Void();
} }

1
fdbserver/Resolver.actor.cpp
View File

@ -26,7 +26,6 @@
#include "fdbclient/SystemData.h" #include "fdbclient/SystemData.h"
#include "fdbserver/ApplyMetadataMutation.h" #include "fdbserver/ApplyMetadataMutation.h"
#include "fdbserver/ConflictSet.h" #include "fdbserver/ConflictSet.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/IKeyValueStore.h" #include "fdbserver/IKeyValueStore.h"
#include "fdbserver/Knobs.h" #include "fdbserver/Knobs.h"
#include "fdbserver/LogSystem.h" #include "fdbserver/LogSystem.h"

7
fdbserver/RestoreLoader.actor.cpp
View File

@ -372,13 +372,10 @@ void handleRestoreSysInfoRequest(const RestoreSysInfoRequest& req, Reference<Res
ACTOR static Future<MutationRef> _decryptMutation(MutationRef mutation, Database cx, Arena* arena) { ACTOR static Future<MutationRef> _decryptMutation(MutationRef mutation, Database cx, Arena* arena) {
ASSERT(mutation.isEncrypted()); ASSERT(mutation.isEncrypted());
Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo; Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
state const BlobCipherEncryptHeader* header = mutation.encryptionHeader();
std::unordered_set<BlobCipherDetails> cipherDetails; std::unordered_set<BlobCipherDetails> cipherDetails;
cipherDetails.insert(header->cipherTextDetails); mutation.updateEncryptCipherDetails(cipherDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>> getCipherKeysResult = std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>> getCipherKeysResult =
wait(getEncryptCipherKeys(dbInfo, cipherDetails, BlobCipherMetrics::BACKUP)); wait(getEncryptCipherKeys(dbInfo, cipherDetails, BlobCipherMetrics::BACKUP));
return mutation.decrypt(getCipherKeysResult, *arena, BlobCipherMetrics::BACKUP); return mutation.decrypt(getCipherKeysResult, *arena, BlobCipherMetrics::BACKUP);

3
fdbserver/SimKmsConnector.actor.cpp
View File

@ -134,6 +134,9 @@ ACTOR Future<Void> ekLookupByIds(Reference<SimKmsConnectorContext> ctx,
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId.get(), itr->first), ""); getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId.get(), itr->first), "");
} }
} else { } else {
TraceEvent("SimKmsEKLookupByIdsKeyNotFound")
.detail("DomId", item.domainId)
.detail("BaseCipherId", item.baseCipherId);
success = false; success = false;
break; break;
} }

110
fdbserver/SimulatedCluster.actor.cpp
View File

@ -332,9 +332,6 @@ class TestConfig : public BasicTestConfig {
if (attrib == "disableRemoteKVS") { if (attrib == "disableRemoteKVS") {
disableRemoteKVS = strcmp(value.c_str(), "true") == 0; disableRemoteKVS = strcmp(value.c_str(), "true") == 0;
} }
if (attrib == "disableEncryption") {
disableEncryption = strcmp(value.c_str(), "true") == 0;
}
if (attrib == "encryptModes") { if (attrib == "encryptModes") {
std::stringstream ss(value); std::stringstream ss(value);
std::string token; std::string token;
@ -410,9 +407,6 @@ public:
bool disableHostname = false; bool disableHostname = false;
// remote key value store is a child process spawned by the SS process to run the storage engine // remote key value store is a child process spawned by the SS process to run the storage engine
bool disableRemoteKVS = false; bool disableRemoteKVS = false;
// 7.2 cannot be downgraded to 7.1 or below after enabling encryption-at-rest.
// TODO: Remove this bool once the encryption knobs are removed
bool disableEncryption = false;
// By default, encryption mode is set randomly (based on the tenant mode) // By default, encryption mode is set randomly (based on the tenant mode)
// If provided, set using EncryptionAtRestMode::fromString // If provided, set using EncryptionAtRestMode::fromString
std::vector<std::string> encryptModes; std::vector<std::string> encryptModes;
@ -493,7 +487,6 @@ public:
.add("disableTss", &disableTss) .add("disableTss", &disableTss)
.add("disableHostname", &disableHostname) .add("disableHostname", &disableHostname)
.add("disableRemoteKVS", &disableRemoteKVS) .add("disableRemoteKVS", &disableRemoteKVS)
.add("disableEncryption", &disableEncryption)
.add("encryptModes", &encryptModes) .add("encryptModes", &encryptModes)
.add("simpleConfig", &simpleConfig) .add("simpleConfig", &simpleConfig)
.add("generateFearless", &generateFearless) .add("generateFearless", &generateFearless)
@ -561,6 +554,11 @@ public:
} }
} }
bool excludedStorageEngineType(int storageEngineType) const {
return std::find(storageEngineExcludeTypes.begin(), storageEngineExcludeTypes.end(), storageEngineType) !=
storageEngineExcludeTypes.end();
}
TestConfig() = default; TestConfig() = default;
TestConfig(const BasicTestConfig& config) : BasicTestConfig(config) {} TestConfig(const BasicTestConfig& config) : BasicTestConfig(config) {}
}; };
@ -1296,14 +1294,6 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
g_knobs.setKnob("remote_kv_store", KnobValueRef::create(bool{ false })); g_knobs.setKnob("remote_kv_store", KnobValueRef::create(bool{ false }));
TraceEvent(SevDebug, "DisableRemoteKVS"); TraceEvent(SevDebug, "DisableRemoteKVS");
} }
// TODO: Remove this code when encryption knobs are removed
if (testConfig->disableEncryption) {
g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_tlog_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_storage_server_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_blob_granule_encryption", KnobValueRef::create(bool{ false }));
TraceEvent(SevDebug, "DisableEncryption");
}
*pConnString = conn; *pConnString = conn;
*pTesterCount = testerCount; *pTesterCount = testerCount;
bool usingSSL = conn.toString().find(":tls") != std::string::npos || listenersPerProcess > 1; bool usingSSL = conn.toString().find(":tls") != std::string::npos || listenersPerProcess > 1;
@ -1596,35 +1586,54 @@ void SimulationConfig::setTenantMode(const TestConfig& testConfig) {
} }
void SimulationConfig::setEncryptionAtRestMode(const TestConfig& testConfig) { void SimulationConfig::setEncryptionAtRestMode(const TestConfig& testConfig) {
EncryptionAtRestMode encryptionMode = EncryptionAtRestMode::DISABLED; std::vector<bool> available;
// Only Redwood support encryption. Disable encryption if non-Redwood storage engine is explicitly specified. std::vector<double> probability;
bool disableEncryption = testConfig.disableEncryption || if (!testConfig.encryptModes.empty()) {
(testConfig.storageEngineType.present() && testConfig.storageEngineType.get() != 3); // If encryptModes are specified explicitly, give them equal probability to be chosen.
// TODO: Remove check on the ENABLE_ENCRYPTION knob once the EKP can start using the db config available = std::vector<bool>(EncryptionAtRestMode::END, false);
if (!disableEncryption && (SERVER_KNOBS->ENABLE_ENCRYPTION || !testConfig.encryptModes.empty())) { probability = std::vector<double>(EncryptionAtRestMode::END, 0);
TenantMode tenantMode = db.tenantMode; for (auto& mode : testConfig.encryptModes) {
if (!testConfig.encryptModes.empty()) { available[EncryptionAtRestMode::fromString(mode).mode] = true;
std::vector<EncryptionAtRestMode> validEncryptModes; probability[EncryptionAtRestMode::fromString(mode).mode] = 1.0 / testConfig.encryptModes.size();
// Get the subset of valid encrypt modes given the tenant mode }
for (int i = 0; i < testConfig.encryptModes.size(); i++) { } else {
EncryptionAtRestMode encryptMode = EncryptionAtRestMode::fromString(testConfig.encryptModes.at(i)); // If encryptModes are not specified, give encryption higher chance to be enabled.
if (encryptMode != EncryptionAtRestMode::DOMAIN_AWARE || tenantMode == TenantMode::REQUIRED) { // The good thing is testing with encryption on doesn't loss test coverage for most of the other features.
validEncryptModes.push_back(encryptMode); available = std::vector<bool>(EncryptionAtRestMode::END, true);
} probability = { 0.25, 0.5, 0.25 };
} // Only Redwood support encryption. Disable encryption if Redwood is not available.
if (validEncryptModes.size() > 0) { if ((testConfig.storageEngineType.present() && testConfig.storageEngineType != 3) ||
encryptionMode = deterministicRandom()->randomChoice(validEncryptModes); testConfig.excludedStorageEngineType(3)) {
} available[(int)EncryptionAtRestMode::DOMAIN_AWARE] = false;
} else { available[(int)EncryptionAtRestMode::CLUSTER_AWARE] = false;
// TODO: These cases should only trigger with probability (BUGGIFY) once the server knob is removed
if (tenantMode == TenantMode::DISABLED || tenantMode == TenantMode::OPTIONAL_TENANT || BUGGIFY) {
// optional and disabled tenant modes currently only support cluster aware encryption
encryptionMode = EncryptionAtRestMode::CLUSTER_AWARE;
} else {
encryptionMode = EncryptionAtRestMode::DOMAIN_AWARE;
}
} }
} }
// domain_aware mode is supported only with required tenant mode.
if (db.tenantMode != TenantMode::REQUIRED) {
available[(int)EncryptionAtRestMode::DOMAIN_AWARE] = false;
}
int lastAvailableMode = EncryptionAtRestMode::END;
double totalProbability = 0;
for (int mode = 0; mode < (int)EncryptionAtRestMode::END; mode++) {
if (available[mode]) {
lastAvailableMode = mode;
totalProbability += probability[mode];
}
}
ASSERT(lastAvailableMode != EncryptionAtRestMode::END); // At least one mode available
double r = deterministicRandom()->random01() * totalProbability;
EncryptionAtRestMode encryptionMode;
for (int mode = 0;; mode++) {
if (available[mode] && (r < probability[mode] || mode == lastAvailableMode)) {
encryptionMode = (EncryptionAtRestMode::Mode)mode;
break;
}
r -= probability[mode];
}
TraceEvent("SimulatedClusterEncryptionMode").detail("Mode", encryptionMode.toString());
CODE_PROBE(encryptionMode == EncryptionAtRestMode::DISABLED, "Disabled encryption in simulation");
CODE_PROBE(encryptionMode == EncryptionAtRestMode::CLUSTER_AWARE, "Enabled cluster-aware encryption in simulation");
CODE_PROBE(encryptionMode == EncryptionAtRestMode::DOMAIN_AWARE, "Enabled domain-aware encryption in simulation");
set_config("encryption_at_rest_mode=" + encryptionMode.toString()); set_config("encryption_at_rest_mode=" + encryptionMode.toString());
} }
@ -1640,9 +1649,7 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
storage_engine_type = testConfig.storageEngineType.get(); storage_engine_type = testConfig.storageEngineType.get();
} else { } else {
// Continuously re-pick the storage engine type if it's the one we want to exclude // Continuously re-pick the storage engine type if it's the one we want to exclude
while (std::find(testConfig.storageEngineExcludeTypes.begin(), while (testConfig.excludedStorageEngineType(storage_engine_type)) {
testConfig.storageEngineExcludeTypes.end(),
storage_engine_type) != testConfig.storageEngineExcludeTypes.end()) {
storage_engine_type = deterministicRandom()->randomInt(0, 6); storage_engine_type = deterministicRandom()->randomInt(0, 6);
} }
} }
@ -2132,19 +2139,6 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
simconfig.db.tLogWriteAntiQuorum = testConfig.logAntiQuorum; simconfig.db.tLogWriteAntiQuorum = testConfig.logAntiQuorum;
} }
// TODO: remove knob hanlding once we move off from encrypption knobs to db config
if (simconfig.db.encryptionAtRestMode.mode == EncryptionAtRestMode::DISABLED) {
g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ false }));
CODE_PROBE(true, "Disabled encryption in simulation");
} else {
g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ true }));
CODE_PROBE(simconfig.db.encryptionAtRestMode.mode == EncryptionAtRestMode::CLUSTER_AWARE,
"Enabled cluster-aware encryption in simulation");
CODE_PROBE(simconfig.db.encryptionAtRestMode.mode == EncryptionAtRestMode::DOMAIN_AWARE,
"Enabled domain-aware encryption in simulation");
}
TraceEvent("SimulatedClusterEncryptionMode").detail("Mode", simconfig.db.encryptionAtRestMode.toString());
g_simulator->blobGranulesEnabled = simconfig.db.blobGranulesEnabled; g_simulator->blobGranulesEnabled = simconfig.db.blobGranulesEnabled;
StatusObject startingConfigJSON = simconfig.db.toJSON(true); StatusObject startingConfigJSON = simconfig.db.toJSON(true);

2
fdbserver/Status.actor.cpp
View File

@ -854,7 +854,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
roles.addRole("consistency_scan", db->get().consistencyScan.get()); roles.addRole("consistency_scan", db->get().consistencyScan.get());
} }
if (SERVER_KNOBS->ENABLE_ENCRYPTION && db->get().encryptKeyProxy.present()) { if (db->get().encryptKeyProxy.present()) {
roles.addRole("encrypt_key_proxy", db->get().encryptKeyProxy.get()); roles.addRole("encrypt_key_proxy", db->get().encryptKeyProxy.get());
} }

6
fdbserver/StorageCache.actor.cpp
View File

@ -1925,11 +1925,7 @@ ACTOR Future<Void> pullAsyncData(StorageCacheData* data) {
cloneReader >> msg; cloneReader >> msg;
if (msg.isEncrypted()) { if (msg.isEncrypted()) {
if (!cipherKeys.present()) { if (!cipherKeys.present()) {
const BlobCipherEncryptHeader* header = msg.encryptionHeader(); msg.updateEncryptCipherDetails(cipherDetails);
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
collectingCipherKeys = true; collectingCipherKeys = true;
} else { } else {
msg = msg.decrypt(cipherKeys.get(), cloneReader.arena(), BlobCipherMetrics::TLOG); msg = msg.decrypt(cipherKeys.get(), cloneReader.arena(), BlobCipherMetrics::TLOG);

10
fdbserver/TLogServer.actor.cpp
View File

@ -2437,15 +2437,7 @@ ACTOR Future<EncryptionAtRestMode> getEncryptionAtRestMode(TLogData* self) {
when(GetEncryptionAtRestModeResponse resp = wait(brokenPromiseToNever( when(GetEncryptionAtRestModeResponse resp = wait(brokenPromiseToNever(
self->dbInfo->get().clusterInterface.getEncryptionAtRestMode.getReply(req)))) { self->dbInfo->get().clusterInterface.getEncryptionAtRestMode.getReply(req)))) {
TraceEvent("GetEncryptionAtRestMode", self->dbgid).detail("Mode", resp.mode); TraceEvent("GetEncryptionAtRestMode", self->dbgid).detail("Mode", resp.mode);
return (EncryptionAtRestMode::Mode)resp.mode;
// TODO: TLOG_ENCTYPTION KNOB shall be removed and db-config check should be sufficient to
// determine tlog (and cluster) encryption status
if ((EncryptionAtRestMode::Mode)resp.mode != EncryptionAtRestMode::Mode::DISABLED &&
SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION) {
return EncryptionAtRestMode((EncryptionAtRestMode::Mode)resp.mode);
} else {
return EncryptionAtRestMode();
}
} }
} }
} catch (Error& e) { } catch (Error& e) {

53
fdbserver/VersionedBTree.actor.cpp
View File

@ -2815,6 +2815,13 @@ public:
wait(self->keyProviderInitialized.getFuture()); wait(self->keyProviderInitialized.getFuture());
ASSERT(self->keyProvider.isValid()); ASSERT(self->keyProvider.isValid());
} }
if (self->keyProvider->expectedEncodingType() != page->getEncodingType()) {
TraceEvent(SevWarnAlways, "RedwoodBTreeUnexpectedNodeEncoding")
.detail("PhysicalPageID", page->getPhysicalPageID())
.detail("EncodingTypeFound", page->getEncodingType())
.detail("EncodingTypeExpected", self->keyProvider->expectedEncodingType());
throw unexpected_encoding_type();
}
ArenaPage::EncryptionKey k = wait(self->keyProvider->getEncryptionKey(page->getEncodingHeader())); ArenaPage::EncryptionKey k = wait(self->keyProvider->getEncryptionKey(page->getEncodingHeader()));
page->encryptionKey = k; page->encryptionKey = k;
} }
@ -2883,6 +2890,17 @@ public:
try { try {
page->postReadHeader(pageIDs.front()); page->postReadHeader(pageIDs.front());
if (page->isEncrypted()) { if (page->isEncrypted()) {
if (!self->keyProvider.isValid()) {
wait(self->keyProviderInitialized.getFuture());
ASSERT(self->keyProvider.isValid());
}
if (self->keyProvider->expectedEncodingType() != page->getEncodingType()) {
TraceEvent(SevWarnAlways, "RedwoodBTreeUnexpectedNodeEncoding")
.detail("PhysicalPageID", page->getPhysicalPageID())
.detail("EncodingTypeFound", page->getEncodingType())
.detail("EncodingTypeExpected", self->keyProvider->expectedEncodingType());
throw unexpected_encoding_type();
}
ArenaPage::EncryptionKey k = wait(self->keyProvider->getEncryptionKey(page->getEncodingHeader())); ArenaPage::EncryptionKey k = wait(self->keyProvider->getEncryptionKey(page->getEncodingHeader()));
page->encryptionKey = k; page->encryptionKey = k;
} }
@ -3595,7 +3613,7 @@ public:
// The next section explicitly cancels all pending operations held in the pager // The next section explicitly cancels all pending operations held in the pager
debug_printf("DWALPager(%s) shutdown kill ioLock\n", self->filename.c_str()); debug_printf("DWALPager(%s) shutdown kill ioLock\n", self->filename.c_str());
self->ioLock->kill(); self->ioLock->halt();
debug_printf("DWALPager(%s) shutdown cancel recovery\n", self->filename.c_str()); debug_printf("DWALPager(%s) shutdown cancel recovery\n", self->filename.c_str());
self->recoverFuture.cancel(); self->recoverFuture.cancel();
@ -3657,7 +3675,14 @@ public:
} else { } else {
g_network->getDiskBytes(parentDirectory(filename), free, total); g_network->getDiskBytes(parentDirectory(filename), free, total);
} }
int64_t pagerSize = header.pageCount * physicalPageSize;
// Size of redwood data file. Note that filePageCountPending is used here instead of filePageCount. This is
// because it is always >= filePageCount and accounts for file size changes which will complete soon.
int64_t pagerPhysicalSize = filePageCountPending * physicalPageSize;
// Size of the pager, which can be less than the data file size. All pages within this size are either in use
// in a data structure or accounted for in one of the pager's free page lists.
int64_t pagerLogicalSize = header.pageCount * physicalPageSize;
// It is not exactly known how many pages on the delayed free list are usable as of right now. It could be // It is not exactly known how many pages on the delayed free list are usable as of right now. It could be
// known, if each commit delayed entries that were freeable were shuffled from the delayed free queue to the // known, if each commit delayed entries that were freeable were shuffled from the delayed free queue to the
@ -3668,12 +3693,17 @@ public:
// Amount of space taken up by the free list queues themselves, as if we were to pop and use // Amount of space taken up by the free list queues themselves, as if we were to pop and use
// items on the free lists the space the items are stored in would also become usable // items on the free lists the space the items are stored in would also become usable
int64_t reusableQueueSpace = (freeList.numPages + delayedFreeList.numPages) * physicalPageSize; int64_t reusableQueueSpace = (freeList.numPages + delayedFreeList.numPages) * physicalPageSize;
int64_t reusable = reusablePageSpace + reusableQueueSpace;
// Pager slack is the space at the end of the pager's logical size until the end of the pager file's size.
// These pages will be used if needed without growing the file size.
int64_t reusablePagerSlackSpace = pagerPhysicalSize - pagerLogicalSize;
int64_t reusable = reusablePageSpace + reusableQueueSpace + reusablePagerSlackSpace;
// Space currently in used by old page versions have have not yet been freed due to the remap cleanup window. // Space currently in used by old page versions have have not yet been freed due to the remap cleanup window.
int64_t temp = remapQueue.numEntries * physicalPageSize; int64_t temp = remapQueue.numEntries * physicalPageSize;
return StorageBytes(free, total, pagerSize - reusable, free + reusable, temp); return StorageBytes(free, total, pagerPhysicalSize, free + reusable, temp);
} }
int64_t getPageCacheCount() override { return pageCache.getCount(); } int64_t getPageCacheCount() override { return pageCache.getCount(); }
@ -5255,6 +5285,7 @@ public:
.detail("InstanceName", self->m_pager->getName()) .detail("InstanceName", self->m_pager->getName())
.detail("UsingEncodingType", self->m_encodingType) .detail("UsingEncodingType", self->m_encodingType)
.detail("ExistingEncodingType", self->m_header.encodingType); .detail("ExistingEncodingType", self->m_header.encodingType);
throw unexpected_encoding_type();
} }
// Verify if encryption mode and encoding type in the header are consistent. // Verify if encryption mode and encoding type in the header are consistent.
// This check can also fail in case of authentication mode mismatch. // This check can also fail in case of authentication mode mismatch.
@ -6231,18 +6262,6 @@ private:
metrics.pageRead += 1; metrics.pageRead += 1;
metrics.pageReadExt += (id.size() - 1); metrics.pageReadExt += (id.size() - 1);
// If BTree encryption is enabled, pages read must be encrypted using the desired encryption type
if (self->m_enforceEncodingType && (page->getEncodingType() != self->m_encodingType)) {
Error e = unexpected_encoding_type();
TraceEvent(SevWarnAlways, "RedwoodBTreeUnexpectedNodeEncoding")
.error(e)
.detail("PhysicalPageID", page->getPhysicalPageID())
.detail("IsEncrypted", page->isEncrypted())
.detail("EncodingTypeFound", page->getEncodingType())
.detail("EncodingTypeExpected", self->m_encodingType);
throw e;
}
return std::move(page); return std::move(page);
} }
@ -11448,8 +11467,8 @@ TEST_CASE("/redwood/correctness/EnforceEncodingType") {
{}, // encryptionMode {}, // encryptionMode
reopenEncodingType, reopenEncodingType,
encryptionKeyProviders.at(reopenEncodingType)); encryptionKeyProviders.at(reopenEncodingType));
wait(kvs->init());
try { try {
wait(kvs->init());
Optional<Value> v = wait(kvs->readValue("foo"_sr)); Optional<Value> v = wait(kvs->readValue("foo"_sr));
UNREACHABLE(); UNREACHABLE();
} catch (Error& e) { } catch (Error& e) {

13
fdbserver/fdbserver.actor.cpp
View File

@ -2255,14 +2255,6 @@ int main(int argc, char* argv[]) {
} }
} }
} }
g_knobs.setKnob("enable_encryption",
KnobValue::create(ini.GetBoolValue("META", "enableEncryption", false)));
g_knobs.setKnob("enable_tlog_encryption",
KnobValue::create(ini.GetBoolValue("META", "enableTLogEncryption", false)));
g_knobs.setKnob("enable_storage_server_encryption",
KnobValue::create(ini.GetBoolValue("META", "enableStorageServerEncryption", false)));
g_knobs.setKnob("enable_blob_granule_encryption",
KnobValue::create(ini.GetBoolValue("META", "enableBlobGranuleEncryption", false)));
g_knobs.setKnob("enable_blob_granule_compression", g_knobs.setKnob("enable_blob_granule_compression",
KnobValue::create(ini.GetBoolValue("META", "enableBlobGranuleEncryption", false))); KnobValue::create(ini.GetBoolValue("META", "enableBlobGranuleEncryption", false)));
g_knobs.setKnob("encrypt_header_auth_token_enabled", g_knobs.setKnob("encrypt_header_auth_token_enabled",
@ -2270,6 +2262,11 @@ int main(int argc, char* argv[]) {
g_knobs.setKnob("encrypt_header_auth_token_algo", g_knobs.setKnob("encrypt_header_auth_token_algo",
KnobValue::create((int)ini.GetLongValue( KnobValue::create((int)ini.GetLongValue(
"META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO))); "META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO)));
g_knobs.setKnob("enable_configurable_encryption",
KnobValue::create(ini.GetBoolValue("META",
"enableConfigurableEncryption",
CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION)));
g_knobs.setKnob( g_knobs.setKnob(
"shard_encode_location_metadata", "shard_encode_location_metadata",
KnobValue::create(ini.GetBoolValue("META", "enableShardEncodeLocationMetadata", false))); KnobValue::create(ini.GetBoolValue("META", "enableShardEncodeLocationMetadata", false)));

1
fdbserver/include/fdbserver/BlobGranuleServerCommon.actor.h
View File

@ -171,6 +171,7 @@ ACTOR Future<Optional<BlobRestoreStatus>> getRestoreStatus(Database db, KeyRange
ACTOR Future<Optional<BlobRestoreArg>> getRestoreArg(Database db, KeyRangeRef range); ACTOR Future<Optional<BlobRestoreArg>> getRestoreArg(Database db, KeyRangeRef range);
ACTOR Future<Version> getRestoreTargetVersion(Database db, KeyRangeRef range, Version defaultVersion); ACTOR Future<Version> getRestoreTargetVersion(Database db, KeyRangeRef range, Version defaultVersion);
ACTOR Future<Version> getManifestVersion(Database db); ACTOR Future<Version> getManifestVersion(Database db);
ACTOR Future<std::string> getMutationLogUrl();
#include "flow/unactorcompiler.h" #include "flow/unactorcompiler.h"
#endif #endif

51
fdbserver/include/fdbserver/EncryptionOpsUtils.h
View File

@ -1,51 +0,0 @@
/*
* EncryptionOpUtils.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBSERVER_ENCRYPTION_OPS_UTIL_H
#define FDBSERVER_ENCRYPTION_OPS_UTIL_H
#pragma once
#include "fdbserver/Knobs.h"
#include "fdbclient/CommitProxyInterface.h"
typedef enum { TLOG_ENCRYPTION = 0, STORAGE_SERVER_ENCRYPTION = 1, BLOB_GRANULE_ENCRYPTION = 2 } EncryptOperationType;
inline bool isEncryptionOpSupported(EncryptOperationType operation_type) {
// We would check against dbInfo.isEncryptionEnabled instead, but the dbInfo may not be available before
// ClusterController broadcast the dbInfo to workers. Before the broadcast encryption may appear to be disabled
// when it should be enabled. Moving the encryption switch to DB config could fix the issue.
if (!SERVER_KNOBS->ENABLE_ENCRYPTION) {
return false;
}
if (operation_type == TLOG_ENCRYPTION) {
return SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION;
} else if (operation_type == STORAGE_SERVER_ENCRYPTION) {
return SERVER_KNOBS->ENABLE_STORAGE_SERVER_ENCRYPTION;
} else if (operation_type == BLOB_GRANULE_ENCRYPTION) {
bool supported = SERVER_KNOBS->ENABLE_BLOB_GRANULE_ENCRYPTION && SERVER_KNOBS->BG_METADATA_SOURCE == "tenant";
ASSERT((supported && SERVER_KNOBS->ENABLE_ENCRYPTION) || !supported);
return supported;
} else {
return false;
}
}
#endif // FDBSERVER_ENCRYPTION_OPS_UTIL_H

81
fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h
View File

@ -18,6 +18,7 @@
* limitations under the License. * limitations under the License.
*/ */
#include "fdbclient/Knobs.h"
#include "fdbclient/TenantManagement.actor.h" #include "fdbclient/TenantManagement.actor.h"
#include "fdbrpc/TenantInfo.h" #include "fdbrpc/TenantInfo.h"
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_IPAGEENCRYPTIONKEYPROVIDER_ACTOR_G_H) #if defined(NO_INTELLISENSE) && !defined(FDBSERVER_IPAGEENCRYPTIONKEYPROVIDER_ACTOR_G_H)
@ -31,7 +32,6 @@
#include "fdbclient/SystemData.h" #include "fdbclient/SystemData.h"
#include "fdbclient/Tenant.h" #include "fdbclient/Tenant.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/IPager.h" #include "fdbserver/IPager.h"
#include "fdbserver/Knobs.h" #include "fdbserver/Knobs.h"
#include "fdbserver/ServerDBInfo.h" #include "fdbserver/ServerDBInfo.h"
@ -44,6 +44,7 @@
#include <functional> #include <functional>
#include <limits> #include <limits>
#include <tuple> #include <tuple>
#include <type_traits>
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
@ -158,6 +159,22 @@ public:
uint8_t xorWith; uint8_t xorWith;
}; };
namespace {
template <EncodingType encodingType>
int64_t getEncryptionDomainIdFromAesEncryptionHeader(const void* encodingHeader) {
using Encoder = typename ArenaPage::AESEncryptionEncoder<encodingType>;
using EncodingHeader = typename Encoder::Header;
ASSERT(encodingHeader != nullptr);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
return headerRef.getCipherDetails().textCipherDetails.encryptDomainId;
} else {
const BlobCipherEncryptHeader& header = reinterpret_cast<const EncodingHeader*>(encodingHeader)->encryption;
return header.cipherTextDetails.encryptDomainId;
}
}
} // anonymous namespace
// Key provider to provider cipher keys randomly from a pre-generated pool. It does not maintain encryption domains. // Key provider to provider cipher keys randomly from a pre-generated pool. It does not maintain encryption domains.
// Use for testing. // Use for testing.
template <EncodingType encodingType, template <EncodingType encodingType,
@ -189,22 +206,37 @@ public:
bool enableEncryptionDomain() const override { return mode > 0; } bool enableEncryptionDomain() const override { return mode > 0; }
Future<EncryptionKey> getEncryptionKey(const void* encodingHeader) override { Future<EncryptionKey> getEncryptionKey(const void* encodingHeader) override {
using Header = typename ArenaPage::AESEncryptionEncoder<encodingType>::Header; using Encoder = typename ArenaPage::AESEncryptionEncoder<encodingType>;
const Header* h = reinterpret_cast<const Header*>(encodingHeader);
EncryptionKey s; EncryptionKey s;
s.aesKey.cipherTextKey = if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
getCipherKey(h->encryption.cipherTextDetails.encryptDomainId, h->encryption.cipherTextDetails.baseCipherId); const BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
s.aesKey.cipherHeaderKey = getCipherKey(h->encryption.cipherHeaderDetails.encryptDomainId, EncryptHeaderCipherDetails details = headerRef.getCipherDetails();
h->encryption.cipherHeaderDetails.baseCipherId); ASSERT(details.textCipherDetails.isValid());
s.aesKey.cipherTextKey =
getCipherKey(details.textCipherDetails.encryptDomainId, details.textCipherDetails.baseCipherId);
if (details.headerCipherDetails.present()) {
ASSERT(details.headerCipherDetails.get().isValid());
s.aesKey.cipherHeaderKey = getCipherKey(details.headerCipherDetails.get().encryptDomainId,
details.headerCipherDetails.get().baseCipherId);
}
} else {
const typename Encoder::Header* h = reinterpret_cast<const typename Encoder::Header*>(encodingHeader);
s.aesKey.cipherTextKey = getCipherKey(h->encryption.cipherTextDetails.encryptDomainId,
h->encryption.cipherTextDetails.baseCipherId);
if (h->encryption.cipherHeaderDetails.isValid()) {
s.aesKey.cipherHeaderKey = getCipherKey(h->encryption.cipherHeaderDetails.encryptDomainId,
h->encryption.cipherHeaderDetails.baseCipherId);
}
}
return s; return s;
} }
Future<EncryptionKey> getLatestEncryptionKey(int64_t domainId) override { Future<EncryptionKey> getLatestEncryptionKey(int64_t domainId) override {
domainId = checkDomainId(domainId); domainId = checkDomainId(domainId);
EncryptionKey s; EncryptionKey s;
s.aesKey.cipherTextKey = getCipherKey(domainId, deterministicRandom()->randomInt(0, NUM_CIPHER)); s.aesKey.cipherTextKey = getCipherKey(domainId, deterministicRandom()->randomInt(1, NUM_CIPHER + 1));
s.aesKey.cipherHeaderKey = s.aesKey.cipherHeaderKey =
getCipherKey(ENCRYPT_HEADER_DOMAIN_ID, deterministicRandom()->randomInt(0, NUM_CIPHER)); getCipherKey(ENCRYPT_HEADER_DOMAIN_ID, deterministicRandom()->randomInt(1, NUM_CIPHER + 1));
return s; return s;
} }
@ -222,10 +254,7 @@ public:
} }
int64_t getEncryptionDomainIdFromHeader(const void* encodingHeader) override { int64_t getEncryptionDomainIdFromHeader(const void* encodingHeader) override {
ASSERT(encodingHeader != nullptr); return getEncryptionDomainIdFromAesEncryptionHeader<encodingType>(encodingHeader);
using Header = typename ArenaPage::AESEncryptionEncoder<encodingType>::Header;
const Header* h = reinterpret_cast<const Header*>(encodingHeader);
return h->encryption.cipherTextDetails.encryptDomainId;
} }
private: private:
@ -260,11 +289,12 @@ private:
Reference<BlobCipherKey> getCipherKey(EncryptCipherDomainId domainId, EncryptCipherBaseKeyId cipherId) { Reference<BlobCipherKey> getCipherKey(EncryptCipherDomainId domainId, EncryptCipherBaseKeyId cipherId) {
// Create a new cipher key by replacing the domain id. // Create a new cipher key by replacing the domain id.
ASSERT(cipherId > 0 && cipherId <= NUM_CIPHER);
return makeReference<BlobCipherKey>(domainId, return makeReference<BlobCipherKey>(domainId,
cipherId, cipherId,
cipherKeys[cipherId]->rawBaseCipher(), cipherKeys[cipherId - 1]->rawBaseCipher(),
AES_256_KEY_LENGTH, AES_256_KEY_LENGTH,
cipherKeys[cipherId]->getSalt(), cipherKeys[cipherId - 1]->getSalt(),
std::numeric_limits<int64_t>::max() /* refreshAt */, std::numeric_limits<int64_t>::max() /* refreshAt */,
std::numeric_limits<int64_t>::max() /* expireAt */); std::numeric_limits<int64_t>::max() /* expireAt */);
} }
@ -282,7 +312,8 @@ template <EncodingType encodingType,
true> true>
class AESEncryptionKeyProvider : public IPageEncryptionKeyProvider { class AESEncryptionKeyProvider : public IPageEncryptionKeyProvider {
public: public:
using EncodingHeader = typename ArenaPage::AESEncryptionEncoder<encodingType>::Header; using Encoder = typename ArenaPage::AESEncryptionEncoder<encodingType>;
using EncodingHeader = typename Encoder::Header;
const StringRef systemKeysPrefix = systemKeys.begin; const StringRef systemKeysPrefix = systemKeys.begin;
@ -303,9 +334,17 @@ public:
} }
ACTOR static Future<EncryptionKey> getEncryptionKey(AESEncryptionKeyProvider* self, const void* encodingHeader) { ACTOR static Future<EncryptionKey> getEncryptionKey(AESEncryptionKeyProvider* self, const void* encodingHeader) {
const BlobCipherEncryptHeader& header = reinterpret_cast<const EncodingHeader*>(encodingHeader)->encryption; state TextAndHeaderCipherKeys cipherKeys;
TextAndHeaderCipherKeys cipherKeys = if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
wait(getEncryptCipherKeys(self->db, header, BlobCipherMetrics::KV_REDWOOD)); BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
TextAndHeaderCipherKeys cks =
wait(getEncryptCipherKeys(self->db, headerRef, BlobCipherMetrics::KV_REDWOOD));
cipherKeys = cks;
} else {
const BlobCipherEncryptHeader& header = reinterpret_cast<const EncodingHeader*>(encodingHeader)->encryption;
TextAndHeaderCipherKeys cks = wait(getEncryptCipherKeys(self->db, header, BlobCipherMetrics::KV_REDWOOD));
cipherKeys = cks;
}
EncryptionKey encryptionKey; EncryptionKey encryptionKey;
encryptionKey.aesKey = cipherKeys; encryptionKey.aesKey = cipherKeys;
return encryptionKey; return encryptionKey;
@ -355,9 +394,7 @@ public:
} }
int64_t getEncryptionDomainIdFromHeader(const void* encodingHeader) override { int64_t getEncryptionDomainIdFromHeader(const void* encodingHeader) override {
ASSERT(encodingHeader != nullptr); return getEncryptionDomainIdFromAesEncryptionHeader<encodingType>(encodingHeader);
const BlobCipherEncryptHeader& header = reinterpret_cast<const EncodingHeader*>(encodingHeader)->encryption;
return header.cipherTextDetails.encryptDomainId;
} }
private: private:

61
fdbserver/include/fdbserver/IPager.h
View File

@ -19,6 +19,7 @@
*/ */
#pragma once #pragma once
#include "fdbclient/Knobs.h"
#ifndef FDBSERVER_IPAGER_H #ifndef FDBSERVER_IPAGER_H
#define FDBSERVER_IPAGER_H #define FDBSERVER_IPAGER_H
@ -387,23 +388,40 @@ public:
// By default, xxhash is used to checksum the page. But ff authentication is enabled (such as when we are using // By default, xxhash is used to checksum the page. But ff authentication is enabled (such as when we are using
// aes256-ctr-hmac-sha256 encryption scheme), the auth tag plays the role of a checksum while assuring authenticity // aes256-ctr-hmac-sha256 encryption scheme), the auth tag plays the role of a checksum while assuring authenticity
// of the data. xxhash checksum is not needed in this case. // of the data. xxhash checksum is not needed in this case.
//
// To support configurable encryption, which may come with variable size encryption header, we assume the encryption
// header size is no larger than that of BlobCipherEncryptHeader. This is true for current supported encryption
// header format types. Moving forward, the plan is to make IPager support variable size encoding header, and let
// Redwood rebuild a page when it tries to in-place update the page, but the reserved buffer for the encoding header
// is not large enough.
// TODO(yiwu): Cleanup the old encryption header, and update headerSize to be the maximum size of the supported
// encryption header format type.
// TODO(yiwu): Support variable size encoding header.
template <EncodingType encodingType, template <EncodingType encodingType,
typename std::enable_if<encodingType == AESEncryption || encodingType == AESEncryptionWithAuth, typename std::enable_if<encodingType == AESEncryption || encodingType == AESEncryptionWithAuth,
bool>::type = true> bool>::type = true>
struct AESEncryptionEncoder { struct AESEncryptionEncoder {
struct AESEncryptionEncodingHeader { struct AESEncryptionEncodingHeader {
BlobCipherEncryptHeader encryption;
XXH64_hash_t checksum; XXH64_hash_t checksum;
union {
BlobCipherEncryptHeader encryption;
uint8_t encryptionHeaderBuf[0]; // for configurable encryption
};
}; };
struct AESEncryptionWithAuthEncodingHeader { struct AESEncryptionWithAuthEncodingHeader {
BlobCipherEncryptHeader encryption; union {
BlobCipherEncryptHeader encryption;
uint8_t encryptionHeaderBuf[0]; // for configurable encryption
};
}; };
using Header = typename std::conditional<encodingType == AESEncryption, using Header = typename std::conditional<encodingType == AESEncryption,
AESEncryptionEncodingHeader, AESEncryptionEncodingHeader,
AESEncryptionWithAuthEncodingHeader>::type; AESEncryptionWithAuthEncodingHeader>::type;
static constexpr size_t headerSize = sizeof(Header);
static void encode(void* header, static void encode(void* header,
const TextAndHeaderCipherKeys& cipherKeys, const TextAndHeaderCipherKeys& cipherKeys,
uint8_t* payload, uint8_t* payload,
@ -415,7 +433,19 @@ public:
getEncryptAuthTokenMode(ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE), getEncryptAuthTokenMode(ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
BlobCipherMetrics::KV_REDWOOD); BlobCipherMetrics::KV_REDWOOD);
Arena arena; Arena arena;
StringRef ciphertext = cipher.encrypt(payload, len, &h->encryption, arena)->toStringRef(); StringRef ciphertext;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef;
ciphertext = cipher.encrypt(payload, len, &headerRef, arena);
Standalone<StringRef> serializedHeader = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
ASSERT(serializedHeader.size() <= headerSize);
memcpy(h->encryptionHeaderBuf, serializedHeader.begin(), serializedHeader.size());
if (serializedHeader.size() < headerSize) {
memset(h->encryptionHeaderBuf + serializedHeader.size(), 0, headerSize - serializedHeader.size());
}
} else {
ciphertext = cipher.encrypt(payload, len, &h->encryption, arena)->toStringRef();
}
ASSERT_EQ(len, ciphertext.size()); ASSERT_EQ(len, ciphertext.size());
memcpy(payload, ciphertext.begin(), len); memcpy(payload, ciphertext.begin(), len);
if constexpr (encodingType == AESEncryption) { if constexpr (encodingType == AESEncryption) {
@ -423,6 +453,13 @@ public:
} }
} }
static BlobCipherEncryptHeaderRef getEncryptionHeaderRef(const void* header) {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
const Header* h = reinterpret_cast<const Header*>(header);
return BlobCipherEncryptHeaderRef::fromStringRef(
StringRef(h->encryptionHeaderBuf, headerSize - (h->encryptionHeaderBuf - (const uint8_t*)h)));
}
static void decode(void* header, static void decode(void* header,
const TextAndHeaderCipherKeys& cipherKeys, const TextAndHeaderCipherKeys& cipherKeys,
uint8_t* payload, uint8_t* payload,
@ -434,10 +471,22 @@ public:
throw page_decoding_failed(); throw page_decoding_failed();
} }
} }
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, h->encryption.iv, BlobCipherMetrics::KV_REDWOOD);
Arena arena; Arena arena;
StringRef plaintext = cipher.decrypt(payload, len, h->encryption, arena)->toStringRef(); StringRef plaintext;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef = getEncryptionHeaderRef(header);
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey,
cipherKeys.cipherHeaderKey,
headerRef.getIV(),
BlobCipherMetrics::KV_REDWOOD);
plaintext = cipher.decrypt(payload, len, headerRef, arena);
} else {
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey,
cipherKeys.cipherHeaderKey,
h->encryption.iv,
BlobCipherMetrics::KV_REDWOOD);
plaintext = cipher.decrypt(payload, len, h->encryption, arena)->toStringRef();
}
ASSERT_EQ(len, plaintext.size()); ASSERT_EQ(len, plaintext.size());
memcpy(payload, plaintext.begin(), len); memcpy(payload, plaintext.begin(), len);
} }

2
fdbserver/include/fdbserver/ProxyCommitData.actor.h
View File

@ -19,8 +19,6 @@
*/ */
#pragma once #pragma once
#include "fdbserver/EncryptionOpsUtils.h"
#include <unordered_map>
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H) #if defined(NO_INTELLISENSE) && !defined(FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H)
#define FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H #define FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H
#include "fdbserver/ProxyCommitData.actor.g.h" #include "fdbserver/ProxyCommitData.actor.g.h"

91
fdbserver/storageserver.actor.cpp
View File

@ -159,6 +159,7 @@ bool canReplyWith(Error e) {
return false; return false;
} }
} }
} // namespace } // namespace
#define PERSIST_PREFIX "\xff\xff" #define PERSIST_PREFIX "\xff\xff"
@ -854,7 +855,6 @@ public:
void clearTenants(StringRef startTenant, StringRef endTenant, Version version); void clearTenants(StringRef startTenant, StringRef endTenant, Version version);
void checkTenantEntry(Version version, TenantInfo tenant); void checkTenantEntry(Version version, TenantInfo tenant);
KeyRangeRef clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena);
std::vector<StorageServerShard> getStorageServerShards(KeyRangeRef range); std::vector<StorageServerShard> getStorageServerShards(KeyRangeRef range);
@ -2078,6 +2078,7 @@ ACTOR Future<Version> waitForMinVersion(StorageServer* data, Version version) {
void StorageServer::checkTenantEntry(Version version, TenantInfo tenantInfo) { void StorageServer::checkTenantEntry(Version version, TenantInfo tenantInfo) {
if (tenantInfo.hasTenant()) { if (tenantInfo.hasTenant()) {
ASSERT(version == latestVersion || (version >= tenantMap.oldestVersion && version <= this->version.get()));
auto view = tenantMap.at(version); auto view = tenantMap.at(version);
auto itr = view.find(tenantInfo.tenantId); auto itr = view.find(tenantInfo.tenantId);
if (itr == view.end()) { if (itr == view.end()) {
@ -3020,11 +3021,7 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
if (doFilterMutations || !req.encrypted) { if (doFilterMutations || !req.encrypted) {
for (auto& m : decodedMutations.back().first) { for (auto& m : decodedMutations.back().first) {
if (m.isEncrypted()) { if (m.isEncrypted()) {
const BlobCipherEncryptHeader* header = m.encryptionHeader(); m.updateEncryptCipherDetails(cipherDetails);
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
} }
} }
} }
@ -4038,17 +4035,6 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
return result; return result;
} }
KeyRangeRef StorageServer::clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena) {
if (tenantInfo.hasTenant()) {
return KeyRangeRef(range.begin.startsWith(tenantInfo.prefix.get()) ? range.begin : tenantInfo.prefix.get(),
range.end.startsWith(tenantInfo.prefix.get())
? range.end
: allKeys.end.withPrefix(tenantInfo.prefix.get(), arena));
} else {
return range;
}
}
ACTOR Future<Key> findKey(StorageServer* data, ACTOR Future<Key> findKey(StorageServer* data,
KeySelectorRef sel, KeySelectorRef sel,
Version version, Version version,
@ -4250,7 +4236,7 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
throw wrong_shard_server(); throw wrong_shard_server();
} }
KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena); KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena);
state int offset1 = 0; state int offset1 = 0;
state int offset2; state int offset2;
@ -5415,7 +5401,7 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
throw wrong_shard_server(); throw wrong_shard_server();
} }
KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena); KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena);
state int offset1 = 0; state int offset1 = 0;
state int offset2; state int offset2;
@ -5616,7 +5602,7 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
throw wrong_shard_server(); throw wrong_shard_server();
} }
KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena); KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena);
state int offset1 = 0; state int offset1 = 0;
state int offset2; state int offset2;
@ -5806,7 +5792,7 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
state uint64_t changeCounter = data->shardChangeCounter; state uint64_t changeCounter = data->shardChangeCounter;
KeyRange shard = getShardKeyRange(data, req.sel); KeyRange shard = getShardKeyRange(data, req.sel);
KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena); KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena);
state int offset; state int offset;
Key absoluteKey = wait(findKey(data, req.sel, version, searchRange, &offset, req.spanContext, req.options)); Key absoluteKey = wait(findKey(data, req.sel, version, searchRange, &offset, req.spanContext, req.options));
@ -6854,9 +6840,7 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
data->counters.feedBytesFetched += remoteResult.expectedSize(); data->counters.feedBytesFetched += remoteResult.expectedSize();
data->fetchKeysBytesBudget -= remoteResult.expectedSize(); data->fetchKeysBytesBudget -= remoteResult.expectedSize();
if (data->fetchKeysBytesBudget <= 0) { data->fetchKeysBudgetUsed.set(data->fetchKeysBytesBudget <= 0);
data->fetchKeysBudgetUsed.set(true);
}
wait(yield()); wait(yield());
} }
} catch (Error& e) { } catch (Error& e) {
@ -7574,16 +7558,15 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
metricReporter.addFetchedBytes(expectedBlockSize, this_block.size()); metricReporter.addFetchedBytes(expectedBlockSize, this_block.size());
// Write this_block to storage // Write this_block to storage
state int sinceYield = 0;
state KeyValueRef* kvItr = this_block.begin(); state KeyValueRef* kvItr = this_block.begin();
for (; kvItr != this_block.end(); ++kvItr) { for (; kvItr != this_block.end(); ++kvItr) {
data->storage.writeKeyValue(*kvItr); data->storage.writeKeyValue(*kvItr);
wait(yield());
}
kvItr = this_block.begin();
for (; kvItr != this_block.end(); ++kvItr) {
data->byteSampleApplySet(*kvItr, invalidVersion); data->byteSampleApplySet(*kvItr, invalidVersion);
wait(yield()); if (++sinceYield > 1000) {
wait(yield());
sinceYield = 0;
}
} }
ASSERT(this_block.readThrough.present() || this_block.size()); ASSERT(this_block.readThrough.present() || this_block.size());
@ -7592,9 +7575,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
this_block = RangeResult(); this_block = RangeResult();
data->fetchKeysBytesBudget -= expectedBlockSize; data->fetchKeysBytesBudget -= expectedBlockSize;
if (data->fetchKeysBytesBudget <= 0) { data->fetchKeysBudgetUsed.set(data->fetchKeysBytesBudget <= 0);
data->fetchKeysBudgetUsed.set(true);
}
} }
} catch (Error& e) { } catch (Error& e) {
if (e.code() != error_code_end_of_stream && e.code() != error_code_connection_failed && if (e.code() != error_code_end_of_stream && e.code() != error_code_connection_failed &&
@ -9257,11 +9238,7 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
} }
if (msg.isEncrypted()) { if (msg.isEncrypted()) {
if (!cipherKeys.present()) { if (!cipherKeys.present()) {
const BlobCipherEncryptHeader* header = msg.encryptionHeader(); msg.updateEncryptCipherDetails(cipherDetails);
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
collectingCipherKeys = true; collectingCipherKeys = true;
} else { } else {
msg = msg.decrypt(cipherKeys.get(), eager.arena, BlobCipherMetrics::TLOG); msg = msg.decrypt(cipherKeys.get(), eager.arena, BlobCipherMetrics::TLOG);
@ -9668,6 +9645,8 @@ ACTOR Future<Void> createCheckpoint(StorageServer* data, CheckpointMetaData meta
ACTOR Future<Void> updateStorage(StorageServer* data) { ACTOR Future<Void> updateStorage(StorageServer* data) {
state UnlimitedCommitBytes unlimitedCommitBytes = UnlimitedCommitBytes::False; state UnlimitedCommitBytes unlimitedCommitBytes = UnlimitedCommitBytes::False;
state Future<Void> durableDelay = Void();
loop { loop {
unlimitedCommitBytes = UnlimitedCommitBytes::False; unlimitedCommitBytes = UnlimitedCommitBytes::False;
ASSERT(data->durableVersion.get() == data->storageVersion()); ASSERT(data->durableVersion.get() == data->storageVersion());
@ -9678,7 +9657,19 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
wait(delay(endTime - now(), TaskPriority::UpdateStorage)); wait(delay(endTime - now(), TaskPriority::UpdateStorage));
} }
} }
wait(data->desiredOldestVersion.whenAtLeast(data->storageVersion() + 1));
// If the fetch keys budget is not used up then we have already waited for the storage commit delay so
// wait for either a new mutation version or the budget to be used up.
// Otherwise, don't wait at all.
if (!data->fetchKeysBudgetUsed.get()) {
wait(data->desiredOldestVersion.whenAtLeast(data->storageVersion() + 1) ||
data->fetchKeysBudgetUsed.onChange());
}
// Yield to TaskPriority::UpdateStorage in case more mutations have arrived but were not processed yet.
// If the fetch keys budget has already been used up, then we likely arrived here without waiting the
// full post storage commit delay, so this will allow the update actor to process some mutations
// before we proceed.
wait(delay(0, TaskPriority::UpdateStorage)); wait(delay(0, TaskPriority::UpdateStorage));
state Promise<Void> durableInProgress; state Promise<Void> durableInProgress;
@ -9783,6 +9774,17 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
break; break;
} }
// Allow data fetch to use an additional bytesLeft but don't penalize fetch budget if bytesLeft is negative
if (bytesLeft > 0) {
data->fetchKeysBytesBudget += bytesLeft;
data->fetchKeysBudgetUsed.set(data->fetchKeysBytesBudget <= 0);
// Dependng on how negative the fetchKeys budget was it could still be used up
if (!data->fetchKeysBudgetUsed.get()) {
wait(durableDelay || data->fetchKeysBudgetUsed.onChange());
}
}
if (addedRanges) { if (addedRanges) {
TraceEvent(SevVerbose, "SSAddKVSRangeMetaData", data->thisServerID) TraceEvent(SevVerbose, "SSAddKVSRangeMetaData", data->thisServerID)
.detail("NewDurableVersion", newOldestVersion) .detail("NewDurableVersion", newOldestVersion)
@ -9882,11 +9884,10 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
wait(data->storage.canCommit()); wait(data->storage.canCommit());
state Future<Void> durable = data->storage.commit(); state Future<Void> durable = data->storage.commit();
++data->counters.kvCommits; ++data->counters.kvCommits;
state Future<Void> durableDelay = Void();
if (bytesLeft > 0) { // If the mutation bytes budget was not fully used then wait some time before the next commit
durableDelay = delay(SERVER_KNOBS->STORAGE_COMMIT_INTERVAL, TaskPriority::UpdateStorage); durableDelay =
} (bytesLeft > 0) ? delay(SERVER_KNOBS->STORAGE_COMMIT_INTERVAL, TaskPriority::UpdateStorage) : Void();
wait(ioTimeoutError(durable, SERVER_KNOBS->MAX_STORAGE_COMMIT_TIME, "StorageCommit")); wait(ioTimeoutError(durable, SERVER_KNOBS->MAX_STORAGE_COMMIT_TIME, "StorageCommit"));
data->storageCommitLatencyHistogram->sampleSeconds(now() - beforeStorageCommit); data->storageCommitLatencyHistogram->sampleSeconds(now() - beforeStorageCommit);
@ -11711,7 +11712,7 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
// If the storage server dies while something that uses self is still on the stack, // If the storage server dies while something that uses self is still on the stack,
// we want that actor to complete before we terminate and that memory goes out of scope // we want that actor to complete before we terminate and that memory goes out of scope
self.ssLock->kill(); self.ssLock->halt();
state Error err = e; state Error err = e;
if (storageServerTerminated(self, persistentData, err)) { if (storageServerTerminated(self, persistentData, err)) {
@ -11804,7 +11805,7 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
throw internal_error(); throw internal_error();
} catch (Error& e) { } catch (Error& e) {
self.ssLock->kill(); self.ssLock->halt();
if (self.byteSampleRecovery.isValid()) { if (self.byteSampleRecovery.isValid()) {
self.byteSampleRecovery.cancel(); self.byteSampleRecovery.cancel();

2
fdbserver/tester.actor.cpp
View File

@ -1388,8 +1388,6 @@ std::map<std::string, std::function<void(const std::string&)>> testSpecGlobalKey
[](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableHostname", ""); } }, [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableHostname", ""); } },
{ "disableRemoteKVS", { "disableRemoteKVS",
[](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedRemoteKVS", ""); } }, [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedRemoteKVS", ""); } },
{ "disableEncryption",
[](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedEncryption", ""); } },
{ "allowDefaultTenant", { "allowDefaultTenant",
[](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDefaultTenant", ""); } } [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDefaultTenant", ""); } }
}; };

72
fdbserver/workloads/AuthzSecurity.actor.cpp
View File

@ -54,14 +54,15 @@ struct AuthzSecurityWorkload : TestWorkload {
WipedString signedTokenAnotherTenant; WipedString signedTokenAnotherTenant;
Standalone<StringRef> tLogConfigKey; Standalone<StringRef> tLogConfigKey;
PerfIntCounter crossTenantGetPositive, crossTenantGetNegative, crossTenantCommitPositive, crossTenantCommitNegative, PerfIntCounter crossTenantGetPositive, crossTenantGetNegative, crossTenantCommitPositive, crossTenantCommitNegative,
publicNonTenantRequestPositive, tLogReadNegative; publicNonTenantRequestPositive, tLogReadNegative, keyLocationLeakNegative;
std::vector<std::function<Future<Void>(Database cx)>> testFunctions; std::vector<std::function<Future<Void>(Database cx)>> testFunctions;
AuthzSecurityWorkload(WorkloadContext const& wcx) AuthzSecurityWorkload(WorkloadContext const& wcx)
: TestWorkload(wcx), crossTenantGetPositive("CrossTenantGetPositive"), : TestWorkload(wcx), crossTenantGetPositive("CrossTenantGetPositive"),
crossTenantGetNegative("CrossTenantGetNegative"), crossTenantCommitPositive("CrossTenantCommitPositive"), crossTenantGetNegative("CrossTenantGetNegative"), crossTenantCommitPositive("CrossTenantCommitPositive"),
crossTenantCommitNegative("CrossTenantCommitNegative"), crossTenantCommitNegative("CrossTenantCommitNegative"),
publicNonTenantRequestPositive("PublicNonTenantRequestPositive"), tLogReadNegative("TLogReadNegative") { publicNonTenantRequestPositive("PublicNonTenantRequestPositive"), tLogReadNegative("TLogReadNegative"),
keyLocationLeakNegative("KeyLocationLeakNegative") {
testDuration = getOption(options, "testDuration"_sr, 10.0); testDuration = getOption(options, "testDuration"_sr, 10.0);
transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 500.0) / clientCount; transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 500.0) / clientCount;
actorCount = getOption(options, "actorsPerClient"_sr, transactionsPerSecond / 5); actorCount = getOption(options, "actorsPerClient"_sr, transactionsPerSecond / 5);
@ -81,6 +82,7 @@ struct AuthzSecurityWorkload : TestWorkload {
testFunctions.push_back( testFunctions.push_back(
[this](Database cx) { return testPublicNonTenantRequestsAllowedWithoutTokens(this, cx); }); [this](Database cx) { return testPublicNonTenantRequestsAllowedWithoutTokens(this, cx); });
testFunctions.push_back([this](Database cx) { return testTLogReadDisallowed(this, cx); }); testFunctions.push_back([this](Database cx) { return testTLogReadDisallowed(this, cx); });
testFunctions.push_back([this](Database cx) { return testKeyLocationLeakDisallowed(this, cx); });
} }
Future<Void> setup(Database const& cx) override { Future<Void> setup(Database const& cx) override {
@ -108,7 +110,8 @@ struct AuthzSecurityWorkload : TestWorkload {
clients.clear(); clients.clear();
return errors == 0 && crossTenantGetPositive.getValue() > 0 && crossTenantGetNegative.getValue() > 0 && return errors == 0 && crossTenantGetPositive.getValue() > 0 && crossTenantGetNegative.getValue() > 0 &&
crossTenantCommitPositive.getValue() > 0 && crossTenantCommitNegative.getValue() > 0 && crossTenantCommitPositive.getValue() > 0 && crossTenantCommitNegative.getValue() > 0 &&
publicNonTenantRequestPositive.getValue() > 0 && tLogReadNegative.getValue() > 0; publicNonTenantRequestPositive.getValue() > 0 && tLogReadNegative.getValue() > 0 &&
keyLocationLeakNegative.getValue() > 0;
} }
void getMetrics(std::vector<PerfMetric>& m) override { void getMetrics(std::vector<PerfMetric>& m) override {
@ -118,6 +121,7 @@ struct AuthzSecurityWorkload : TestWorkload {
m.push_back(crossTenantCommitNegative.getMetric()); m.push_back(crossTenantCommitNegative.getMetric());
m.push_back(publicNonTenantRequestPositive.getMetric()); m.push_back(publicNonTenantRequestPositive.getMetric());
m.push_back(tLogReadNegative.getMetric()); m.push_back(tLogReadNegative.getMetric());
m.push_back(keyLocationLeakNegative.getMetric());
} }
void setAuthToken(Transaction& tr, StringRef token) { void setAuthToken(Transaction& tr, StringRef token) {
@ -400,6 +404,68 @@ struct AuthzSecurityWorkload : TestWorkload {
return Void(); return Void();
} }
ACTOR static Future<Void> testKeyLocationLeakDisallowed(AuthzSecurityWorkload* self, Database cx) {
state Key key = self->randomString();
state Value value = self->randomString();
state Version v1 =
wait(setAndCommitKeyValueAndGetVersion(self, cx, self->tenant, self->signedToken, key, value));
state Version v2 = wait(setAndCommitKeyValueAndGetVersion(
self, cx, self->anotherTenant, self->signedTokenAnotherTenant, key, value));
{
GetKeyServerLocationsReply rep =
wait(basicLoadBalance(cx->getCommitProxies(UseProvisionalProxies::False),
&CommitProxyInterface::getKeyServersLocations,
GetKeyServerLocationsRequest(SpanContext(),
TenantInfo(self->tenant->id(), self->signedToken),
key,
Optional<KeyRef>(),
100,
false,
v2,
Arena())));
for (auto const& [range, ssIfaces] : rep.results) {
if (!range.begin.startsWith(self->tenant->prefix())) {
TraceEvent(SevError, "AuthzSecurityKeyRangeLeak")
.detail("TenantId", self->tenant->id())
.detail("LeakingRangeBegin", range.begin.printable());
}
if (!range.end.startsWith(self->tenant->prefix())) {
TraceEvent(SevError, "AuthzSecurityKeyRangeLeak")
.detail("TenantId", self->tenant->id())
.detail("LeakingRangeEnd", range.end.printable());
}
}
}
{
GetKeyServerLocationsReply rep = wait(basicLoadBalance(
cx->getCommitProxies(UseProvisionalProxies::False),
&CommitProxyInterface::getKeyServersLocations,
GetKeyServerLocationsRequest(SpanContext(),
TenantInfo(self->anotherTenant->id(), self->signedTokenAnotherTenant),
key,
Optional<KeyRef>(),
100,
false,
v2,
Arena())));
for (auto const& [range, ssIfaces] : rep.results) {
if (!range.begin.startsWith(self->anotherTenant->prefix())) {
TraceEvent(SevError, "AuthzSecurityKeyRangeLeak")
.detail("TenantId", self->anotherTenant->id())
.detail("LeakingRangeBegin", range.begin.printable());
}
if (!range.end.startsWith(self->anotherTenant->prefix())) {
TraceEvent(SevError, "AuthzSecurityKeyRangeLeak")
.detail("TenantId", self->anotherTenant->id())
.detail("LeakingRangeEnd", range.end.printable());
}
}
}
++self->keyLocationLeakNegative;
return Void();
}
ACTOR static Future<Void> runTestClient(AuthzSecurityWorkload* self, Database cx) { ACTOR static Future<Void> runTestClient(AuthzSecurityWorkload* self, Database cx) {
state double lastTime = now(); state double lastTime = now();
state double delay = self->actorCount / self->transactionsPerSecond; state double delay = self->actorCount / self->transactionsPerSecond;

16
fdbserver/workloads/BlobGranuleRangesWorkload.actor.cpp
View File

@ -553,8 +553,8 @@ struct BlobGranuleRangesWorkload : TestWorkload {
ASSERT(!fail6); ASSERT(!fail6);
bool fail7 = wait(cx->blobbifyRange(KeyRangeRef(middleKey, activeRange.end), self->tenant)); bool fail7 = wait(cx->blobbifyRange(KeyRangeRef(middleKey, activeRange.end), self->tenant));
ASSERT(!fail7);
ASSERT(!fail7);
bool fail8 = wait(cx->blobbifyRange(KeyRangeRef(middleKey, middleKey2), self->tenant)); bool fail8 = wait(cx->blobbifyRange(KeyRangeRef(middleKey, middleKey2), self->tenant));
ASSERT(!fail8); ASSERT(!fail8);
@ -681,6 +681,15 @@ struct BlobGranuleRangesWorkload : TestWorkload {
return Void(); return Void();
} }
ACTOR Future<Void> blobbifyBlockingUnit(Database cx, BlobGranuleRangesWorkload* self, KeyRange range) {
bool setSuccess = wait(cx->blobbifyRangeBlocking(range, self->tenant));
ASSERT(setSuccess);
bool verifySuccess = wait(self->isRangeActive(cx, range, self->tenant));
ASSERT(verifySuccess);
return Void();
}
enum UnitTestTypes { enum UnitTestTypes {
VERIFY_RANGE_UNIT, VERIFY_RANGE_UNIT,
VERIFY_RANGE_GAP_UNIT, VERIFY_RANGE_GAP_UNIT,
@ -688,7 +697,8 @@ struct BlobGranuleRangesWorkload : TestWorkload {
BLOBBIFY_IDEMPOTENT, BLOBBIFY_IDEMPOTENT,
RE_BLOBBIFY, RE_BLOBBIFY,
ADJACENT_PURGE, ADJACENT_PURGE,
OP_COUNT = 6 /* keep this last */ BLOBBIFY_BLOCKING_UNIT,
OP_COUNT = 7 /* keep this last */
}; };
ACTOR Future<Void> blobGranuleRangesUnitTests(Database cx, BlobGranuleRangesWorkload* self) { ACTOR Future<Void> blobGranuleRangesUnitTests(Database cx, BlobGranuleRangesWorkload* self) {
@ -732,6 +742,8 @@ struct BlobGranuleRangesWorkload : TestWorkload {
wait(self->reBlobbifyUnit(cx, self, range)); wait(self->reBlobbifyUnit(cx, self, range));
} else if (op == ADJACENT_PURGE) { } else if (op == ADJACENT_PURGE) {
wait(self->adjacentPurge(cx, self, range)); wait(self->adjacentPurge(cx, self, range));
} else if (op == BLOBBIFY_BLOCKING_UNIT) {
wait(self->blobbifyBlockingUnit(cx, self, range));
} else { } else {
ASSERT(false); ASSERT(false);
} }

22
fdbserver/workloads/BlobGranuleVerifier.actor.cpp
View File

@ -168,27 +168,13 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
} }
} }
// FIXME: run the actual FDBCLI command instead of copy/pasting its implementation
// Sets the whole user keyspace to be blobified // Sets the whole user keyspace to be blobified
ACTOR Future<Void> setUpBlobRange(Database cx) { ACTOR Future<Void> setUpBlobRange(Database cx) {
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx); bool success = wait(cx->blobbifyRange(normalKeys));
loop { ASSERT(success);
try { return Void();
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr->set(blobRangeChangeKey, deterministicRandom()->randomUniqueID().toString());
wait(krmSetRange(tr, blobRangeKeys.begin, KeyRange(normalKeys), "1"_sr));
wait(tr->commit());
if (BGV_DEBUG) {
printf("Successfully set up blob granule range for normalKeys\n");
}
TraceEvent("BlobGranuleVerifierSetup");
return Void();
} catch (Error& e) {
wait(tr->onError(e));
}
}
} }
void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.emplace("Attrition"); } void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.emplace("Attrition"); }
Future<Void> setup(Database const& cx) override { return _setup(cx, this); } Future<Void> setup(Database const& cx) override { return _setup(cx, this); }

8
fdbserver/workloads/BlobRestoreWorkload.actor.cpp
View File

@ -25,7 +25,6 @@
#include "fdbclient/BackupContainer.h" #include "fdbclient/BackupContainer.h"
#include "fdbclient/BackupContainerFileSystem.h" #include "fdbclient/BackupContainerFileSystem.h"
#include "fdbclient/FDBTypes.h" #include "fdbclient/FDBTypes.h"
#include "fdbclient/Knobs.h"
#include "fdbclient/SystemData.h" #include "fdbclient/SystemData.h"
#include "fdbserver/workloads/workloads.actor.h" #include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/BlobGranuleServerCommon.actor.h" #include "fdbserver/BlobGranuleServerCommon.actor.h"
@ -73,6 +72,10 @@ struct BlobRestoreWorkload : TestWorkload {
if (self->performRestore_) { if (self->performRestore_) {
fmt::print("Perform blob restore\n"); fmt::print("Perform blob restore\n");
// disable manifest backup and log truncation
KnobValueRef knobFalse = KnobValueRef::create(bool{ false });
IKnobCollection::getMutableGlobalKnobCollection().setKnob("blob_manifest_backup", knobFalse);
wait(store(result, self->extraDb_->blobRestore(normalKeys, {}))); wait(store(result, self->extraDb_->blobRestore(normalKeys, {})));
state std::vector<Future<Void>> futures; state std::vector<Future<Void>> futures;
@ -178,8 +181,9 @@ struct BlobRestoreWorkload : TestWorkload {
if (src[i].value != dest[i].value) { if (src[i].value != dest[i].value) {
fmt::print("Value mismatch at {}\n", i); fmt::print("Value mismatch at {}\n", i);
TraceEvent(SevError, "TestFailure") TraceEvent(SevError, "TestFailure")
.detail("Reason", "Key Mismatch") .detail("Reason", "Value Mismatch")
.detail("Index", i) .detail("Index", i)
.detail("Key", src[i].key.printable())
.detail("SrcValue", src[i].value.printable()) .detail("SrcValue", src[i].value.printable())
.detail("DestValue", dest[i].value.printable()); .detail("DestValue", dest[i].value.printable());
return false; return false;

26
fdbserver/workloads/EncryptionOps.actor.cpp
View File

@ -120,7 +120,6 @@ struct EncryptionOpsWorkload : TestWorkload {
std::unique_ptr<uint8_t[]> buff; std::unique_ptr<uint8_t[]> buff;
int enableTTLTest; int enableTTLTest;
Arena arena;
std::unique_ptr<WorkloadMetrics> metrics; std::unique_ptr<WorkloadMetrics> metrics;
EncryptCipherDomainId minDomainId; EncryptCipherDomainId minDomainId;
@ -279,7 +278,8 @@ struct EncryptionOpsWorkload : TestWorkload {
int len, int len,
const EncryptAuthTokenMode authMode, const EncryptAuthTokenMode authMode,
const EncryptAuthTokenAlgo authAlgo, const EncryptAuthTokenAlgo authAlgo,
BlobCipherEncryptHeader* header) { BlobCipherEncryptHeader* header,
Arena& arena) {
uint8_t iv[AES_256_IV_LENGTH]; uint8_t iv[AES_256_IV_LENGTH];
deterministicRandom()->randomBytes(&iv[0], AES_256_IV_LENGTH); deterministicRandom()->randomBytes(&iv[0], AES_256_IV_LENGTH);
EncryptBlobCipherAes265Ctr encryptor( EncryptBlobCipherAes265Ctr encryptor(
@ -304,7 +304,8 @@ struct EncryptionOpsWorkload : TestWorkload {
int len, int len,
const EncryptAuthTokenMode authMode, const EncryptAuthTokenMode authMode,
const EncryptAuthTokenAlgo authAlgo, const EncryptAuthTokenAlgo authAlgo,
BlobCipherEncryptHeaderRef* headerRef) { BlobCipherEncryptHeaderRef* headerRef,
Arena& arena) {
uint8_t iv[AES_256_IV_LENGTH]; uint8_t iv[AES_256_IV_LENGTH];
deterministicRandom()->randomBytes(&iv[0], AES_256_IV_LENGTH); deterministicRandom()->randomBytes(&iv[0], AES_256_IV_LENGTH);
EncryptBlobCipherAes265Ctr encryptor( EncryptBlobCipherAes265Ctr encryptor(
@ -345,7 +346,8 @@ struct EncryptionOpsWorkload : TestWorkload {
int len, int len,
const BlobCipherEncryptHeader& header, const BlobCipherEncryptHeader& header,
uint8_t* originalPayload, uint8_t* originalPayload,
Reference<BlobCipherKey> orgCipherKey) { Reference<BlobCipherKey> orgCipherKey,
Arena& arena) {
ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION); ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION);
ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR); ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR);
@ -379,7 +381,8 @@ struct EncryptionOpsWorkload : TestWorkload {
int len, int len,
const Standalone<StringRef>& headerStr, const Standalone<StringRef>& headerStr,
uint8_t* originalPayload, uint8_t* originalPayload,
Reference<BlobCipherKey> orgCipherKey) { Reference<BlobCipherKey> orgCipherKey,
Arena& arena) {
BlobCipherEncryptHeaderRef headerRef = BlobCipherEncryptHeaderRef::fromStringRef(headerStr); BlobCipherEncryptHeaderRef headerRef = BlobCipherEncryptHeaderRef::fromStringRef(headerStr);
ASSERT_EQ(headerRef.flagsVersion(), CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION); ASSERT_EQ(headerRef.flagsVersion(), CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION);
@ -436,6 +439,7 @@ struct EncryptionOpsWorkload : TestWorkload {
setupCipherEssentials(); setupCipherEssentials();
for (int i = 0; i < numIterations; i++) { for (int i = 0; i < numIterations; i++) {
Arena tmpArena;
bool updateBaseCipher = deterministicRandom()->randomInt(1, 100) < 5; bool updateBaseCipher = deterministicRandom()->randomInt(1, 100) < 5;
// Step-1: Encryption key derivation, caching the cipher for later use // Step-1: Encryption key derivation, caching the cipher for later use
@ -482,23 +486,23 @@ struct EncryptionOpsWorkload : TestWorkload {
try { try {
BlobCipherEncryptHeader header; BlobCipherEncryptHeader header;
Reference<EncryptBuf> encrypted = Reference<EncryptBuf> encrypted = doEncryption(
doEncryption(cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &header); cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &header, tmpArena);
// Decrypt the payload - parses the BlobCipherEncryptHeader, fetch corresponding cipherKey and // Decrypt the payload - parses the BlobCipherEncryptHeader, fetch corresponding cipherKey and
// decrypt // decrypt
doDecryption(encrypted, dataLen, header, buff.get(), cipherKey); doDecryption(encrypted, dataLen, header, buff.get(), cipherKey, tmpArena);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef; BlobCipherEncryptHeaderRef headerRef;
StringRef encrypted = StringRef encrypted = doEncryption(
doEncryption(cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &headerRef); cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &headerRef, tmpArena);
// simulate 'header' on-disk read, serialize buffer and deserialize on decryption // simulate 'header' on-disk read, serialize buffer and deserialize on decryption
Standalone<StringRef> serHeader = BlobCipherEncryptHeaderRef::toStringRef(headerRef); Standalone<StringRef> serHeader = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
// Decrypt the payload - parses the BlobCipherEncryptHeader, fetch corresponding cipherKey and // Decrypt the payload - parses the BlobCipherEncryptHeader, fetch corresponding cipherKey and
// decrypt // decrypt
doDecryption(encrypted, dataLen, serHeader, buff.get(), cipherKey); doDecryption(encrypted, dataLen, serHeader, buff.get(), cipherKey, tmpArena);
} }
} catch (Error& e) { } catch (Error& e) {
TraceEvent("Failed") TraceEvent("Failed")

8
fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp
View File

@ -299,22 +299,18 @@ struct MetaclusterManagementWorkload : TestWorkload {
} catch (Error& e) { } catch (Error& e) {
if (e.code() == error_code_conflicting_restore) { if (e.code() == error_code_conflicting_restore) {
ASSERT(retried); ASSERT(retried);
CODE_PROBE(true, "MetaclusterRestore: 2 restores on the same cluster simultaneously"); CODE_PROBE(true, "MetaclusterManagementWorkload: timed out restore conflicts with retried restore");
continue; continue;
} }
throw; throw;
} }
} }
ASSERT(dataDb->registered);
if (!dryRun) {
dataDb->detached = false;
}
} catch (Error& e) { } catch (Error& e) {
if (e.code() == error_code_cluster_not_found) { if (e.code() == error_code_cluster_not_found) {
ASSERT(!dataDb->registered); ASSERT(!dataDb->registered);
return Void(); return Void();
} }
TraceEvent(SevError, "RestoreClusterFailure").error(e).detail("ClusterName", clusterName); TraceEvent(SevError, "RestoreClusterFailure").error(e).detail("ClusterName", clusterName);
ASSERT(false); ASSERT(false);
} }

7
fdbserver/workloads/SaveAndKill.actor.cpp
View File

@ -79,13 +79,8 @@ struct SaveAndKillWorkload : TestWorkload {
if (cx->defaultTenant.present()) { if (cx->defaultTenant.present()) {
ini.SetValue("META", "defaultTenant", cx->defaultTenant.get().toString().c_str()); ini.SetValue("META", "defaultTenant", cx->defaultTenant.get().toString().c_str());
} }
ini.SetBoolValue("META", "enableEncryption", SERVER_KNOBS->ENABLE_ENCRYPTION);
ini.SetBoolValue("META", "enableTLogEncryption", SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION);
ini.SetBoolValue("META", "enableStorageServerEncryption", SERVER_KNOBS->ENABLE_STORAGE_SERVER_ENCRYPTION);
ini.SetBoolValue("META", "enableBlobGranuleEncryption", SERVER_KNOBS->ENABLE_BLOB_GRANULE_ENCRYPTION);
ini.SetBoolValue("META", "enableShardEncodeLocationMetadata", SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA); ini.SetBoolValue("META", "enableShardEncodeLocationMetadata", SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
ini.SetBoolValue("META", "enableConfigurableEncryption", CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
ini.SetBoolValue("META", "encryptHeaderAuthTokenEnabled", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED); ini.SetBoolValue("META", "encryptHeaderAuthTokenEnabled", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED);
ini.SetLongValue("META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO); ini.SetLongValue("META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO);

6
flow/Arena.cpp
View File

@ -22,6 +22,8 @@
#include "flow/UnitTest.h" #include "flow/UnitTest.h"
#include "flow/config.h"
// We don't align memory properly, and we need to tell lsan about that. // We don't align memory properly, and we need to tell lsan about that.
extern "C" const char* __lsan_default_options(void) { extern "C" const char* __lsan_default_options(void) {
return "use_unaligned=1"; return "use_unaligned=1";
@ -998,7 +1000,7 @@ TEST_CASE("/flow/Arena/OptionalMap") {
} }
TEST_CASE("/flow/Arena/Secure") { TEST_CASE("/flow/Arena/Secure") {
#ifndef ADDRESS_SANITIZER #ifndef USE_SANITIZER
// Note: Assumptions underlying this unit test are speculative. // Note: Assumptions underlying this unit test are speculative.
// Disable for a build configuration or entirely if deemed flaky. // Disable for a build configuration or entirely if deemed flaky.
// As of writing, below equivalency of (buf == newBuf) holds except for ASAN builds. // As of writing, below equivalency of (buf == newBuf) holds except for ASAN builds.
@ -1051,6 +1053,6 @@ TEST_CASE("/flow/Arena/Secure") {
} }
} }
fmt::print("Total iterations: {}, # of times check passed: {}\n", totalIters, samePtrCount); fmt::print("Total iterations: {}, # of times check passed: {}\n", totalIters, samePtrCount);
#endif // ADDRESS_SANITIZER #endif // USE_SANITIZER
return Void(); return Void();
} }

6
flow/Platform.actor.cpp
View File

@ -651,7 +651,11 @@ void getDiskBytes(std::string const& directory, int64_t& free, int64_t& total) {
#endif #endif
free = std::min((uint64_t)std::numeric_limits<int64_t>::max(), buf.f_bavail * blockSize); free = std::min((uint64_t)std::numeric_limits<int64_t>::max(), buf.f_bavail * blockSize);
total = std::min((uint64_t)std::numeric_limits<int64_t>::max(), buf.f_blocks * blockSize);
// f_blocks is the total fs space but (f_bfree - f_bavail) is space only available to privileged users
// so that amount will be subtracted from the reported total since FDB can't use it.
total = std::min((uint64_t)std::numeric_limits<int64_t>::max(),
(buf.f_blocks - (buf.f_bfree - buf.f_bavail)) * blockSize);
#elif defined(_WIN32) #elif defined(_WIN32)
std::string fullPath = abspath(directory); std::string fullPath = abspath(directory);

36
flow/include/flow/PriorityMultiLock.actor.h
View File

@ -88,7 +88,7 @@ public:
: PriorityMultiLock(concurrency, parseStringToVector<int>(weights, ',')) {} : PriorityMultiLock(concurrency, parseStringToVector<int>(weights, ',')) {}
PriorityMultiLock(int concurrency, std::vector<int> weightsByPriority) PriorityMultiLock(int concurrency, std::vector<int> weightsByPriority)
: concurrency(concurrency), available(concurrency), waiting(0), totalPendingWeights(0) { : concurrency(concurrency), available(concurrency), waiting(0), totalPendingWeights(0), killed(false) {
priorities.resize(weightsByPriority.size()); priorities.resize(weightsByPriority.size());
for (int i = 0; i < priorities.size(); ++i) { for (int i = 0; i < priorities.size(); ++i) {
@ -102,6 +102,9 @@ public:
~PriorityMultiLock() { kill(); } ~PriorityMultiLock() { kill(); }
Future<Lock> lock(int priority = 0) { Future<Lock> lock(int priority = 0) {
if (killed)
throw broken_promise();
Priority& p = priorities[priority]; Priority& p = priorities[priority];
Queue& q = p.queue; Queue& q = p.queue;
@ -135,17 +138,33 @@ public:
return w.lockPromise.getFuture(); return w.lockPromise.getFuture();
} }
void kill() { // Halt stops the PML from handing out any new locks but leaves waiters and runners alone.
pml_debug_printf("kill %s\n", toString().c_str()); // Existing and new waiters will not see an error, they will just never get a lock.
// Can be safely called multiple times.
void halt() {
pml_debug_printf("halt %s\n", toString().c_str());
brokenOnDestruct.reset(); brokenOnDestruct.reset();
// handleRelease will not free up any execution slots when it ends via cancel if (fRunner.isValid()) {
fRunner.cancel(); fRunner.cancel();
available = 0; // Adjust available and concurrency so that if all runners finish the available
available -= concurrency;
concurrency = 0;
}
waitingPriorities.clear(); waitingPriorities.clear();
for (auto& p : priorities) { }
p.queue.clear();
// Halt, then make existing and new waiters get a broken_promise error.
// Can be safely called multiple times.
void kill() {
if (!killed) {
// Killed must be set first because waiters which ignore exceptions could call wait again immediately.
killed = true;
halt();
for (auto& p : priorities) {
p.queue.clear();
}
} }
} }
@ -231,6 +250,7 @@ private:
Future<Void> fRunner; Future<Void> fRunner;
AsyncTrigger wakeRunner; AsyncTrigger wakeRunner;
Promise<Void> brokenOnDestruct; Promise<Void> brokenOnDestruct;
bool killed;
ACTOR static void handleRelease(Reference<PriorityMultiLock> self, Priority* priority, Future<Void> holder) { ACTOR static void handleRelease(Reference<PriorityMultiLock> self, Priority* priority, Future<Void> holder) {
pml_debug_printf("%f handleRelease self=%p start\n", now(), self.getPtr()); pml_debug_printf("%f handleRelease self=%p start\n", now(), self.getPtr());

2
tests/BlobGranuleFileUnit.toml
View File

@ -2,8 +2,6 @@
encryptModes = ['domain_aware', 'cluster_aware'] encryptModes = ['domain_aware', 'cluster_aware']
[[knobs]] [[knobs]]
enable_encryption = true
enable_blob_file_encryption = true
enable_blob_file_compression = true enable_blob_file_compression = true
[[test]] [[test]]

9
tests/CMakeLists.txt
View File

@ -333,15 +333,15 @@ if(WITH_PYTHON)
add_fdb_test( add_fdb_test(
TEST_FILES restarting/from_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-1.toml TEST_FILES restarting/from_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-1.toml
restarting/from_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-2.toml) restarting/from_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-2.toml)
add_fdb_test(
TEST_FILES restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-1.toml
restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-2.toml)
add_fdb_test( add_fdb_test(
TEST_FILES restarting/from_7.2.0_until_7.3.0/VersionVectorDisableRestart-1.toml TEST_FILES restarting/from_7.2.0_until_7.3.0/VersionVectorDisableRestart-1.toml
restarting/from_7.2.0_until_7.3.0/VersionVectorDisableRestart-2.toml) restarting/from_7.2.0_until_7.3.0/VersionVectorDisableRestart-2.toml)
add_fdb_test( add_fdb_test(
TEST_FILES restarting/from_7.2.0_until_7.3.0/VersionVectorEnableRestart-1.toml TEST_FILES restarting/from_7.2.0_until_7.3.0/VersionVectorEnableRestart-1.toml
restarting/from_7.2.0_until_7.3.0/VersionVectorEnableRestart-2.toml) restarting/from_7.2.0_until_7.3.0/VersionVectorEnableRestart-2.toml)
add_fdb_test(
TEST_FILES restarting/from_7.2.0/DrUpgradeRestart-1.toml
restarting/from_7.2.0/DrUpgradeRestart-2.toml)
add_fdb_test( add_fdb_test(
TEST_FILES restarting/from_7.2.4_until_7.3.0/UpgradeAndBackupRestore-1.toml TEST_FILES restarting/from_7.2.4_until_7.3.0/UpgradeAndBackupRestore-1.toml
restarting/from_7.2.4_until_7.3.0/UpgradeAndBackupRestore-2.toml) restarting/from_7.2.4_until_7.3.0/UpgradeAndBackupRestore-2.toml)
@ -351,6 +351,9 @@ if(WITH_PYTHON)
add_fdb_test( add_fdb_test(
TEST_FILES restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-1.toml TEST_FILES restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-1.toml
restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-2.toml) restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-2.toml)
add_fdb_test(
TEST_FILES restarting/from_7.3.0/DrUpgradeRestart-1.toml
restarting/from_7.3.0/DrUpgradeRestart-2.toml)
add_fdb_test( add_fdb_test(
TEST_FILES restarting/from_7.3.0/UpgradeAndBackupRestore-1.toml TEST_FILES restarting/from_7.3.0/UpgradeAndBackupRestore-1.toml
restarting/from_7.3.0/UpgradeAndBackupRestore-2.toml) restarting/from_7.3.0/UpgradeAndBackupRestore-2.toml)

5
tests/TestRunner/local_cluster.py
View File

@ -114,6 +114,7 @@ logdir = {logdir}
{bg_knob_line} {bg_knob_line}
{encrypt_knob_line1} {encrypt_knob_line1}
{encrypt_knob_line2} {encrypt_knob_line2}
{encrypt_knob_line3}
{tls_config} {tls_config}
{authz_public_key_config} {authz_public_key_config}
{custom_config} {custom_config}
@ -256,13 +257,14 @@ logdir = {logdir}
bg_knob_line = "" bg_knob_line = ""
encrypt_knob_line1 = "" encrypt_knob_line1 = ""
encrypt_knob_line2 = "" encrypt_knob_line2 = ""
encrypt_knob_line3 = ""
if self.use_legacy_conf_syntax: if self.use_legacy_conf_syntax:
conf_template = conf_template.replace("-", "_") conf_template = conf_template.replace("-", "_")
if self.blob_granules_enabled: if self.blob_granules_enabled:
bg_knob_line = "knob_bg_url=file://" + str(self.data) + "/fdbblob/" bg_knob_line = "knob_bg_url=file://" + str(self.data) + "/fdbblob/"
if self.enable_encryption_at_rest: if self.enable_encryption_at_rest:
encrypt_knob_line1 = "knob_enable_encryption=true"
encrypt_knob_line2 = "knob_kms_connector_type=FDBPerfKmsConnector" encrypt_knob_line2 = "knob_kms_connector_type=FDBPerfKmsConnector"
encrypt_knob_line3 = "knob_enable_configurable_encryption=true"
f.write( f.write(
conf_template.format( conf_template.format(
etcdir=self.etc, etcdir=self.etc,
@ -273,6 +275,7 @@ logdir = {logdir}
bg_knob_line=bg_knob_line, bg_knob_line=bg_knob_line,
encrypt_knob_line1=encrypt_knob_line1, encrypt_knob_line1=encrypt_knob_line1,
encrypt_knob_line2=encrypt_knob_line2, encrypt_knob_line2=encrypt_knob_line2,
encrypt_knob_line3=encrypt_knob_line3,
tls_config=self.tls_conf_string(), tls_config=self.tls_conf_string(),
authz_public_key_config=self.authz_public_key_conf_string(), authz_public_key_config=self.authz_public_key_conf_string(),
optional_tls=":tls" if self.tls_config is not None else "", optional_tls=":tls" if self.tls_config is not None else "",

3
tests/fast/BackupCorrectnessWithEKPKeyFetchFailures.toml
View File

@ -4,9 +4,6 @@ tenantModes = ['required']
allowCreatingTenants = false allowCreatingTenants = false
encryptModes = ['domain_aware'] encryptModes = ['domain_aware']
[[knobs]]
enable_encryption = true
[[test]] [[test]]
testTitle = 'BackupAndRestoreWithEKPKeyFetchFailures' testTitle = 'BackupAndRestoreWithEKPKeyFetchFailures'
clearAfterTest = false clearAfterTest = false

3
tests/fast/BackupCorrectnessWithTenantDeletion.toml
View File

@ -3,6 +3,9 @@ allowDefaultTenant = false
tenantModes = ['required'] tenantModes = ['required']
allowCreatingTenants = false allowCreatingTenants = false
[[knobs]]
simulation_enable_snapshot_encryption_checks = false
[[test]] [[test]]
testTitle = 'BackupAndRestoreWithTenantDeletion' testTitle = 'BackupAndRestoreWithTenantDeletion'
clearAfterTest = false clearAfterTest = false

3
tests/fast/EncryptKeyProxyTest.toml
View File

@ -2,9 +2,6 @@
testClass = "Encryption" testClass = "Encryption"
encryptModes = ['domain_aware', 'cluster_aware'] encryptModes = ['domain_aware', 'cluster_aware']
[[knobs]]
enable_encryption = true
[[test]] [[test]]
testTitle = 'EncryptKeyProxy' testTitle = 'EncryptKeyProxy'

4
tests/fast/EncryptedBackupCorrectness.toml
View File

@ -5,10 +5,6 @@ tenantModes = ['required']
encryptModes = ['domain_aware'] encryptModes = ['domain_aware']
[[knobs]] [[knobs]]
enable_encryption = true
enable_tlog_encryption = true
enable_storage_server_encryption = false
enable_blob_granule_encryption = true
max_write_transaction_life_versions = 5000000 max_write_transaction_life_versions = 5000000
[[test]] [[test]]

2
tests/fast/EncryptionOps.toml
View File

@ -1,7 +1,7 @@
[configuration] [configuration]
buggify = false buggify = false
testClass = "Encryption" testClass = "Encryption"
disableEncryption = true encryptModes = ['disabled']
[[knobs]] [[knobs]]
enable_configurable_encryption = true enable_configurable_encryption = true

3
tests/fast/IncrementalBackupWithEKPKeyFetchFailures.toml
View File

@ -4,9 +4,6 @@ tenantModes = ['required']
allowCreatingTenants = false allowCreatingTenants = false
encryptModes = ['domain_aware'] encryptModes = ['domain_aware']
[[knobs]]
enable_encryption = true
[[test]] [[test]]
testTitle = 'SubmitBackup' testTitle = 'SubmitBackup'
simBackupAgents = 'BackupToFile' simBackupAgents = 'BackupToFile'

3
tests/fast/IncrementalBackupWithTenantDeletion.toml
View File

@ -3,6 +3,9 @@ allowDefaultTenant = false
tenantModes = ['required'] tenantModes = ['required']
allowCreatingTenants = false allowCreatingTenants = false
[[knobs]]
simulation_enable_snapshot_encryption_checks = false
[[test]] [[test]]
testTitle = 'SubmitBackup' testTitle = 'SubmitBackup'
simBackupAgents = 'BackupToFile' simBackupAgents = 'BackupToFile'

0
tests/restarting/from_7.2.0/DrUpgradeRestart-1.toml → tests/restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-1.toml
View File

0
tests/restarting/from_7.2.0/DrUpgradeRestart-2.toml → tests/restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-2.toml
View File

28
tests/restarting/from_7.3.0/DrUpgradeRestart-1.toml Normal file
View File

@ -0,0 +1,28 @@
[configuration]
extraDatabaseMode = "Local"
# In 7.2, DR is not supported in required tenant mode
allowDefaultTenant = false
encryptModes = ['disabled']
[[test]]
testTitle = "DrUpgrade"
clearAfterTest = false
simBackupAgents = "BackupToDB"
[[test.workload]]
testName = "Cycle"
nodeCount = 30000
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
[[test.workload]]
testName = "BackupToDBUpgrade"
backupAfter = 10.0
stopDifferentialAfter = 50.0
backupRangesCount = -1
[[test.workload]]
testName = "SaveAndKill"
restartInfoLocation = "simfdb/restartInfo.ini"
testDuration = 40.0

Some files were not shown because too many files have changed in this diff Show More