wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:apple/foundationdb into split-tenant-metadata

This commit is contained in:
Jon Fu 2023-02-21 10:11:35 -08:00
parent 8e6800663b 2497aa5701
commit 428eb07766
107 changed files with 3299 additions and 1413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -97,7 +97,10 @@ else()
set(FDB_VERSION ${PROJECT_VERSION})
endif()
if (NOT FDB_RELEASE)
string(TIMESTAMP FDB_BUILDTIME %Y%m%d%H%M%S)
string(TIMESTAMP FDB_BUILD_TIMESTMAP %Y%m%d%H%M%S)
# Adding support to pass custom fdb_build timestamp,
# required to achieve uniform naming across different builds
set(FDB_BUILDTIME "${FDB_BUILD_TIMESTMAP}" CACHE STRING "A timestamp for packages")
set(FDB_BUILDTIME_STRING ".${FDB_BUILDTIME}")
set(PRERELEASE_TAG "prerelease")
endif()

397
bindings/bindingtester/tests/api.py
View File

@ -33,23 +33,32 @@ fdb.api_version(FDB_API_VERSION)
def matches_op(op, target_op):
return op == target_op or op == target_op + '_SNAPSHOT' or op == target_op + '_DATABASE' or op == target_op + '_TENANT'
return (
op == target_op
or op == target_op + "_SNAPSHOT"
or op == target_op + "_DATABASE"
or op == target_op + "_TENANT"
)
def is_non_transaction_op(op):
return op.endswith('_DATABASE') or op.endswith('_TENANT')
return op.endswith("_DATABASE") or op.endswith("_TENANT")
class ApiTest(Test):
def __init__(self, subspace):
super(ApiTest, self).__init__(subspace)
self.workspace = self.subspace['workspace'] # The keys and values here must match between subsequent runs of the same test
self.scratch = self.subspace['scratch'] # The keys and values here can differ between runs
self.stack_subspace = self.subspace['stack']
self.workspace = self.subspace[
"workspace"
] # The keys and values here must match between subsequent runs of the same test
self.scratch = self.subspace[
"scratch"
] # The keys and values here can differ between runs
self.stack_subspace = self.subspace["stack"]
self.versionstamped_values = self.scratch['versionstamped_values']
self.versionstamped_values_2 = self.scratch['versionstamped_values_2']
self.versionstamped_keys = self.scratch['versionstamped_keys']
self.versionstamped_values = self.scratch["versionstamped_values"]
self.versionstamped_values_2 = self.scratch["versionstamped_values_2"]
self.versionstamped_keys = self.scratch["versionstamped_keys"]
def setup(self, args):
self.stack_size = 0
@ -64,7 +73,9 @@ class ApiTest(Test):
self.generated_keys = []
self.outstanding_ops = []
self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version, args.types)
self.random = test_util.RandomGenerator(
args.max_int_bits, args.api_version, args.types
)
self.api_version = args.api_version
self.allocated_tenants = set()
@ -88,7 +99,9 @@ class ApiTest(Test):
self.string_depth = max(0, self.string_depth - num)
self.key_depth = max(0, self.key_depth - num)
self.outstanding_ops = [i for i in self.outstanding_ops if i[0] <= self.stack_size]
self.outstanding_ops = [
i for i in self.outstanding_ops if i[0] <= self.stack_size
]
def ensure_string(self, instructions, num):
while self.string_depth < num:
@ -101,7 +114,7 @@ class ApiTest(Test):
if random.random() < float(len(self.generated_keys)) / self.max_keys:
tup = random.choice(self.generated_keys)
if random.random() < 0.3:
return self.workspace.pack(tup[0:random.randint(0, len(tup))])
return self.workspace.pack(tup[0 : random.randint(0, len(tup))])
return self.workspace.pack(tup)
@ -119,7 +132,9 @@ class ApiTest(Test):
def ensure_key_value(self, instructions):
if self.string_depth == 0:
instructions.push_args(self.choose_key(), self.random.random_string(random.randint(0, 100)))
instructions.push_args(
self.choose_key(), self.random.random_string(random.randint(0, 100))
)
elif self.string_depth == 1 or self.key_depth == 0:
self.ensure_key(instructions, 1)
@ -131,7 +146,7 @@ class ApiTest(Test):
def preload_database(self, instructions, num):
for i in range(num):
self.ensure_key_value(instructions)
instructions.append('SET')
instructions.append("SET")
if i % 100 == 99:
test_util.blocking_commit(instructions)
@ -140,42 +155,81 @@ class ApiTest(Test):
self.add_stack_items(1)
def wait_for_reads(self, instructions):
while len(self.outstanding_ops) > 0 and self.outstanding_ops[-1][0] <= self.stack_size:
while (
len(self.outstanding_ops) > 0
and self.outstanding_ops[-1][0] <= self.stack_size
):
read = self.outstanding_ops.pop()
# print '%d. waiting for read at instruction %r' % (len(instructions), read)
test_util.to_front(instructions, self.stack_size - read[0])
instructions.append('WAIT_FUTURE')
instructions.append("WAIT_FUTURE")
def choose_tenant(self, new_tenant_probability):
if len(self.allocated_tenants) == 0 or random.random() < new_tenant_probability:
return self.random.random_string(random.randint(0, 30))
else:
return random.choice(list(self.allocated_tenants))
tenant_list = list(self.allocated_tenants)
# sort to ensure deterministic selection of a tenant
tenant_list.sort()
return random.choice(tenant_list)
def generate(self, args, thread_number):
instructions = InstructionSet()
op_choices = ['NEW_TRANSACTION', 'COMMIT']
op_choices = ["NEW_TRANSACTION", "COMMIT"]
reads = ['GET', 'GET_KEY', 'GET_RANGE', 'GET_RANGE_STARTS_WITH', 'GET_RANGE_SELECTOR']
mutations = ['SET', 'CLEAR', 'CLEAR_RANGE', 'CLEAR_RANGE_STARTS_WITH', 'ATOMIC_OP']
snapshot_reads = [x + '_SNAPSHOT' for x in reads]
database_reads = [x + '_DATABASE' for x in reads]
database_mutations = [x + '_DATABASE' for x in mutations]
tenant_reads = [x + '_TENANT' for x in reads]
tenant_mutations = [x + '_TENANT' for x in mutations]
mutations += ['VERSIONSTAMP']
versions = ['GET_READ_VERSION', 'SET_READ_VERSION', 'GET_COMMITTED_VERSION']
snapshot_versions = ['GET_READ_VERSION_SNAPSHOT']
tuples = ['TUPLE_PACK', 'TUPLE_UNPACK', 'TUPLE_RANGE', 'TUPLE_SORT', 'SUB', 'ENCODE_FLOAT', 'ENCODE_DOUBLE', 'DECODE_DOUBLE', 'DECODE_FLOAT']
if 'versionstamp' in args.types:
tuples.append('TUPLE_PACK_WITH_VERSIONSTAMP')
resets = ['ON_ERROR', 'RESET', 'CANCEL']
read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY']
write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT']
txn_sizes = ['GET_APPROXIMATE_SIZE']
storage_metrics = ['GET_ESTIMATED_RANGE_SIZE', 'GET_RANGE_SPLIT_POINTS']
tenants = ['TENANT_CREATE', 'TENANT_DELETE', 'TENANT_SET_ACTIVE', 'TENANT_CLEAR_ACTIVE', 'TENANT_LIST', 'TENANT_GET_ID']
reads = [
"GET",
"GET_KEY",
"GET_RANGE",
"GET_RANGE_STARTS_WITH",
"GET_RANGE_SELECTOR",
]
mutations = [
"SET",
"CLEAR",
"CLEAR_RANGE",
"CLEAR_RANGE_STARTS_WITH",
"ATOMIC_OP",
]
snapshot_reads = [x + "_SNAPSHOT" for x in reads]
database_reads = [x + "_DATABASE" for x in reads]
database_mutations = [x + "_DATABASE" for x in mutations]
tenant_reads = [x + "_TENANT" for x in reads]
tenant_mutations = [x + "_TENANT" for x in mutations]
mutations += ["VERSIONSTAMP"]
versions = ["GET_READ_VERSION", "SET_READ_VERSION", "GET_COMMITTED_VERSION"]
snapshot_versions = ["GET_READ_VERSION_SNAPSHOT"]
tuples = [
"TUPLE_PACK",
"TUPLE_UNPACK",
"TUPLE_RANGE",
"TUPLE_SORT",
"SUB",
"ENCODE_FLOAT",
"ENCODE_DOUBLE",
"DECODE_DOUBLE",
"DECODE_FLOAT",
]
if "versionstamp" in args.types:
tuples.append("TUPLE_PACK_WITH_VERSIONSTAMP")
resets = ["ON_ERROR", "RESET", "CANCEL"]
read_conflicts = ["READ_CONFLICT_RANGE", "READ_CONFLICT_KEY"]
write_conflicts = [
"WRITE_CONFLICT_RANGE",
"WRITE_CONFLICT_KEY",
"DISABLE_WRITE_CONFLICT",
]
txn_sizes = ["GET_APPROXIMATE_SIZE"]
storage_metrics = ["GET_ESTIMATED_RANGE_SIZE", "GET_RANGE_SPLIT_POINTS"]
tenants = [
"TENANT_CREATE",
"TENANT_DELETE",
"TENANT_SET_ACTIVE",
"TENANT_CLEAR_ACTIVE",
"TENANT_LIST",
"TENANT_GET_ID",
]
op_choices += reads
op_choices += mutations
@ -196,16 +250,23 @@ class ApiTest(Test):
op_choices += tenant_reads
op_choices += tenant_mutations
idempotent_atomic_ops = ['BIT_AND', 'BIT_OR', 'MAX', 'MIN', 'BYTE_MIN', 'BYTE_MAX']
atomic_ops = idempotent_atomic_ops + ['ADD', 'BIT_XOR', 'APPEND_IF_FITS']
idempotent_atomic_ops = [
"BIT_AND",
"BIT_OR",
"MAX",
"MIN",
"BYTE_MIN",
"BYTE_MAX",
]
atomic_ops = idempotent_atomic_ops + ["ADD", "BIT_XOR", "APPEND_IF_FITS"]
if args.concurrency > 1:
self.max_keys = random.randint(100, 1000)
else:
self.max_keys = random.randint(100, 10000)
instructions.append('NEW_TRANSACTION')
instructions.append('GET_READ_VERSION')
instructions.append("NEW_TRANSACTION")
instructions.append("GET_READ_VERSION")
self.preload_database(instructions, self.max_keys)
@ -218,25 +279,29 @@ class ApiTest(Test):
# print 'Adding instruction %s at %d' % (op, index)
if args.concurrency == 1 and (op in database_mutations or op in tenant_mutations or op in ['TENANT_CREATE', 'TENANT_DELETE']):
if args.concurrency == 1 and (
op in database_mutations
or op in tenant_mutations
or op in ["TENANT_CREATE", "TENANT_DELETE"]
):
self.wait_for_reads(instructions)
test_util.blocking_commit(instructions)
self.can_get_commit_version = False
self.add_stack_items(1)
if op in resets or op == 'NEW_TRANSACTION':
if op in resets or op == "NEW_TRANSACTION":
if args.concurrency == 1:
self.wait_for_reads(instructions)
self.outstanding_ops = []
if op == 'NEW_TRANSACTION':
if op == "NEW_TRANSACTION":
instructions.append(op)
self.can_get_commit_version = True
self.can_set_version = True
self.can_use_key_selectors = True
elif op == 'ON_ERROR':
elif op == "ON_ERROR":
instructions.push_args(random.randint(0, 5000))
instructions.append(op)
@ -244,20 +309,20 @@ class ApiTest(Test):
if args.concurrency == 1:
self.wait_for_reads(instructions)
instructions.append('NEW_TRANSACTION')
instructions.append("NEW_TRANSACTION")
self.can_get_commit_version = True
self.can_set_version = True
self.can_use_key_selectors = True
self.add_strings(1)
elif matches_op(op, 'GET'):
elif matches_op(op, "GET"):
self.ensure_key(instructions, 1)
instructions.append(op)
self.add_strings(1)
self.can_set_version = False
read_performed = True
elif matches_op(op, 'GET_KEY'):
elif matches_op(op, "GET_KEY"):
if is_non_transaction_op(op) or self.can_use_key_selectors:
self.ensure_key(instructions, 1)
instructions.push_args(self.workspace.key())
@ -270,7 +335,7 @@ class ApiTest(Test):
self.can_set_version = False
read_performed = True
elif matches_op(op, 'GET_RANGE'):
elif matches_op(op, "GET_RANGE"):
self.ensure_key(instructions, 2)
range_params = self.random.random_range_params()
instructions.push_args(*range_params)
@ -278,7 +343,9 @@ class ApiTest(Test):
test_util.to_front(instructions, 4)
instructions.append(op)
if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large
if (
range_params[0] >= 1 and range_params[0] <= 1000
): # avoid adding a string if the limit is large
self.add_strings(1)
else:
self.add_stack_items(1)
@ -286,7 +353,7 @@ class ApiTest(Test):
self.can_set_version = False
read_performed = True
elif matches_op(op, 'GET_RANGE_STARTS_WITH'):
elif matches_op(op, "GET_RANGE_STARTS_WITH"):
# TODO: not tested well
self.ensure_key(instructions, 1)
range_params = self.random.random_range_params()
@ -294,7 +361,9 @@ class ApiTest(Test):
test_util.to_front(instructions, 3)
instructions.append(op)
if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large
if (
range_params[0] >= 1 and range_params[0] <= 1000
): # avoid adding a string if the limit is large
self.add_strings(1)
else:
self.add_stack_items(1)
@ -302,7 +371,7 @@ class ApiTest(Test):
self.can_set_version = False
read_performed = True
elif matches_op(op, 'GET_RANGE_SELECTOR'):
elif matches_op(op, "GET_RANGE_SELECTOR"):
if is_non_transaction_op(op) or self.can_use_key_selectors:
self.ensure_key(instructions, 2)
instructions.push_args(self.workspace.key())
@ -314,7 +383,9 @@ class ApiTest(Test):
test_util.to_front(instructions, 9)
instructions.append(op)
if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large
if (
range_params[0] >= 1 and range_params[0] <= 1000
): # avoid adding a string if the limit is large
self.add_strings(1)
else:
self.add_stack_items(1)
@ -322,29 +393,29 @@ class ApiTest(Test):
self.can_set_version = False
read_performed = True
elif matches_op(op, 'GET_READ_VERSION'):
elif matches_op(op, "GET_READ_VERSION"):
instructions.append(op)
self.has_version = self.can_set_version
self.add_strings(1)
elif matches_op(op, 'SET'):
elif matches_op(op, "SET"):
self.ensure_key_value(instructions)
instructions.append(op)
if is_non_transaction_op(op):
self.add_stack_items(1)
elif op == 'SET_READ_VERSION':
elif op == "SET_READ_VERSION":
if self.has_version and self.can_set_version:
instructions.append(op)
self.can_set_version = False
elif matches_op(op, 'CLEAR'):
elif matches_op(op, "CLEAR"):
self.ensure_key(instructions, 1)
instructions.append(op)
if is_non_transaction_op(op):
self.add_stack_items(1)
elif matches_op(op, 'CLEAR_RANGE'):
elif matches_op(op, "CLEAR_RANGE"):
# Protect against inverted range
key1 = self.workspace.pack(self.random.random_tuple(5))
key2 = self.workspace.pack(self.random.random_tuple(5))
@ -358,13 +429,13 @@ class ApiTest(Test):
if is_non_transaction_op(op):
self.add_stack_items(1)
elif matches_op(op, 'CLEAR_RANGE_STARTS_WITH'):
elif matches_op(op, "CLEAR_RANGE_STARTS_WITH"):
self.ensure_key(instructions, 1)
instructions.append(op)
if is_non_transaction_op(op):
self.add_stack_items(1)
elif matches_op(op, 'ATOMIC_OP'):
elif matches_op(op, "ATOMIC_OP"):
self.ensure_key_value(instructions)
if is_non_transaction_op(op) and args.concurrency == 1:
instructions.push_args(random.choice(idempotent_atomic_ops))
@ -375,50 +446,58 @@ class ApiTest(Test):
if is_non_transaction_op(op):
self.add_stack_items(1)
elif op == 'VERSIONSTAMP':
elif op == "VERSIONSTAMP":
rand_str1 = self.random.random_string(100)
key1 = self.versionstamped_values.pack((rand_str1,))
key2 = self.versionstamped_values_2.pack((rand_str1,))
split = random.randint(0, 70)
prefix = self.random.random_string(20 + split)
if prefix.endswith(b'\xff'):
if prefix.endswith(b"\xff"):
# Necessary to make sure that the SET_VERSIONSTAMPED_VALUE check
# correctly finds where the version is supposed to fit in.
prefix += b'\x00'
prefix += b"\x00"
suffix = self.random.random_string(70 - split)
rand_str2 = prefix + fdb.tuple.Versionstamp._UNSET_TR_VERSION + suffix
key3 = self.versionstamped_keys.pack() + rand_str2
index = len(self.versionstamped_keys.pack()) + len(prefix)
key3 = self.versionstamp_key(key3, index)
instructions.push_args('SET_VERSIONSTAMPED_VALUE',
key1,
self.versionstamp_value(fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2))
instructions.append('ATOMIC_OP')
instructions.push_args(
"SET_VERSIONSTAMPED_VALUE",
key1,
self.versionstamp_value(
fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2
),
)
instructions.append("ATOMIC_OP")
if args.api_version >= 520:
instructions.push_args('SET_VERSIONSTAMPED_VALUE', key2, self.versionstamp_value(rand_str2, len(prefix)))
instructions.append('ATOMIC_OP')
instructions.push_args(
"SET_VERSIONSTAMPED_VALUE",
key2,
self.versionstamp_value(rand_str2, len(prefix)),
)
instructions.append("ATOMIC_OP")
instructions.push_args('SET_VERSIONSTAMPED_KEY', key3, rand_str1)
instructions.append('ATOMIC_OP')
instructions.push_args("SET_VERSIONSTAMPED_KEY", key3, rand_str1)
instructions.append("ATOMIC_OP")
self.can_use_key_selectors = False
elif op == 'READ_CONFLICT_RANGE' or op == 'WRITE_CONFLICT_RANGE':
elif op == "READ_CONFLICT_RANGE" or op == "WRITE_CONFLICT_RANGE":
self.ensure_key(instructions, 2)
instructions.append(op)
self.add_strings(1)
elif op == 'READ_CONFLICT_KEY' or op == 'WRITE_CONFLICT_KEY':
elif op == "READ_CONFLICT_KEY" or op == "WRITE_CONFLICT_KEY":
self.ensure_key(instructions, 1)
instructions.append(op)
self.add_strings(1)
elif op == 'DISABLE_WRITE_CONFLICT':
elif op == "DISABLE_WRITE_CONFLICT":
instructions.append(op)
elif op == 'COMMIT':
elif op == "COMMIT":
if args.concurrency == 1 or i < self.max_keys or random.random() < 0.9:
if args.concurrency == 1:
self.wait_for_reads(instructions)
@ -431,23 +510,23 @@ class ApiTest(Test):
instructions.append(op)
self.add_strings(1)
elif op == 'RESET':
elif op == "RESET":
instructions.append(op)
self.can_get_commit_version = False
self.can_set_version = True
self.can_use_key_selectors = True
elif op == 'CANCEL':
elif op == "CANCEL":
instructions.append(op)
self.can_set_version = False
elif op == 'GET_COMMITTED_VERSION':
elif op == "GET_COMMITTED_VERSION":
if self.can_get_commit_version:
do_commit = random.random() < 0.5
if do_commit:
instructions.append('COMMIT')
instructions.append('WAIT_FUTURE')
instructions.append("COMMIT")
instructions.append("WAIT_FUTURE")
self.add_stack_items(1)
instructions.append(op)
@ -456,35 +535,47 @@ class ApiTest(Test):
self.add_strings(1)
if do_commit:
instructions.append('RESET')
instructions.append("RESET")
self.can_get_commit_version = False
self.can_set_version = True
self.can_use_key_selectors = True
elif op == 'GET_APPROXIMATE_SIZE':
elif op == "GET_APPROXIMATE_SIZE":
instructions.append(op)
self.add_strings(1)
elif op == 'TUPLE_PACK' or op == 'TUPLE_RANGE':
elif op == "TUPLE_PACK" or op == "TUPLE_RANGE":
tup = self.random.random_tuple(10)
instructions.push_args(len(tup), *tup)
instructions.append(op)
if op == 'TUPLE_PACK':
if op == "TUPLE_PACK":
self.add_strings(1)
else:
self.add_strings(2)
elif op == 'TUPLE_PACK_WITH_VERSIONSTAMP':
tup = (self.random.random_string(20),) + self.random.random_tuple(10, incomplete_versionstamps=True)
elif op == "TUPLE_PACK_WITH_VERSIONSTAMP":
tup = (self.random.random_string(20),) + self.random.random_tuple(
10, incomplete_versionstamps=True
)
prefix = self.versionstamped_keys.pack()
instructions.push_args(prefix, len(tup), *tup)
instructions.append(op)
self.add_strings(1)
versionstamp_param = prefix + fdb.tuple.pack(tup)
first_incomplete = versionstamp_param.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION)
second_incomplete = -1 if first_incomplete < 0 else \
versionstamp_param.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION, first_incomplete + len(fdb.tuple.Versionstamp._UNSET_TR_VERSION) + 1)
first_incomplete = versionstamp_param.find(
fdb.tuple.Versionstamp._UNSET_TR_VERSION
)
second_incomplete = (
-1
if first_incomplete < 0
else versionstamp_param.find(
fdb.tuple.Versionstamp._UNSET_TR_VERSION,
first_incomplete
+ len(fdb.tuple.Versionstamp._UNSET_TR_VERSION)
+ 1,
)
)
# If there is exactly one incomplete versionstamp, perform the versionstamp operation.
if first_incomplete >= 0 and second_incomplete < 0:
@ -492,76 +583,90 @@ class ApiTest(Test):
instructions.push_args(rand_str)
test_util.to_front(instructions, 1)
instructions.push_args('SET_VERSIONSTAMPED_KEY')
instructions.append('ATOMIC_OP')
instructions.push_args("SET_VERSIONSTAMPED_KEY")
instructions.append("ATOMIC_OP")
if self.api_version >= 520:
version_value_key_2 = self.versionstamped_values_2.pack((rand_str,))
versionstamped_value = self.versionstamp_value(fdb.tuple.pack(tup), first_incomplete - len(prefix))
instructions.push_args('SET_VERSIONSTAMPED_VALUE', version_value_key_2, versionstamped_value)
instructions.append('ATOMIC_OP')
version_value_key_2 = self.versionstamped_values_2.pack(
(rand_str,)
)
versionstamped_value = self.versionstamp_value(
fdb.tuple.pack(tup), first_incomplete - len(prefix)
)
instructions.push_args(
"SET_VERSIONSTAMPED_VALUE",
version_value_key_2,
versionstamped_value,
)
instructions.append("ATOMIC_OP")
version_value_key = self.versionstamped_values.pack((rand_str,))
instructions.push_args('SET_VERSIONSTAMPED_VALUE', version_value_key,
self.versionstamp_value(fdb.tuple.Versionstamp._UNSET_TR_VERSION + fdb.tuple.pack(tup)))
instructions.append('ATOMIC_OP')
instructions.push_args(
"SET_VERSIONSTAMPED_VALUE",
version_value_key,
self.versionstamp_value(
fdb.tuple.Versionstamp._UNSET_TR_VERSION
+ fdb.tuple.pack(tup)
),
)
instructions.append("ATOMIC_OP")
self.can_use_key_selectors = False
elif op == 'TUPLE_UNPACK':
elif op == "TUPLE_UNPACK":
tup = self.random.random_tuple(10)
instructions.push_args(len(tup), *tup)
instructions.append('TUPLE_PACK')
instructions.append("TUPLE_PACK")
instructions.append(op)
self.add_strings(len(tup))
elif op == 'TUPLE_SORT':
elif op == "TUPLE_SORT":
tups = self.random.random_tuple_list(10, 30)
for tup in tups:
instructions.push_args(len(tup), *tup)
instructions.append('TUPLE_PACK')
instructions.append("TUPLE_PACK")
instructions.push_args(len(tups))
instructions.append(op)
self.add_strings(len(tups))
# Use SUB to test if integers are correctly unpacked
elif op == 'SUB':
elif op == "SUB":
a = self.random.random_int() // 2
b = self.random.random_int() // 2
instructions.push_args(0, a, b)
instructions.append(op)
instructions.push_args(1)
instructions.append('SWAP')
instructions.append("SWAP")
instructions.append(op)
instructions.push_args(1)
instructions.append('TUPLE_PACK')
instructions.append("TUPLE_PACK")
self.add_stack_items(1)
elif op == 'ENCODE_FLOAT':
elif op == "ENCODE_FLOAT":
f = self.random.random_float(8)
f_bytes = struct.pack('>f', f)
f_bytes = struct.pack(">f", f)
instructions.push_args(f_bytes)
instructions.append(op)
self.add_stack_items(1)
elif op == 'ENCODE_DOUBLE':
elif op == "ENCODE_DOUBLE":
d = self.random.random_float(11)
d_bytes = struct.pack('>d', d)
d_bytes = struct.pack(">d", d)
instructions.push_args(d_bytes)
instructions.append(op)
self.add_stack_items(1)
elif op == 'DECODE_FLOAT':
elif op == "DECODE_FLOAT":
f = self.random.random_float(8)
instructions.push_args(fdb.tuple.SingleFloat(f))
instructions.append(op)
self.add_strings(1)
elif op == 'DECODE_DOUBLE':
elif op == "DECODE_DOUBLE":
d = self.random.random_float(11)
instructions.push_args(d)
instructions.append(op)
self.add_strings(1)
elif op == 'GET_ESTIMATED_RANGE_SIZE':
elif op == "GET_ESTIMATED_RANGE_SIZE":
# Protect against inverted range and identical keys
key1 = self.workspace.pack(self.random.random_tuple(1))
key2 = self.workspace.pack(self.random.random_tuple(1))
@ -576,7 +681,7 @@ class ApiTest(Test):
instructions.push_args(key1, key2)
instructions.append(op)
self.add_strings(1)
elif op == 'GET_RANGE_SPLIT_POINTS':
elif op == "GET_RANGE_SPLIT_POINTS":
# Protect against inverted range and identical keys
key1 = self.workspace.pack(self.random.random_tuple(1))
key2 = self.workspace.pack(self.random.random_tuple(1))
@ -593,27 +698,27 @@ class ApiTest(Test):
instructions.push_args(key1, key2, chunkSize)
instructions.append(op)
self.add_strings(1)
elif op == 'TENANT_CREATE':
elif op == "TENANT_CREATE":
tenant_name = self.choose_tenant(0.8)
self.allocated_tenants.add(tenant_name)
instructions.push_args(tenant_name)
instructions.append(op)
self.add_strings(1)
elif op == 'TENANT_DELETE':
elif op == "TENANT_DELETE":
tenant_name = self.choose_tenant(0.2)
if tenant_name in self.allocated_tenants:
self.allocated_tenants.remove(tenant_name)
instructions.push_args(tenant_name)
instructions.append(op)
self.add_strings(1)
elif op == 'TENANT_SET_ACTIVE':
elif op == "TENANT_SET_ACTIVE":
tenant_name = self.choose_tenant(0.8)
instructions.push_args(tenant_name)
instructions.append(op)
self.add_strings(1)
elif op == 'TENANT_CLEAR_ACTIVE':
elif op == "TENANT_CLEAR_ACTIVE":
instructions.append(op)
elif op == 'TENANT_LIST':
elif op == "TENANT_LIST":
self.ensure_string(instructions, 2)
instructions.push_args(self.random.random_int())
test_util.to_front(instructions, 2)
@ -624,27 +729,33 @@ class ApiTest(Test):
instructions.append(op)
self.add_strings(1)
else:
assert False, 'Unknown operation: ' + op
assert False, "Unknown operation: " + op
if read_performed and op not in database_reads and op not in tenant_reads:
self.outstanding_ops.append((self.stack_size, len(instructions) - 1))
if args.concurrency == 1 and (op in database_reads or op in database_mutations or op in tenant_reads or op in tenant_mutations or op in ['TENANT_CREATE', 'TENANT_DELETE']):
instructions.append('WAIT_FUTURE')
if args.concurrency == 1 and (
op in database_reads
or op in database_mutations
or op in tenant_reads
or op in tenant_mutations
or op in ["TENANT_CREATE", "TENANT_DELETE"]
):
instructions.append("WAIT_FUTURE")
instructions.begin_finalization()
if not args.no_tenants:
instructions.append('TENANT_CLEAR_ACTIVE')
instructions.append("TENANT_CLEAR_ACTIVE")
if args.concurrency == 1:
self.wait_for_reads(instructions)
test_util.blocking_commit(instructions)
self.add_stack_items(1)
instructions.append('NEW_TRANSACTION')
instructions.append("NEW_TRANSACTION")
instructions.push_args(self.stack_subspace.key())
instructions.append('LOG_STACK')
instructions.append("LOG_STACK")
test_util.blocking_commit(instructions)
@ -654,22 +765,30 @@ class ApiTest(Test):
def check_versionstamps(self, tr, begin_key, limit):
next_begin = None
incorrect_versionstamps = 0
for k, v in tr.get_range(begin_key, self.versionstamped_values.range().stop, limit=limit):
next_begin = k + b'\x00'
for k, v in tr.get_range(
begin_key, self.versionstamped_values.range().stop, limit=limit
):
next_begin = k + b"\x00"
random_id = self.versionstamped_values.unpack(k)[0]
versioned_value = v[10:].replace(fdb.tuple.Versionstamp._UNSET_TR_VERSION, v[:10], 1)
versioned_value = v[10:].replace(
fdb.tuple.Versionstamp._UNSET_TR_VERSION, v[:10], 1
)
versioned_key = self.versionstamped_keys.pack() + versioned_value
if tr[versioned_key] != random_id:
util.get_logger().error(' INCORRECT VERSIONSTAMP:')
util.get_logger().error(' %s != %s', repr(tr[versioned_key]), repr(random_id))
util.get_logger().error(" INCORRECT VERSIONSTAMP:")
util.get_logger().error(
" %s != %s", repr(tr[versioned_key]), repr(random_id)
)
incorrect_versionstamps += 1
if self.api_version >= 520:
k2 = self.versionstamped_values_2.pack((random_id,))
if tr[k2] != versioned_value:
util.get_logger().error(' INCORRECT VERSIONSTAMP:')
util.get_logger().error(' %s != %s', repr(tr[k2]), repr(versioned_value))
util.get_logger().error(" INCORRECT VERSIONSTAMP:")
util.get_logger().error(
" %s != %s", repr(tr[k2]), repr(versioned_value)
)
incorrect_versionstamps += 1
return (next_begin, incorrect_versionstamps)
@ -681,16 +800,26 @@ class ApiTest(Test):
incorrect_versionstamps = 0
while begin is not None:
(begin, current_incorrect_versionstamps) = self.check_versionstamps(db, begin, 100)
(begin, current_incorrect_versionstamps) = self.check_versionstamps(
db, begin, 100
)
incorrect_versionstamps += current_incorrect_versionstamps
if incorrect_versionstamps > 0:
errors.append('There were %d failed version stamp operations' % incorrect_versionstamps)
errors.append(
"There were %d failed version stamp operations"
% incorrect_versionstamps
)
return errors
def get_result_specifications(self):
return [
ResultSpecification(self.workspace, global_error_filter=[1007, 1009, 1021]),
ResultSpecification(self.stack_subspace, key_start_index=1, ordering_index=1, global_error_filter=[1007, 1009, 1021])
ResultSpecification(
self.stack_subspace,
key_start_index=1,
ordering_index=1,
global_error_filter=[1007, 1009, 1021],
),
]

95
bindings/bindingtester/tests/test_util.py
View File

@ -33,16 +33,20 @@ from bindingtester.known_testers import COMMON_TYPES
class RandomGenerator(object):
def __init__(self, max_int_bits=64, api_version=FDB_API_VERSION, types=COMMON_TYPES):
def __init__(
self, max_int_bits=64, api_version=FDB_API_VERSION, types=COMMON_TYPES
):
self.max_int_bits = max_int_bits
self.api_version = api_version
self.types = list(types)
def random_unicode_str(self, length):
return ''.join(self.random_unicode_char() for i in range(0, length))
return "".join(self.random_unicode_char() for i in range(0, length))
def random_int(self):
num_bits = random.randint(0, self.max_int_bits) # This way, we test small numbers with higher probability
num_bits = random.randint(
0, self.max_int_bits
) # This way, we test small numbers with higher probability
max_value = (1 << num_bits) - 1
min_value = -max_value - 1
@ -54,11 +58,15 @@ class RandomGenerator(object):
def random_float(self, exp_bits):
if random.random() < 0.05:
# Choose a special value.
return random.choice([float('-nan'), float('-inf'), -0.0, 0.0, float('inf'), float('nan')])
return random.choice(
[float("-nan"), float("-inf"), -0.0, 0.0, float("inf"), float("nan")]
)
else:
# Choose a value from all over the range of acceptable floats for this precision.
sign = -1 if random.random() < 0.5 else 1
exponent = random.randint(-(1 << (exp_bits - 1)) - 10, (1 << (exp_bits - 1) - 1))
exponent = random.randint(
-(1 << (exp_bits - 1)) - 10, (1 << (exp_bits - 1) - 1)
)
mantissa = random.random()
result = sign * math.pow(2, exponent) * mantissa
@ -73,38 +81,38 @@ class RandomGenerator(object):
for i in range(size):
choice = random.choice(self.types)
if choice == 'int':
if choice == "int":
tup.append(self.random_int())
elif choice == 'null':
elif choice == "null":
tup.append(None)
elif choice == 'bytes':
elif choice == "bytes":
tup.append(self.random_string(random.randint(0, 100)))
elif choice == 'string':
elif choice == "string":
tup.append(self.random_unicode_str(random.randint(0, 100)))
elif choice == 'uuid':
tup.append(uuid.uuid4())
elif choice == 'bool':
elif choice == "uuid":
tup.append(uuid.UUID(int=random.getrandbits(128)))
elif choice == "bool":
b = random.random() < 0.5
if self.api_version < 500:
tup.append(int(b))
else:
tup.append(b)
elif choice == 'float':
elif choice == "float":
tup.append(fdb.tuple.SingleFloat(self.random_float(8)))
elif choice == 'double':
elif choice == "double":
tup.append(self.random_float(11))
elif choice == 'tuple':
elif choice == "tuple":
length = random.randint(0, max_size - size)
if length == 0:
tup.append(())
else:
tup.append(self.random_tuple(length))
elif choice == 'versionstamp':
elif choice == "versionstamp":
if incomplete_versionstamps and random.random() < 0.5:
tr_version = fdb.tuple.Versionstamp._UNSET_TR_VERSION
else:
tr_version = self.random_string(10)
user_version = random.randint(0, 0xffff)
user_version = random.randint(0, 0xFFFF)
tup.append(fdb.tuple.Versionstamp(tr_version, user_version))
else:
assert False
@ -123,12 +131,19 @@ class RandomGenerator(object):
smaller_size = random.randint(1, len(to_add))
tuples.append(to_add[:smaller_size])
else:
non_empty = [x for x in enumerate(to_add) if (isinstance(x[1], list) or isinstance(x[1], tuple)) and len(x[1]) > 0]
non_empty = [
x
for x in enumerate(to_add)
if (isinstance(x[1], list) or isinstance(x[1], tuple))
and len(x[1]) > 0
]
if len(non_empty) > 0 and random.random() < 0.25:
# Add a smaller list to test prefixes of nested structures.
idx, choice = random.choice(non_empty)
smaller_size = random.randint(0, len(to_add[idx]))
tuples.append(to_add[:idx] + (choice[:smaller_size],) + to_add[idx + 1:])
tuples.append(
to_add[:idx] + (choice[:smaller_size],) + to_add[idx + 1 :]
)
random.shuffle(tuples)
return tuples
@ -153,30 +168,40 @@ class RandomGenerator(object):
def random_string(self, length):
if length == 0:
return b''
return b""
return bytes([random.randint(0, 254)] + [random.randint(0, 255) for i in range(0, length - 1)])
return bytes(
[random.randint(0, 254)]
+ [random.randint(0, 255) for i in range(0, length - 1)]
)
def random_unicode_char(self):
while True:
if random.random() < 0.05:
# Choose one of these special character sequences.
specials = ['\U0001f4a9', '\U0001f63c', '\U0001f3f3\ufe0f\u200d\U0001f308', '\U0001f1f5\U0001f1f2', '\uf8ff',
'\U0002a2b2', '\u05e9\u05dc\u05d5\u05dd']
specials = [
"\U0001f4a9",
"\U0001f63c",
"\U0001f3f3\ufe0f\u200d\U0001f308",
"\U0001f1f5\U0001f1f2",
"\uf8ff",
"\U0002a2b2",
"\u05e9\u05dc\u05d5\u05dd",
]
return random.choice(specials)
c = random.randint(0, 0xffff)
if unicodedata.category(chr(c))[0] in 'LMNPSZ':
c = random.randint(0, 0xFFFF)
if unicodedata.category(chr(c))[0] in "LMNPSZ":
return chr(c)
def error_string(error_code):
return fdb.tuple.pack((b'ERROR', bytes(str(error_code), 'utf-8')))
return fdb.tuple.pack((b"ERROR", bytes(str(error_code), "utf-8")))
def blocking_commit(instructions):
instructions.append('COMMIT')
instructions.append('WAIT_FUTURE')
instructions.append('RESET')
instructions.append("COMMIT")
instructions.append("WAIT_FUTURE")
instructions.append("RESET")
def to_front(instructions, index):
@ -184,19 +209,19 @@ def to_front(instructions, index):
pass
elif index == 1:
instructions.push_args(1)
instructions.append('SWAP')
instructions.append("SWAP")
elif index == 2:
instructions.push_args(index - 1)
instructions.append('SWAP')
instructions.append("SWAP")
instructions.push_args(index)
instructions.append('SWAP')
instructions.append("SWAP")
else:
instructions.push_args(index - 1)
instructions.append('SWAP')
instructions.append("SWAP")
instructions.push_args(index)
instructions.append('SWAP')
instructions.append("SWAP")
instructions.push_args(index - 1)
instructions.append('SWAP')
instructions.append("SWAP")
to_front(instructions, index - 1)

63
bindings/c/fdb_c.cpp
View File

@ -386,8 +386,7 @@ void setBlobFilePointer(FDBBGFilePointer* dest, const BlobFilePointerRef& source
dest->file_offset = source.offset;
dest->file_length = source.length;
dest->full_file_length = source.fullFileLength;
// FIXME: add version info to each source file pointer
dest->file_version = 0;
dest->file_version = source.fileVersion;
// handle encryption
if (source.cipherKeysCtx.present()) {
@ -717,6 +716,17 @@ extern "C" DLLEXPORT FDBFuture* fdb_database_blobbify_range(FDBDatabase* db,
.extractPtr());
}
extern "C" DLLEXPORT FDBFuture* fdb_database_blobbify_range_blocking(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length) {
return (FDBFuture*)(DB(db)
->blobbifyRangeBlocking(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
StringRef(end_key_name, end_key_name_length)))
.extractPtr());
}
extern "C" DLLEXPORT FDBFuture* fdb_database_unblobbify_range(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
@ -758,6 +768,25 @@ extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_verify_blob_rang
.extractPtr());
}
extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_flush_blob_range(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version) {
Optional<Version> rv;
if (version != latestVersion) {
rv = version;
}
return (FDBFuture*)(DB(db)
->flushBlobRange(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
StringRef(end_key_name, end_key_name_length)),
compact,
rv)
.extractPtr());
}
extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_client_status(FDBDatabase* db) {
return (FDBFuture*)(DB(db)->getClientStatus().extractPtr());
}
@ -799,6 +828,17 @@ extern "C" DLLEXPORT FDBFuture* fdb_tenant_blobbify_range(FDBTenant* tenant,
.extractPtr());
}
extern "C" DLLEXPORT FDBFuture* fdb_tenant_blobbify_range_blocking(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length) {
return (FDBFuture*)(TENANT(tenant)
->blobbifyRangeBlocking(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
StringRef(end_key_name, end_key_name_length)))
.extractPtr());
}
extern "C" DLLEXPORT FDBFuture* fdb_tenant_unblobbify_range(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
@ -840,6 +880,25 @@ extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_verify_blob_range(
.extractPtr());
}
extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_flush_blob_range(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version) {
Optional<Version> rv;
if (version != latestVersion) {
rv = version;
}
return (FDBFuture*)(TENANT(tenant)
->flushBlobRange(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length),
StringRef(end_key_name, end_key_name_length)),
compact,
rv)
.extractPtr());
}
extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_get_id(FDBTenant* tenant) {
return (FDBFuture*)(TENANT(tenant)->getId().extractPtr());
}

28
bindings/c/foundationdb/fdb_c.h
View File

@ -417,6 +417,12 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_blobbify_range(FDBDatabase*
uint8_t const* end_key_name,
int end_key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_blobbify_range_blocking(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_unblobbify_range(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
@ -437,6 +443,14 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_verify_blob_range(FDBDataba
int end_key_name_length,
int64_t version);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_flush_blob_range(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_client_status(FDBDatabase* db);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_tenant_create_transaction(FDBTenant* tenant,
@ -460,6 +474,12 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_blobbify_range(FDBTenant* ten
uint8_t const* end_key_name,
int end_key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_blobbify_range_blocking(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_unblobbify_range(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
@ -487,6 +507,14 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_verify_blob_range(FDBTenant*
int end_key_name_length,
int64_t version);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_flush_blob_range(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_get_id(FDBTenant* tenant);
DLLEXPORT void fdb_tenant_destroy(FDBTenant* tenant);

46
bindings/c/test/apitester/TesterApiWorkload.cpp
View File

@ -380,7 +380,6 @@ void ApiWorkload::setupBlobGranules(TTaskFct cont) {
void ApiWorkload::blobbifyTenant(std::optional<int> tenantId,
std::shared_ptr<std::atomic<int>> blobbifiedCount,
TTaskFct cont) {
auto retBlobbifyRange = std::make_shared<bool>(false);
execOperation(
[=](auto ctx) {
fdb::Key begin(1, '\x00');
@ -388,48 +387,17 @@ void ApiWorkload::blobbifyTenant(std::optional<int> tenantId,
info(fmt::format("setup: blobbifying {}: [\\x00 - \\xff)\n", debugTenantStr(tenantId)));
fdb::Future f = ctx->dbOps()->blobbifyRange(begin, end).eraseType();
ctx->continueAfter(f, [ctx, retBlobbifyRange, f]() {
*retBlobbifyRange = f.get<fdb::future_var::Bool>();
// wait for blobbification before returning
fdb::Future f = ctx->dbOps()->blobbifyRangeBlocking(begin, end).eraseType();
ctx->continueAfter(f, [ctx, f]() {
bool success = f.get<fdb::future_var::Bool>();
ASSERT(success);
ctx->done();
});
},
[=]() {
if (!*retBlobbifyRange) {
schedule([=]() { blobbifyTenant(tenantId, blobbifiedCount, cont); });
} else {
schedule([=]() { verifyTenant(tenantId, blobbifiedCount, cont); });
}
},
/*tenant=*/getTenant(tenantId),
/* failOnError = */ false);
}
void ApiWorkload::verifyTenant(std::optional<int> tenantId,
std::shared_ptr<std::atomic<int>> blobbifiedCount,
TTaskFct cont) {
auto retVerifyVersion = std::make_shared<int64_t>(-1);
execOperation(
[=](auto ctx) {
fdb::Key begin(1, '\x00');
fdb::Key end(1, '\xff');
info(fmt::format("setup: verifying {}: [\\x00 - \\xff)\n", debugTenantStr(tenantId)));
fdb::Future f = ctx->dbOps()->verifyBlobRange(begin, end, /*latest_version*/ -2).eraseType();
ctx->continueAfter(f, [ctx, retVerifyVersion, f]() {
*retVerifyVersion = f.get<fdb::future_var::Int64>();
ctx->done();
});
},
[=]() {
if (*retVerifyVersion == -1) {
schedule([=]() { verifyTenant(tenantId, blobbifiedCount, cont); });
} else {
if (blobbifiedCount->fetch_sub(1) == 1) {
schedule(cont);
}
if (blobbifiedCount->fetch_sub(1) == 1) {
schedule(cont);
}
},
/*tenant=*/getTenant(tenantId),

1
bindings/c/test/apitester/TesterApiWorkload.h
View File

@ -135,7 +135,6 @@ protected:
// Generic BlobGranules setup.
void setupBlobGranules(TTaskFct cont);
void blobbifyTenant(std::optional<int> tenantId, std::shared_ptr<std::atomic<int>> blobbifiedCount, TTaskFct cont);
void verifyTenant(std::optional<int> tenantId, std::shared_ptr<std::atomic<int>> blobbifiedCount, TTaskFct cont);
private:
void populateDataTx(TTaskFct cont, std::optional<int> tenantId);

65
bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp
View File

@ -37,6 +37,9 @@ public:
if (Random::get().randomInt(0, 1) == 0) {
excludedOpTypes.push_back(OP_CLEAR_RANGE);
}
if (Random::get().randomInt(0, 1) == 0) {
excludedOpTypes.push_back(OP_FLUSH);
}
}
private:
@ -51,7 +54,8 @@ private:
OP_GET_BLOB_RANGES,
OP_VERIFY,
OP_READ_DESC,
OP_LAST = OP_READ_DESC
OP_FLUSH,
OP_LAST = OP_FLUSH
};
std::vector<OpType> excludedOpTypes;
@ -303,7 +307,10 @@ private:
fdb::native::FDBReadBlobGranuleContext& bgCtx,
fdb::GranuleFilePointer snapshotFile,
fdb::KeyRange keyRange,
fdb::native::FDBBGTenantPrefix const* tenantPrefix) {
fdb::native::FDBBGTenantPrefix const* tenantPrefix,
int64_t& prevFileVersion) {
ASSERT(snapshotFile.fileVersion > prevFileVersion);
prevFileVersion = snapshotFile.fileVersion;
if (validatedFiles.contains(snapshotFile.filename)) {
return;
}
@ -339,7 +346,10 @@ private:
fdb::GranuleFilePointer deltaFile,
fdb::KeyRange keyRange,
fdb::native::FDBBGTenantPrefix const* tenantPrefix,
int64_t& lastDFMaxVersion) {
int64_t& lastDFMaxVersion,
int64_t& prevFileVersion) {
ASSERT(deltaFile.fileVersion > prevFileVersion);
prevFileVersion = deltaFile.fileVersion;
if (validatedFiles.contains(deltaFile.filename)) {
return;
}
@ -380,6 +390,9 @@ private:
}
lastDFMaxVersion = std::max(lastDFMaxVersion, thisDFMaxVersion);
// can be higher due to empty versions but must not be lower
ASSERT(lastDFMaxVersion <= prevFileVersion);
// TODO have delta mutations update map
}
@ -392,22 +405,28 @@ private:
ASSERT(desc.keyRange.beginKey < desc.keyRange.endKey);
ASSERT(tenantId.has_value() == desc.tenantPrefix.present);
// beginVersion of zero means snapshot present
int64_t prevFileVersion = 0;
// validate snapshot file
ASSERT(desc.snapshotFile.has_value());
if (BG_API_DEBUG_VERBOSE) {
info(fmt::format("Loading snapshot file {0}\n", fdb::toCharsRef(desc.snapshotFile->filename)));
}
validateSnapshotData(ctx, bgCtx, *desc.snapshotFile, desc.keyRange, &desc.tenantPrefix);
validateSnapshotData(ctx, bgCtx, *desc.snapshotFile, desc.keyRange, &desc.tenantPrefix, prevFileVersion);
// validate delta files
int64_t lastDFMaxVersion = 0;
for (int i = 0; i < desc.deltaFiles.size(); i++) {
validateDeltaData(ctx, bgCtx, desc.deltaFiles[i], desc.keyRange, &desc.tenantPrefix, lastDFMaxVersion);
validateDeltaData(
ctx, bgCtx, desc.deltaFiles[i], desc.keyRange, &desc.tenantPrefix, lastDFMaxVersion, prevFileVersion);
}
// validate memory mutations
int64_t lastVersion = 0;
if (desc.memoryMutations.size()) {
ASSERT(desc.memoryMutations.front().version > lastDFMaxVersion);
ASSERT(desc.memoryMutations.front().version > prevFileVersion);
}
int64_t lastVersion = prevFileVersion;
for (int i = 0; i < desc.memoryMutations.size(); i++) {
fdb::GranuleMutation& m = desc.memoryMutations[i];
ASSERT(m.type == 0 || m.type == 1);
@ -494,6 +513,33 @@ private:
getTenant(tenantId));
}
void randomFlushOp(TTaskFct cont, std::optional<int> tenantId) {
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::native::fdb_bool_t compact = Random::get().randomBool(0.5);
auto result = std::make_shared<bool>(false);
debugOp(compact ? "Flush" : "Compact", keyRange, tenantId, "starting");
execOperation(
[keyRange, compact, result](auto ctx) {
fdb::Future f =
ctx->dbOps()
->flushBlobRange(keyRange.beginKey, keyRange.endKey, compact, -2 /* latest version*/)
.eraseType();
ctx->continueAfter(f, [ctx, result, f]() {
*result = f.get<fdb::future_var::Bool>();
ctx->done();
});
},
[this, keyRange, compact, result, tenantId, cont]() {
ASSERT(*result);
debugOp(compact ? "Flush " : "Compact ", keyRange, tenantId, "Complete");
schedule(cont);
},
getTenant(tenantId),
/* failOnError = */ false);
}
void randomOperation(TTaskFct cont) override {
std::optional<int> tenantId = randomTenant();
@ -530,6 +576,13 @@ private:
case OP_READ_DESC:
randomReadDescription(cont, tenantId);
break;
case OP_FLUSH:
randomFlushOp(cont, tenantId);
// don't do too many flushes because they're expensive
if (Random::get().randomInt(0, 1) == 0) {
excludedOpTypes.push_back(OP_FLUSH);
}
break;
}
}
};

143
bindings/c/test/apitester/TesterBlobGranuleErrorsWorkload.cpp
View File

@ -44,7 +44,9 @@ private:
OP_CANCEL_BLOBBIFY,
OP_CANCEL_UNBLOBBIFY,
OP_CANCEL_PURGE,
OP_LAST = OP_CANCEL_PURGE
OP_CANCEL_FLUSH,
OP_FLUSH_TOO_OLD,
OP_LAST = OP_FLUSH_TOO_OLD
};
void setup(TTaskFct cont) override { setupBlobGranules(cont); }
@ -122,14 +124,10 @@ private:
void randomPurgeUnalignedOp(TTaskFct cont) {
// blobbify/unblobbify need to be aligned to blob range boundaries, so this should always fail
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
execOperation(
[this, begin, end](auto ctx) {
fdb::Future f = ctx->db().purgeBlobGranules(begin, end, -2, false).eraseType();
[this, keyRange](auto ctx) {
fdb::Future f = ctx->db().purgeBlobGranules(keyRange.beginKey, keyRange.endKey, -2, false).eraseType();
ctx->continueAfter(
f,
[this, ctx, f]() {
@ -144,16 +142,12 @@ private:
void randomBlobbifyUnalignedOp(bool blobbify, TTaskFct cont) {
// blobbify/unblobbify need to be aligned to blob range boundaries, so this should always return false
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
auto success = std::make_shared<bool>(false);
execOperation(
[begin, end, blobbify, success](auto ctx) {
fdb::Future f = blobbify ? ctx->db().blobbifyRange(begin, end).eraseType()
: ctx->db().unblobbifyRange(begin, end).eraseType();
[keyRange, blobbify, success](auto ctx) {
fdb::Future f = blobbify ? ctx->db().blobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType()
: ctx->db().unblobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType();
ctx->continueAfter(
f,
[ctx, f, success]() {
@ -169,103 +163,116 @@ private:
}
void randomCancelGetGranulesOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
execTransaction(
[begin, end](auto ctx) {
fdb::Future f = ctx->tx().getBlobGranuleRanges(begin, end, 1000).eraseType();
[keyRange](auto ctx) {
fdb::Future f = ctx->tx().getBlobGranuleRanges(keyRange.beginKey, keyRange.endKey, 1000).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}
void randomCancelGetRangesOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
execOperation(
[begin, end](auto ctx) {
fdb::Future f = ctx->db().listBlobbifiedRanges(begin, end, 1000).eraseType();
[keyRange](auto ctx) {
fdb::Future f = ctx->db().listBlobbifiedRanges(keyRange.beginKey, keyRange.endKey, 1000).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}
void randomCancelVerifyOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
execOperation(
[begin, end](auto ctx) {
fdb::Future f = ctx->db().verifyBlobRange(begin, end, -2 /* latest version*/).eraseType();
[keyRange](auto ctx) {
fdb::Future f =
ctx->db().verifyBlobRange(keyRange.beginKey, keyRange.endKey, -2 /* latest version*/).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}
void randomCancelSummarizeOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
execTransaction(
[begin, end](auto ctx) {
fdb::Future f = ctx->tx().summarizeBlobGranules(begin, end, -2, 1000).eraseType();
[keyRange](auto ctx) {
fdb::Future f =
ctx->tx().summarizeBlobGranules(keyRange.beginKey, keyRange.endKey, -2, 1000).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}
void randomCancelBlobbifyOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
execOperation(
[begin, end](auto ctx) {
fdb::Future f = ctx->db().blobbifyRange(begin, end).eraseType();
[keyRange](auto ctx) {
fdb::Future f;
if (Random::get().randomBool(0.5)) {
f = ctx->db().blobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType();
} else {
f = ctx->db().blobbifyRangeBlocking(keyRange.beginKey, keyRange.endKey).eraseType();
}
ctx->done();
},
[this, cont]() { schedule(cont); });
}
void randomCancelUnblobbifyOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
execOperation(
[begin, end](auto ctx) {
fdb::Future f = ctx->db().unblobbifyRange(begin, end).eraseType();
[keyRange](auto ctx) {
fdb::Future f = ctx->db().unblobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}
void randomCancelPurgeOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
execOperation(
[begin, end](auto ctx) {
fdb::Future f = ctx->db().purgeBlobGranules(begin, end, -2, false).eraseType();
[keyRange](auto ctx) {
fdb::Future f = ctx->db().purgeBlobGranules(keyRange.beginKey, keyRange.endKey, -2, false).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}
void randomCancelFlushOp(TTaskFct cont) {
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::native::fdb_bool_t compact = Random::get().randomBool(0.5);
execOperation(
[keyRange, compact](auto ctx) {
fdb::Future f = ctx->db().flushBlobRange(keyRange.beginKey, keyRange.endKey, compact, -2).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}
void randomFlushTooOldOp(TTaskFct cont) {
fdb::KeyRange keyRange = randomNonEmptyKeyRange();
fdb::native::fdb_bool_t compact = Random::get().randomBool(0.5);
auto success = std::make_shared<bool>(false);
execOperation(
[keyRange, compact, success](auto ctx) {
// version of 1 should be too old
fdb::Future f = ctx->db().flushBlobRange(keyRange.beginKey, keyRange.endKey, compact, 1).eraseType();
ctx->continueAfter(
f,
[ctx, f, success]() {
*success = f.get<fdb::future_var::Bool>();
ctx->done();
},
true);
},
[this, cont, success]() {
ASSERT(!(*success));
schedule(cont);
});
}
void randomOperation(TTaskFct cont) override {
OpType txType = (OpType)Random::get().randomInt(0, OP_LAST);
switch (txType) {
@ -309,6 +316,12 @@ private:
case OP_CANCEL_PURGE:
randomCancelPurgeOp(cont);
break;
case OP_CANCEL_FLUSH:
randomCancelPurgeOp(cont);
break;
case OP_FLUSH_TOO_OLD:
randomCancelPurgeOp(cont);
break;
}
}
};

32
bindings/c/test/fdb_api.hpp
View File

@ -86,6 +86,7 @@ struct GranuleFilePointer {
int64_t offset;
int64_t length;
int64_t fullFileLength;
int64_t fileVersion;
// just keep raw data structures to pass to callbacks
native::FDBBGEncryptionCtx encryptionCtx;
@ -95,6 +96,7 @@ struct GranuleFilePointer {
offset = nativePointer.file_offset;
length = nativePointer.file_length;
fullFileLength = nativePointer.full_file_length;
fileVersion = nativePointer.file_version;
encryptionCtx = nativePointer.encryption_ctx;
}
};
@ -829,11 +831,13 @@ public:
virtual Transaction createTransaction() = 0;
virtual TypedFuture<future_var::Bool> blobbifyRange(KeyRef begin, KeyRef end) = 0;
virtual TypedFuture<future_var::Bool> blobbifyRangeBlocking(KeyRef begin, KeyRef end) = 0;
virtual TypedFuture<future_var::Bool> unblobbifyRange(KeyRef begin, KeyRef end) = 0;
virtual TypedFuture<future_var::KeyRangeRefArray> listBlobbifiedRanges(KeyRef begin,
KeyRef end,
int rangeLimit) = 0;
virtual TypedFuture<future_var::Int64> verifyBlobRange(KeyRef begin, KeyRef end, int64_t version) = 0;
virtual TypedFuture<future_var::Bool> flushBlobRange(KeyRef begin, KeyRef end, bool compact, int64_t version) = 0;
virtual TypedFuture<future_var::KeyRef> purgeBlobGranules(KeyRef begin,
KeyRef end,
int64_t version,
@ -901,6 +905,13 @@ public:
return native::fdb_tenant_blobbify_range(tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end));
}
TypedFuture<future_var::Bool> blobbifyRangeBlocking(KeyRef begin, KeyRef end) override {
if (!tenant)
throw std::runtime_error("blobbifyRangeBlocking() from null tenant");
return native::fdb_tenant_blobbify_range_blocking(
tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end));
}
TypedFuture<future_var::Bool> unblobbifyRange(KeyRef begin, KeyRef end) override {
if (!tenant)
throw std::runtime_error("unblobbifyRange() from null tenant");
@ -922,6 +933,13 @@ public:
tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end), version);
}
TypedFuture<future_var::Bool> flushBlobRange(KeyRef begin, KeyRef end, bool compact, int64_t version) override {
if (!tenant)
throw std::runtime_error("flushBlobRange() from null tenant");
return native::fdb_tenant_flush_blob_range(
tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end), compact, version);
}
TypedFuture<future_var::Int64> getId() {
if (!tenant)
throw std::runtime_error("getId() from null tenant");
@ -1026,12 +1044,26 @@ public:
db.get(), begin.data(), intSize(begin), end.data(), intSize(end), version);
}
TypedFuture<future_var::Bool> flushBlobRange(KeyRef begin, KeyRef end, bool compact, int64_t version) override {
if (!db)
throw std::runtime_error("flushBlobRange from null database");
return native::fdb_database_flush_blob_range(
db.get(), begin.data(), intSize(begin), end.data(), intSize(end), compact, version);
}
TypedFuture<future_var::Bool> blobbifyRange(KeyRef begin, KeyRef end) override {
if (!db)
throw std::runtime_error("blobbifyRange from null database");
return native::fdb_database_blobbify_range(db.get(), begin.data(), intSize(begin), end.data(), intSize(end));
}
TypedFuture<future_var::Bool> blobbifyRangeBlocking(KeyRef begin, KeyRef end) override {
if (!db)
throw std::runtime_error("blobbifyRangeBlocking from null database");
return native::fdb_database_blobbify_range_blocking(
db.get(), begin.data(), intSize(begin), end.data(), intSize(end));
}
TypedFuture<future_var::Bool> unblobbifyRange(KeyRef begin, KeyRef end) override {
if (!db)
throw std::runtime_error("unblobbifyRange from null database");

148
bindings/java/fdbJNI.cpp
View File

@ -942,6 +942,41 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1blobbi
return (jlong)f;
}
JNIEXPORT jlong JNICALL
Java_com_apple_foundationdb_FDBDatabase_Database_1blobbifyRangeBlocking(JNIEnv* jenv,
jobject,
jlong dbPtr,
jbyteArray beginKeyBytes,
jbyteArray endKeyBytes) {
if (!dbPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv);
return 0;
}
FDBDatabase* database = (FDBDatabase*)dbPtr;
uint8_t* beginKeyArr = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
if (!beginKeyArr) {
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
uint8_t* endKeyArr = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
if (!endKeyArr) {
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
FDBFuture* f = fdb_database_blobbify_range_blocking(
database, beginKeyArr, jenv->GetArrayLength(beginKeyBytes), endKeyArr, jenv->GetArrayLength(endKeyBytes));
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKeyArr, JNI_ABORT);
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1unblobbifyRange(JNIEnv* jenv,
jobject,
jlong dbPtr,
@ -1058,6 +1093,46 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1getCli
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1flushBlobRange(JNIEnv* jenv,
jobject,
jlong dbPtr,
jbyteArray beginKeyBytes,
jbyteArray endKeyBytes,
jboolean compact,
jlong version) {
if (!dbPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv);
return 0;
}
FDBDatabase* tr = (FDBDatabase*)dbPtr;
uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
if (!startKey) {
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
if (!endKey) {
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
FDBFuture* f = fdb_database_flush_blob_range(tr,
startKey,
jenv->GetArrayLength(beginKeyBytes),
endKey,
jenv->GetArrayLength(endKeyBytes),
(fdb_bool_t)compact,
version);
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT);
return (jlong)f;
}
JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv* jenv,
jobject,
jint predicate,
@ -1212,6 +1287,39 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1blobbifyRa
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1blobbifyRangeBlocking(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray beginKeyBytes,
jbyteArray endKeyBytes) {
if (!tPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv);
return 0;
}
FDBTenant* tenant = (FDBTenant*)tPtr;
uint8_t* beginKeyArr = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
if (!beginKeyArr) {
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
uint8_t* endKeyArr = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
if (!endKeyArr) {
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
FDBFuture* f = fdb_tenant_blobbify_range_blocking(
tenant, beginKeyArr, jenv->GetArrayLength(beginKeyBytes), endKeyArr, jenv->GetArrayLength(endKeyBytes));
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKeyArr, JNI_ABORT);
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1unblobbifyRange(JNIEnv* jenv,
jobject,
jlong tPtr,
@ -1313,6 +1421,46 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1verifyBlob
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1flushBlobRange(JNIEnv* jenv,
jobject,
jlong tPtr,
jbyteArray beginKeyBytes,
jbyteArray endKeyBytes,
jboolean compact,
jlong version) {
if (!tPtr || !beginKeyBytes || !endKeyBytes) {
throwParamNotNull(jenv);
return 0;
}
FDBTenant* tenant = (FDBTenant*)tPtr;
uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
if (!startKey) {
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
if (!endKey) {
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
if (!jenv->ExceptionOccurred())
throwRuntimeEx(jenv, "Error getting handle to native resources");
return 0;
}
FDBFuture* f = fdb_tenant_flush_blob_range(tenant,
startKey,
jenv->GetArrayLength(beginKeyBytes),
endKey,
jenv->GetArrayLength(endKeyBytes),
(fdb_bool_t)compact,
version);
jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT);
jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT);
return (jlong)f;
}
JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1getId(JNIEnv* jenv, jobject, jlong tPtr) {
if (!tPtr) {
throwParamNotNull(jenv);

47
bindings/java/src/integration/com/apple/foundationdb/BlobGranuleIntegrationTest.java
View File

@ -50,20 +50,6 @@ class BlobGranuleIntegrationTest {
}
}
void waitForVerify(Database db, Range blobRange) throws InterruptedException {
System.out.println("Verifying");
while (true) {
Long verifyVersion = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
if (verifyVersion != null && verifyVersion > 0) {
System.out.println("Verify succeeded @ " + verifyVersion);
return;
} else {
System.out.println("Verify failed, sleeping");
Thread.sleep(100);
}
}
}
@Test
void blobManagementFunctionsTest() throws Exception {
/*
@ -79,9 +65,12 @@ class BlobGranuleIntegrationTest {
Range blobRange = Range.startsWith(key);
try (Database db = fdb.open()) {
db.blobbifyRange(blobRange.begin, blobRange.end).join();
boolean blobbifySuccess = db.blobbifyRangeBlocking(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(blobbifySuccess);
waitForVerify(db, blobRange);
Long verifyVersion = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(verifyVersion >= 0);
// list blob ranges
KeyRangeArrayResult blobRanges = db.listBlobbifiedRanges(blobRange.begin, blobRange.end, 2).join();
@ -89,23 +78,41 @@ class BlobGranuleIntegrationTest {
Assertions.assertArrayEquals(blobRange.begin, blobRanges.getKeyRanges().get(0).begin);
Assertions.assertArrayEquals(blobRange.end, blobRanges.getKeyRanges().get(0).end);
boolean flushSuccess = db.flushBlobRange(blobRange.begin, blobRange.end, false).join();
Assertions.assertTrue(flushSuccess);
// verify after flush
Long verifyVersionAfterFlush = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(verifyVersionAfterFlush >= 0);
Assertions.assertTrue(verifyVersionAfterFlush >= verifyVersion);
boolean compactSuccess = db.flushBlobRange(blobRange.begin, blobRange.end, true).join();
Assertions.assertTrue(compactSuccess);
Long verifyVersionAfterCompact = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(verifyVersionAfterCompact >= 0);
Assertions.assertTrue(verifyVersionAfterCompact >= verifyVersionAfterFlush);
// purge/wait
byte[] purgeKey = db.purgeBlobGranules(blobRange.begin, blobRange.end, -2, false).join();
db.waitPurgeGranulesComplete(purgeKey).join();
// verify again
waitForVerify(db, blobRange);
Long verifyVersionAfterPurge = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(verifyVersionAfterPurge >= 0);
Assertions.assertTrue(verifyVersionAfterPurge >= verifyVersionAfterCompact);
// force purge/wait
byte[] forcePurgeKey = db.purgeBlobGranules(blobRange.begin, blobRange.end, -2, true).join();
db.waitPurgeGranulesComplete(forcePurgeKey).join();
// check verify fails
Long verifyVersion = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
Assertions.assertEquals(-1, verifyVersion);
Long verifyVersionLast = db.verifyBlobRange(blobRange.begin, blobRange.end).join();
Assertions.assertEquals(-1, verifyVersionLast);
// unblobbify
db.unblobbifyRange(blobRange.begin, blobRange.end).join();
boolean unblobbifySuccess = db.unblobbifyRange(blobRange.begin, blobRange.end).join();
Assertions.assertTrue(unblobbifySuccess);
System.out.println("Blob granule management tests complete!");
}

65
bindings/java/src/main/com/apple/foundationdb/Database.java
View File

@ -246,6 +246,31 @@ public interface Database extends AutoCloseable, TransactionContext {
*/
CompletableFuture<Boolean> blobbifyRange(byte[] beginKey, byte[] endKey, Executor e);
/**
* Runs {@link #blobbifyRange(byte[] beginKey, byte[] endKey, boolean wait)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param wait wait for blobbification to complete
* @return if the recording of the range was successful
*/
default CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey) {
return blobbifyRangeBlocking(beginKey, endKey, getExecutor());
}
/**
* Sets a range to be blobbified in the database. Must be a completely unblobbified range.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param wait wait for blobbification to complete
* @param e the {@link Executor} to use for asynchronous callbacks
* @return if the recording of the range was successful
*/
CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e);
/**
* Runs {@link #unblobbifyRange(byte[] beginKey, byte[] endKey)} on the default executor.
*
@ -331,6 +356,46 @@ public interface Database extends AutoCloseable, TransactionContext {
*/
CompletableFuture<Long> verifyBlobRange(byte[] beginKey, byte[] endKey, long version, Executor e);
/**
* Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
*
* @return a future with a boolean for success or failure
*/
default CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact) {
return flushBlobRange(beginKey, endKey, compact, -2, getExecutor());
}
/**
* Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey, long version)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
* @param version version to flush at
*
* @return a future with a boolean for success or failure
*/
default CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version) {
return flushBlobRange(beginKey, endKey, compact, version, getExecutor());
}
/**
* Checks if a blob range is blobbified.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
* @param version version to flush at
* @param e the {@link Executor} to use for asynchronous callbacks
*
* @return a future with a boolean for success or failure
*/
CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e);
/**
* Runs a read-only transactional function against this {@code Database} with retry logic.
* {@link Function#apply(Object) apply(ReadTransaction)} will be called on the

22
bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java
View File

@ -229,6 +229,16 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume
}
}
@Override
public CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e) {
pointerReadLock.lock();
try {
return new FutureBool(Database_blobbifyRangeBlocking(getPtr(), beginKey, endKey), e);
} finally {
pointerReadLock.unlock();
}
}
@Override
public CompletableFuture<Boolean> unblobbifyRange(byte[] beginKey, byte[] endKey, Executor e) {
pointerReadLock.lock();
@ -259,6 +269,16 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume
}
}
@Override
public CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e) {
pointerReadLock.lock();
try {
return new FutureBool(Database_flushBlobRange(getPtr(), beginKey, endKey, compact, version), e);
} finally {
pointerReadLock.unlock();
}
}
@Override
public Executor getExecutor() {
return executor;
@ -287,8 +307,10 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume
private native long Database_purgeBlobGranules(long cPtr, byte[] beginKey, byte[] endKey, long purgeVersion, boolean force);
private native long Database_waitPurgeGranulesComplete(long cPtr, byte[] purgeKey);
private native long Database_blobbifyRange(long cPtr, byte[] beginKey, byte[] endKey);
private native long Database_blobbifyRangeBlocking(long cPtr, byte[] beginKey, byte[] endKey);
private native long Database_unblobbifyRange(long cPtr, byte[] beginKey, byte[] endKey);
private native long Database_listBlobbifiedRanges(long cPtr, byte[] beginKey, byte[] endKey, int rangeLimit);
private native long Database_verifyBlobRange(long cPtr, byte[] beginKey, byte[] endKey, long version);
private native long Database_flushBlobRange(long cPtr, byte[] beginKey, byte[] endKey, boolean compact, long version);
private native long Database_getClientStatus(long cPtr);
}

22
bindings/java/src/main/com/apple/foundationdb/FDBTenant.java
View File

@ -170,6 +170,16 @@ class FDBTenant extends NativeObjectWrapper implements Tenant {
}
}
@Override
public CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e) {
pointerReadLock.lock();
try {
return new FutureBool(Tenant_blobbifyRangeBlocking(getPtr(), beginKey, endKey), e);
} finally {
pointerReadLock.unlock();
}
}
@Override
public CompletableFuture<Boolean> unblobbifyRange(byte[] beginKey, byte[] endKey, Executor e) {
pointerReadLock.lock();
@ -200,6 +210,16 @@ class FDBTenant extends NativeObjectWrapper implements Tenant {
}
}
@Override
public CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e) {
pointerReadLock.lock();
try {
return new FutureBool(Tenant_flushBlobRange(getPtr(), beginKey, endKey, compact, version), e);
} finally {
pointerReadLock.unlock();
}
}
@Override
public CompletableFuture<Long> getId(Executor e) {
pointerReadLock.lock();
@ -233,8 +253,10 @@ class FDBTenant extends NativeObjectWrapper implements Tenant {
private native long Tenant_purgeBlobGranules(long cPtr, byte[] beginKey, byte[] endKey, long purgeVersion, boolean force);
private native long Tenant_waitPurgeGranulesComplete(long cPtr, byte[] purgeKey);
private native long Tenant_blobbifyRange(long cPtr, byte[] beginKey, byte[] endKey);
private native long Tenant_blobbifyRangeBlocking(long cPtr, byte[] beginKey, byte[] endKey);
private native long Tenant_unblobbifyRange(long cPtr, byte[] beginKey, byte[] endKey);
private native long Tenant_listBlobbifiedRanges(long cPtr, byte[] beginKey, byte[] endKey, int rangeLimit);
private native long Tenant_verifyBlobRange(long cPtr, byte[] beginKey, byte[] endKey, long version);
private native long Tenant_flushBlobRange(long cPtr, byte[] beginKey, byte[] endKey, boolean compact, long version);
private native long Tenant_getId(long cPtr);
}

65
bindings/java/src/main/com/apple/foundationdb/Tenant.java
View File

@ -333,6 +333,31 @@ public interface Tenant extends AutoCloseable, TransactionContext {
*/
CompletableFuture<Boolean> blobbifyRange(byte[] beginKey, byte[] endKey, Executor e);
/**
* Runs {@link #blobbifyRange(byte[] beginKey, byte[] endKey, boolean wait)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param wait wait for blobbification to complete
* @return if the recording of the range was successful
*/
default CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey) {
return blobbifyRangeBlocking(beginKey, endKey, getExecutor());
}
/**
* Sets a range to be blobbified in this tenant. Must be a completely unblobbified range.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param wait wait for blobbification to complete
* @param e the {@link Executor} to use for asynchronous callbacks
* @return if the recording of the range was successful
*/
CompletableFuture<Boolean> blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e);
/**
* Runs {@link #unblobbifyRange(byte[] beginKey, byte[] endKey)} on the default executor.
*
@ -418,6 +443,46 @@ public interface Tenant extends AutoCloseable, TransactionContext {
*/
CompletableFuture<Long> verifyBlobRange(byte[] beginKey, byte[] endKey, long version, Executor e);
/**
* Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
*
* @return a future with a boolean for success or failure
*/
default CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact) {
return flushBlobRange(beginKey, endKey, compact, -2, getExecutor());
}
/**
* Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey, long version)} on the default executor.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
* @param version version to flush at
*
* @return a future with a boolean for success or failure
*/
default CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version) {
return flushBlobRange(beginKey, endKey, compact, version, getExecutor());
}
/**
* Checks if a blob range is blobbified.
*
* @param beginKey start of the key range
* @param endKey end of the key range
* @param compact force compact or just flush
* @param version version to flush at
* @param e the {@link Executor} to use for asynchronous callbacks
*
* @return a future with a boolean for success or failure
*/
CompletableFuture<Boolean> flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e);
/**
* Runs {@link #getId()} on the default executor.
*

20
fdbcli/BlobRangeCommand.actor.cpp
View File

@ -24,7 +24,6 @@
#include "fdbclient/IClientApi.h"
#include "fdbclient/ManagementAPI.actor.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/BlobGranuleRequest.actor.h"
#include "flow/Arena.h"
#include "flow/FastRef.h"
@ -90,17 +89,16 @@ ACTOR Future<Void> doBlobCheck(Database db, Key startKey, Key endKey, Optional<V
}
ACTOR Future<Void> doBlobFlush(Database db, Key startKey, Key endKey, Optional<Version> version, bool compact) {
// TODO make DB function?
state Version flushVersion;
if (version.present()) {
flushVersion = version.get();
} else {
wait(store(flushVersion, getLatestReadVersion(db)));
}
state double elapsed = -timer_monotonic();
state KeyRange keyRange(KeyRangeRef(startKey, endKey));
bool result = wait(db->flushBlobRange(keyRange, compact, version));
elapsed += timer_monotonic();
KeyRange range(KeyRangeRef(startKey, endKey));
FlushGranuleRequest req(-1, range, flushVersion, compact);
wait(success(doBlobGranuleRequests(db, range, req, &BlobWorkerInterface::flushGranuleRequest)));
fmt::print("Blob Flush [{0} - {1}) {2} in {3:.6f} seconds\n",
startKey.printable(),
endKey.printable(),
result ? "succeeded" : "failed",
elapsed);
return Void();
}

83
fdbcli/IdempotencyIdsCommand.actor.cpp Normal file
View File

@ -0,0 +1,83 @@
/*
* IdempotencyIdsCommand.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbcli/fdbcli.actor.h"
#include "fdbclient/IdempotencyId.actor.h"
#include "fdbclient/JsonBuilder.h"
#include "fdbclient/json_spirit/json_spirit_reader_template.h"
#include "flow/actorcompiler.h" // This must be the last include
namespace {
Optional<double> parseAgeValue(StringRef token) {
try {
return std::stod(token.toString());
} catch (...) {
return {};
}
}
} // namespace
namespace fdb_cli {
ACTOR Future<bool> idempotencyIdsCommandActor(Database db, std::vector<StringRef> tokens) {
if (tokens.size() < 2 || tokens.size() > 3) {
printUsage(tokens[0]);
return false;
} else {
auto const action = tokens[1];
if (action == "status"_sr) {
if (tokens.size() != 2) {
printUsage(tokens[0]);
return false;
}
JsonBuilderObject status = wait(getIdmpKeyStatus(db));
fmt::print("{}\n", status.getJson());
return true;
} else if (action == "clear"_sr) {
if (tokens.size() != 3) {
printUsage(tokens[0]);
return false;
}
auto const age = parseAgeValue(tokens[2]);
if (!age.present()) {
printUsage(tokens[0]);
return false;
}
wait(cleanIdempotencyIds(db, age.get()));
fmt::print("Successfully cleared idempotency IDs.\n");
return true;
} else {
printUsage(tokens[0]);
return false;
}
}
}
CommandFactory idempotencyIdsCommandFactory(
"idempotencyids",
CommandHelp(
"idempotencyids [status | clear <min_age_seconds>]",
"View status of idempotency ids, or reclaim space used by idempotency ids older than the given age",
"View status of idempotency ids currently in the cluster, or reclaim space by clearing all the idempotency ids "
"older than min_age_seconds (which will expire all transaction versions older than this age)\n"));
} // namespace fdb_cli

8
fdbcli/fdbcli.actor.cpp
View File

@ -2137,6 +2137,14 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise, Reference<ClusterCo
continue;
}
if (tokencmp(tokens[0], "idempotencyids")) {
bool _result = wait(makeInterruptable(idempotencyIdsCommandActor(localDb, tokens)));
if (!_result) {
is_error = true;
}
continue;
}
fprintf(stderr, "ERROR: Unknown command `%s'. Try `help'?\n", formatStringRef(tokens[0]).c_str());
is_error = true;
}

2
fdbcli/include/fdbcli/fdbcli.actor.h
View File

@ -274,6 +274,8 @@ ACTOR Future<bool> tssqCommandActor(Reference<IDatabase> db, std::vector<StringR
ACTOR Future<bool> versionEpochCommandActor(Reference<IDatabase> db, Database cx, std::vector<StringRef> tokens);
// targetversion command
ACTOR Future<bool> targetVersionCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens);
// idempotencyids command
ACTOR Future<bool> idempotencyIdsCommandActor(Database cx, std::vector<StringRef> tokens);
} // namespace fdb_cli

1297
fdbcli/tests/fdbcli_tests.py
View File

File diff suppressed because it is too large Load Diff

113
fdbcli/tests/metacluster_fdbcli_tests.py
View File

@ -38,29 +38,7 @@ def enable_logging(level=logging.DEBUG):
return func_decorator
def run_command(*args):
commands = ["{}".format(args)]
print(commands)
try:
process = subprocess.run(commands, stdout=subprocess.PIPE, env=fdbcli_env, timeout=20)
return process.stdout.decode('utf-8').strip()
except subprocess.TimeoutExpired:
raise Exception('the command is stuck')
def run_fdbcli_command(cluster_file, *args):
command_template = [fdbcli_bin, '-C', "{}".format(cluster_file), '--exec']
commands = command_template + ["{}".format(' '.join(args))]
print(commands)
try:
# if the fdbcli command is stuck for more than 20 seconds, the database is definitely unavailable
process = subprocess.run(commands, stdout=subprocess.PIPE, env=fdbcli_env, timeout=20)
return process.stdout.decode('utf-8').strip()
except subprocess.TimeoutExpired:
raise Exception('The fdbcli command is stuck, database is unavailable')
def run_fdbcli_command_and_get_error(cluster_file, *args):
"""run the fdbcli statement: fdbcli --exec '<arg1> <arg2> ... <argN>'.
Returns:
@ -70,7 +48,10 @@ def run_fdbcli_command_and_get_error(cluster_file, *args):
commands = command_template + ["{}".format(' '.join(args))]
try:
process = subprocess.run(commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env, timeout=20)
return process.stdout.decode('utf-8').strip(), process.stderr.decode('utf-8').strip()
rc = process.returncode
out = process.stdout.decode('utf-8').strip()
err = process.stderr.decode('utf-8').strip()
return rc, out, err
except subprocess.TimeoutExpired:
raise Exception('The fdbcli command is stuck, database is unavailable')
@ -81,35 +62,46 @@ def get_cluster_connection_str(cluster_file_path):
return conn_str
def metacluster_create(cluster_file, name, tenant_id_prefix):
return run_fdbcli_command(cluster_file, "metacluster create_experimental", name, str(tenant_id_prefix))
@enable_logging()
def metacluster_create(logger, cluster_file, name, tenant_id_prefix):
rc, out, err = run_fdbcli_command(cluster_file, "metacluster create_experimental", name, str(tenant_id_prefix))
if rc != 0:
raise Exception(err)
logger.debug(out)
logger.debug('Metacluster {} created'.format(name))
def metacluster_register(management_cluster_file, data_cluster_file, name):
def metacluster_register(management_cluster_file, data_cluster_file, name, max_tenant_groups):
conn_str = get_cluster_connection_str(data_cluster_file)
return run_fdbcli_command(management_cluster_file, "metacluster register", name, "connection_string={}".format(
conn_str), 'max_tenant_groups=6')
rc, out, err = run_fdbcli_command(management_cluster_file,
"metacluster register",
name,
"connection_string={}".format(conn_str),
'max_tenant_groups={}'.format(max_tenant_groups))
if rc != 0:
raise Exception(err)
@enable_logging()
def setup_metacluster(logger, management_cluster, data_clusters):
def setup_metacluster(logger, management_cluster, data_clusters, max_tenant_groups_per_cluster):
management_cluster_file = management_cluster[0]
management_cluster_name = management_cluster[1]
tenant_id_prefix = random.randint(0, 32767)
output = metacluster_create(management_cluster_file, management_cluster_name, tenant_id_prefix)
logger.debug(output)
logger.debug('management cluster: {}'.format(management_cluster_name))
logger.debug('data clusters: {}'.format([name for (cf, name) in data_clusters]))
metacluster_create(management_cluster_file, management_cluster_name, tenant_id_prefix)
for (cf, name) in data_clusters:
output = metacluster_register(management_cluster_file, cf, name)
logger.debug(output)
metacluster_register(management_cluster_file, cf, name, max_tenant_groups=max_tenant_groups_per_cluster)
def metacluster_status(cluster_file):
return run_fdbcli_command(cluster_file, "metacluster status")
_, out, _ = run_fdbcli_command(cluster_file, "metacluster status")
return out
def setup_tenants(management_cluster_file, data_cluster_files, tenants):
for tenant in tenants:
output = run_fdbcli_command(management_cluster_file, 'tenant create', tenant)
_, output, _ = run_fdbcli_command(management_cluster_file, 'tenant create', tenant)
expected_output = 'The tenant `{}\' has been created'.format(tenant)
assert output == expected_output
@ -121,61 +113,74 @@ def configure_tenant(management_cluster_file, data_cluster_files, tenant, tenant
if assigned_cluster:
command = command + ' assigned_cluster={}'.format(assigned_cluster)
output, err = run_fdbcli_command_and_get_error(management_cluster_file, command)
_, output, err = run_fdbcli_command(management_cluster_file, command)
return output, err
@enable_logging()
def clear_database_and_tenants(logger, management_cluster_file, data_cluster_files):
def clear_database_and_tenants(management_cluster_file, data_cluster_files):
subcmd1 = 'writemode on'
subcmd2 = 'option on SPECIAL_KEY_SPACE_ENABLE_WRITES'
subcmd3 = 'clearrange ' '"" \\xff'
subcmd4 = 'clearrange \\xff\\xff/management/tenant/map/ \\xff\\xff/management/tenant/map0'
output = run_fdbcli_command(management_cluster_file, subcmd1, subcmd2, subcmd3, subcmd4)
logger.debug(output)
def run_tenant_test(management_cluster_file, data_cluster_files, test_func):
test_func(management_cluster_file, data_cluster_files)
clear_database_and_tenants(management_cluster_file, data_cluster_files)
run_fdbcli_command(management_cluster_file, subcmd1, subcmd2, subcmd3, subcmd4)
@enable_logging()
def clusters_status_test(logger, cluster_files):
def clusters_status_test(logger, cluster_files, max_tenant_groups_per_cluster):
logger.debug('Verifying no cluster is part of a metacluster')
for cf in cluster_files:
output = metacluster_status(cf)
assert output == "This cluster is not part of a metacluster"
logger.debug('Verified')
num_clusters = len(cluster_files)
names = ['meta_mgmt']
names.extend(['data{}'.format(i) for i in range(1, num_clusters)])
setup_metacluster([cluster_files[0], names[0]], zip(cluster_files[1:], names[1:]))
logger.debug('Setting up a metacluster')
setup_metacluster([cluster_files[0], names[0]], list(zip(cluster_files[1:], names[1:])),
max_tenant_groups_per_cluster=max_tenant_groups_per_cluster)
expected = """
number of data clusters: {}
tenant group capacity: {}
allocated tenant groups: 0
"""
expected = expected.format(num_clusters - 1, 12).strip()
expected = expected.format(num_clusters - 1, (num_clusters - 1) * max_tenant_groups_per_cluster).strip()
output = metacluster_status(cluster_files[0])
assert expected == output
logger.debug('Metacluster setup correctly')
for (cf, name) in zip(cluster_files[1:], names[1:]):
output = metacluster_status(cf)
expected = "This cluster \"{}\" is a data cluster within the metacluster named \"{" \
"}\"".format(name, names[0])
expected = "This cluster \"{}\" is a data cluster within the metacluster named \"{}\"".format(name, names[0])
assert expected == output
@enable_logging()
def configure_tenants_test_disableClusterAssignment(logger, cluster_files):
tenants = ['tenant1', 'tenant2']
logger.debug('Creating tenants {}'.format(tenants))
logger.debug('Tenants to create: {}'.format(tenants))
setup_tenants(cluster_files[0], cluster_files[1:], tenants)
# Once we reach here, the tenants have been created successfully
logger.debug('Tenants created: {}'.format(tenants))
for tenant in tenants:
out, err = configure_tenant(cluster_files[0], cluster_files[1:], tenant, assigned_cluster='cluster')
assert err == 'ERROR: Tenant configuration is invalid (2140)'
logger.debug('Tenants configured')
clear_database_and_tenants(cluster_files[0], cluster_files[1:])
logger.debug('Tenants cleared')
@enable_logging()
def test_main(logger):
logger.debug('Tests start')
# This must be the first test to run, since it sets up the metacluster that
# will be used throughout the test
clusters_status_test(cluster_files, max_tenant_groups_per_cluster=5)
configure_tenants_test_disableClusterAssignment(cluster_files)
logger.debug('Tests complete')
if __name__ == "__main__":
@ -197,8 +202,4 @@ if __name__ == "__main__":
fdbcli_bin = args.build_dir + '/bin/fdbcli'
# This must be the first test to run, since it sets up the metacluster that
# will be used throughout the test
clusters_status_test(cluster_files)
configure_tenants_test_disableClusterAssignment(cluster_files)
test_main()

16
fdbclient/BackupAgentBase.actor.cpp
View File

@ -262,7 +262,7 @@ bool validTenantAccess(std::map<int64_t, TenantName>* tenantMap,
}
int64_t tenantId = TenantInfo::INVALID_TENANT;
if (m.isEncrypted()) {
tenantId = m.encryptionHeader()->cipherTextDetails.encryptDomainId;
tenantId = m.encryptDomainId();
} else {
tenantId = TenantAPI::extractTenantIdFromMutation(m);
}
@ -383,12 +383,18 @@ ACTOR static Future<Void> decodeBackupLogValue(Arena* arena,
// Decrypt mutation ref if encrypted
if (logValue.isEncrypted()) {
encryptedLogValue = logValue;
state EncryptCipherDomainId domainId = logValue.encryptionHeader()->cipherTextDetails.encryptDomainId;
state EncryptCipherDomainId domainId = logValue.encryptDomainId();
Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
try {
TextAndHeaderCipherKeys cipherKeys =
wait(getEncryptCipherKeys(dbInfo, *logValue.encryptionHeader(), BlobCipherMetrics::RESTORE));
logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::BACKUP);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(
dbInfo, logValue.configurableEncryptionHeader(), BlobCipherMetrics::RESTORE));
logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::RESTORE);
} else {
TextAndHeaderCipherKeys cipherKeys = wait(
getEncryptCipherKeys(dbInfo, *logValue.encryptionHeader(), BlobCipherMetrics::RESTORE));
logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::RESTORE);
}
} catch (Error& e) {
// It's possible a tenant was deleted and the encrypt key fetch failed
TraceEvent(SevWarnAlways, "MutationLogRestoreEncryptKeyFetchFailed")

36
fdbclient/BlobCipher.cpp
View File

@ -40,7 +40,6 @@
#include "flow/Trace.h"
#include "flow/UnitTest.h"
#include "flow/xxhash.h"
#include "include/fdbclient/BlobCipher.h"
#include <chrono>
#include <cstring>
@ -211,6 +210,10 @@ EncryptAuthTokenMode BlobCipherEncryptHeaderRef::getAuthTokenMode() const {
flags);
}
EncryptCipherDomainId BlobCipherEncryptHeaderRef::getDomainId() const {
return std::visit([](auto& h) { return h.v1.cipherTextDetails.encryptDomainId; }, algoHeader);
}
void BlobCipherEncryptHeaderRef::validateEncryptionHeaderDetails(const BlobCipherDetails& textCipherDetails,
const BlobCipherDetails& headerCipherDetails,
const StringRef& ivRef) const {
@ -754,8 +757,7 @@ std::vector<Reference<BlobCipherKey>> BlobCipherKeyCache::getAllCiphers(const En
return keyIdCache->getAllCipherKeys();
}
namespace {
int getEncryptAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo) {
int getEncryptCurrentAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo) {
if (mode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
return CLIENT_KNOBS->ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION;
} else {
@ -768,7 +770,20 @@ int getEncryptAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAu
}
}
}
} // namespace
void BlobCipherDetails::validateCipherDetailsWithCipherKey(Reference<BlobCipherKey> cipherKey) {
if (!(baseCipherId == cipherKey->getBaseCipherId() && encryptDomainId == cipherKey->getDomainId() &&
salt == cipherKey->getSalt())) {
TraceEvent(SevWarn, "EncryptionHeaderCipherMismatch")
.detail("TextDomainId", cipherKey->getDomainId())
.detail("ExpectedTextDomainId", encryptDomainId)
.detail("TextBaseCipherId", cipherKey->getBaseCipherId())
.detail("ExpectedTextBaseCipherId", baseCipherId)
.detail("TextSalt", cipherKey->getSalt())
.detail("ExpectedTextSalt", salt);
throw encrypt_header_metadata_mismatch();
}
}
// EncryptBlobCipherAes265Ctr class methods
@ -896,8 +911,8 @@ void EncryptBlobCipherAes265Ctr::setCipherAlgoHeaderV1(const uint8_t* ciphertext
const BlobCipherEncryptHeaderFlagsV1& flags,
BlobCipherEncryptHeaderRef* headerRef) {
ASSERT_EQ(1,
getEncryptAlgoHeaderVersion((EncryptAuthTokenMode)flags.authTokenMode,
(EncryptAuthTokenAlgo)flags.authTokenAlgo));
getEncryptCurrentAlgoHeaderVersion((EncryptAuthTokenMode)flags.authTokenMode,
(EncryptAuthTokenAlgo)flags.authTokenAlgo));
if (flags.authTokenMode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
setCipherAlgoHeaderNoAuthV1(flags, headerRef);
@ -930,7 +945,7 @@ void EncryptBlobCipherAes265Ctr::updateEncryptHeader(const uint8_t* ciphertext,
updateEncryptHeaderFlagsV1(headerRef, &flags);
// update cipher algo header
int algoHeaderVersion = getEncryptAlgoHeaderVersion(authTokenMode, authTokenAlgo);
int algoHeaderVersion = getEncryptCurrentAlgoHeaderVersion(authTokenMode, authTokenAlgo);
ASSERT_EQ(algoHeaderVersion, 1);
setCipherAlgoHeaderV1(ciphertext, ciphertextLen, flags, headerRef);
}
@ -1045,9 +1060,10 @@ Reference<EncryptBuf> EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plainte
if (authTokenMode != ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) {
header->cipherHeaderDetails = headerCipherKeyOpt.get()->details();
} else {
header->cipherHeaderDetails.encryptDomainId = INVALID_ENCRYPT_DOMAIN_ID;
header->cipherHeaderDetails.baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID;
header->cipherHeaderDetails.salt = INVALID_ENCRYPT_RANDOM_SALT;
header->cipherHeaderDetails = BlobCipherDetails();
ASSERT_EQ(INVALID_ENCRYPT_DOMAIN_ID, header->cipherHeaderDetails.encryptDomainId);
ASSERT_EQ(INVALID_ENCRYPT_CIPHER_KEY_ID, header->cipherHeaderDetails.baseCipherId);
ASSERT_EQ(INVALID_ENCRYPT_RANDOM_SALT, header->cipherHeaderDetails.salt);
}
memcpy(&header->iv[0], &iv[0], AES_256_IV_LENGTH);

28
fdbclient/BlobGranuleFiles.cpp
View File

@ -1531,6 +1531,7 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
GranuleMaterializeStats& stats) {
// TODO REMOVE with early replying
ASSERT(readVersion == chunk.includedVersion);
Version lastFileVersion = 0;
// Arena to hold all allocations for applying deltas. Most of it, and the arenas produced by reading the files,
// will likely be tossed if there are a significant number of mutations, so we copy at the end instead of doing a
@ -1569,6 +1570,8 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
arena.dependsOn(streams.back().arena());
stats.snapshotRows += snapshotRows.size();
}
ASSERT_WE_THINK(lastFileVersion < chunk.snapshotFile.get().fileVersion);
lastFileVersion = chunk.snapshotFile.get().fileVersion;
} else {
ASSERT(!chunk.snapshotFile.present());
}
@ -1593,6 +1596,9 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
arena.dependsOn(streams.back().arena());
}
arena.dependsOn(deltaRows.arena());
ASSERT_WE_THINK(lastFileVersion < chunk.deltaFiles[deltaIdx].fileVersion);
lastFileVersion = chunk.deltaFiles[deltaIdx].fileVersion;
}
if (BG_READ_DEBUG) {
fmt::print("Applying {} memory deltas\n", chunk.newDeltas.size());
@ -1600,6 +1606,7 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk,
if (!chunk.newDeltas.empty()) {
stats.inputBytes += chunk.newDeltas.expectedSize();
// TODO REMOVE validation
ASSERT_WE_THINK(lastFileVersion < chunk.newDeltas.front().version);
ASSERT(beginVersion <= chunk.newDeltas.front().version);
ASSERT(readVersion >= chunk.newDeltas.back().version);
auto memoryRows = sortMemoryDeltas(chunk.newDeltas, chunk.keyRange, requestRange, beginVersion, readVersion);
@ -2188,7 +2195,8 @@ struct KeyValueGen {
int targetMutationsPerDelta;
KeyRange allRange;
Version version = 0;
// start at higher version to allow snapshot files to have version 1
Version version = 10;
// encryption/compression settings
// TODO: possibly different cipher keys or meta context per file?
@ -2551,7 +2559,7 @@ void checkDeltaRead(const KeyValueGen& kvGen,
deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), readVersion, ".delta");
Standalone<BlobGranuleChunkRef> chunk;
chunk.deltaFiles.emplace_back_deep(
chunk.arena(), filename, 0, serialized[0].size(), serialized[0].size(), kvGen.cipherKeys);
chunk.arena(), filename, 0, serialized[0].size(), serialized[0].size(), 1, kvGen.cipherKeys);
chunk.keyRange = kvGen.allRange;
chunk.includedVersion = readVersion;
chunk.snapshotVersion = invalidVersion;
@ -2672,8 +2680,13 @@ void checkGranuleRead(const KeyValueGen& kvGen,
if (beginVersion == 0) {
std::string snapshotFilename = randomBGFilename(
deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), 0, ".snapshot");
chunk.snapshotFile = BlobFilePointerRef(
chunk.arena(), snapshotFilename, 0, serializedSnapshot.size(), serializedSnapshot.size(), kvGen.cipherKeys);
chunk.snapshotFile = BlobFilePointerRef(chunk.arena(),
snapshotFilename,
0,
serializedSnapshot.size(),
serializedSnapshot.size(),
1,
kvGen.cipherKeys);
}
int deltaIdx = 0;
while (deltaIdx < serializedDeltas.size() && serializedDeltas[deltaIdx].first < beginVersion) {
@ -2684,7 +2697,8 @@ void checkGranuleRead(const KeyValueGen& kvGen,
std::string deltaFilename = randomBGFilename(
deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), readVersion, ".delta");
size_t fsize = serializedDeltas[deltaIdx].second.size();
chunk.deltaFiles.emplace_back_deep(chunk.arena(), deltaFilename, 0, fsize, fsize, kvGen.cipherKeys);
chunk.deltaFiles.emplace_back_deep(
chunk.arena(), deltaFilename, 0, fsize, fsize, serializedDeltas[deltaIdx].first, kvGen.cipherKeys);
deltaPtrsVector.push_back(serializedDeltas[deltaIdx].second);
if (serializedDeltas[deltaIdx].first >= readVersion) {
@ -3207,12 +3221,12 @@ void chunkFromFileSet(const FileSet& fileSet,
int numDeltaFiles) {
size_t snapshotSize = std::get<3>(fileSet.snapshotFile).size();
chunk.snapshotFile =
BlobFilePointerRef(chunk.arena(), std::get<0>(fileSet.snapshotFile), 0, snapshotSize, snapshotSize, keys);
BlobFilePointerRef(chunk.arena(), std::get<0>(fileSet.snapshotFile), 0, snapshotSize, snapshotSize, 1, keys);
for (int i = 0; i < numDeltaFiles; i++) {
size_t deltaSize = std::get<3>(fileSet.deltaFiles[i]).size();
chunk.deltaFiles.emplace_back_deep(
chunk.arena(), std::get<0>(fileSet.deltaFiles[i]), 0, deltaSize, deltaSize, keys);
chunk.arena(), std::get<0>(fileSet.deltaFiles[i]), 0, deltaSize, deltaSize, 2 + i, keys);
deltaPtrs[i] = std::get<2>(fileSet.deltaFiles[i]);
}

4
fdbclient/ClientKnobs.cpp
View File

@ -24,6 +24,7 @@
#include "fdbclient/Tenant.h"
#include "flow/IRandom.h"
#include "flow/UnitTest.h"
#include "flow/flow.h"
#define init(...) KNOB_FN(__VA_ARGS__, INIT_ATOMIC_KNOB, INIT_KNOB)(__VA_ARGS__)
@ -302,8 +303,9 @@ void ClientKnobs::initialize(Randomize randomize) {
init( CLIENT_ENABLE_USING_CLUSTER_ID_KEY, false );
init( ENABLE_ENCRYPTION_CPU_TIME_LOGGING, false );
init( SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS, true );
init( SIMULATION_EKP_TENANT_IDS_TO_DROP, "-1" );
init( ENABLE_CONFIGURABLE_ENCRYPTION, false );
init( ENABLE_CONFIGURABLE_ENCRYPTION, true );
init( ENCRYPT_HEADER_FLAGS_VERSION, 1 );
init( ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION, 1 );
init( ENCRYPT_HEADER_AES_CTR_AES_CMAC_AUTH_VERSION, 1 );

177
fdbclient/FileBackupAgent.actor.cpp
View File

@ -61,6 +61,7 @@
#include <algorithm>
#include <unordered_map>
#include <utility>
#include <variant>
#include "flow/actorcompiler.h" // This must be the last #include.
@ -551,14 +552,13 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
struct Options {
constexpr static FileIdentifier file_identifier = 3152016;
// TODO: Compression is not currently supported so this should always be false
bool compressionEnabled = false;
bool configurableEncryptionEnabled = false;
Options() {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, compressionEnabled);
serializer(ar, configurableEncryptionEnabled);
}
};
@ -575,54 +575,44 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
wPtr = mutateString(buffer);
}
static void validateEncryptionHeader(Optional<Reference<BlobCipherKey>> headerCipherKey,
Reference<BlobCipherKey> textCipherKey,
BlobCipherEncryptHeader& header) {
// Validate encryption header 'cipherHeader' details
if (header.cipherHeaderDetails.isValid() &&
(!headerCipherKey.present() || header.cipherHeaderDetails != headerCipherKey.get()->details())) {
TraceEvent(SevWarn, "EncryptionHeader_CipherHeaderMismatch")
.detail("HeaderDomainId", headerCipherKey.get()->getDomainId())
.detail("ExpectedHeaderDomainId", header.cipherHeaderDetails.encryptDomainId)
.detail("HeaderBaseCipherId", headerCipherKey.get()->getBaseCipherId())
.detail("ExpectedHeaderBaseCipherId", header.cipherHeaderDetails.baseCipherId)
.detail("HeaderSalt", headerCipherKey.get()->getSalt())
.detail("ExpectedHeaderSalt", header.cipherHeaderDetails.salt);
throw encrypt_header_metadata_mismatch();
}
// Validate encryption text 'cipherText' details sanity
if (!header.cipherTextDetails.isValid() || header.cipherTextDetails != textCipherKey->details()) {
TraceEvent(SevWarn, "EncryptionHeader_CipherTextMismatch")
.detail("TextDomainId", textCipherKey->getDomainId())
.detail("ExpectedTextDomainId", header.cipherTextDetails.encryptDomainId)
.detail("TextBaseCipherId", textCipherKey->getBaseCipherId())
.detail("ExpectedTextBaseCipherId", header.cipherTextDetails.baseCipherId)
.detail("TextSalt", textCipherKey->getSalt())
.detail("ExpectedTextSalt", header.cipherTextDetails.salt);
throw encrypt_header_metadata_mismatch();
}
}
ACTOR static Future<StringRef> decryptImpl(Database cx,
BlobCipherEncryptHeader header,
const uint8_t* dataP,
int64_t dataLen,
Arena* arena) {
ACTOR static Future<StringRef> decryptImpl(
Database cx,
std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> headerVariant,
const uint8_t* dataP,
int64_t dataLen,
Arena* arena) {
Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, header, BlobCipherMetrics::RESTORE));
validateEncryptionHeader(cipherKeys.cipherHeaderKey, cipherKeys.cipherTextKey, header);
DecryptBlobCipherAes256Ctr decryptor(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.iv, BlobCipherMetrics::BACKUP);
return decryptor.decrypt(dataP, dataLen, header, *arena)->toStringRef();
if (std::holds_alternative<BlobCipherEncryptHeaderRef>(headerVariant)) { // configurable encryption
state BlobCipherEncryptHeaderRef headerRef = std::get<BlobCipherEncryptHeaderRef>(headerVariant);
TextAndHeaderCipherKeys cipherKeys =
wait(getEncryptCipherKeys(dbInfo, headerRef, BlobCipherMetrics::RESTORE));
EncryptHeaderCipherDetails cipherDetails = headerRef.getCipherDetails();
cipherDetails.textCipherDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherTextKey);
if (cipherDetails.headerCipherDetails.present()) {
cipherDetails.headerCipherDetails.get().validateCipherDetailsWithCipherKey(cipherKeys.cipherHeaderKey);
}
DecryptBlobCipherAes256Ctr decryptor(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, headerRef.getIV(), BlobCipherMetrics::RESTORE);
return decryptor.decrypt(dataP, dataLen, headerRef, *arena);
} else {
state BlobCipherEncryptHeader header = std::get<BlobCipherEncryptHeader>(headerVariant);
TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, header, BlobCipherMetrics::RESTORE));
header.cipherTextDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherTextKey);
if (header.cipherHeaderDetails.isValid()) {
header.cipherHeaderDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherHeaderKey);
}
DecryptBlobCipherAes256Ctr decryptor(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.iv, BlobCipherMetrics::RESTORE);
return decryptor.decrypt(dataP, dataLen, header, *arena)->toStringRef();
}
}
static Future<StringRef> decrypt(Database cx,
BlobCipherEncryptHeader headerS,
std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> header,
const uint8_t* dataP,
int64_t dataLen,
Arena* arena) {
return decryptImpl(cx, headerS, dataP, dataLen, arena);
return decryptImpl(cx, header, dataP, dataLen, arena);
}
ACTOR static Future<Reference<BlobCipherKey>> refreshKey(EncryptedRangeFileWriter* self,
@ -655,12 +645,26 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
AES_256_IV_LENGTH,
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
BlobCipherMetrics::BACKUP);
Arena arena;
int64_t payloadSize = self->wPtr - self->dataPayloadStart;
auto encryptedData = encryptor.encrypt(self->dataPayloadStart, payloadSize, self->encryptHeader, arena);
StringRef encryptedData;
if (self->options.configurableEncryptionEnabled) {
BlobCipherEncryptHeaderRef headerRef;
encryptedData = encryptor.encrypt(self->dataPayloadStart, payloadSize, &headerRef, *self->arena);
Standalone<StringRef> serialized = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
self->arena->dependsOn(serialized.arena());
ASSERT(serialized.size() == self->encryptHeader.size());
std::memcpy(mutateString(self->encryptHeader), serialized.begin(), self->encryptHeader.size());
} else {
BlobCipherEncryptHeader header;
encryptedData =
encryptor.encrypt(self->dataPayloadStart, payloadSize, &header, *self->arena)->toStringRef();
StringRef encryptHeaderStringRef = BlobCipherEncryptHeader::toStringRef(header, *self->arena);
ASSERT(encryptHeaderStringRef.size() == self->encryptHeader.size());
std::memcpy(mutateString(self->encryptHeader), encryptHeaderStringRef.begin(), self->encryptHeader.size());
}
// re-write encrypted data to buffer
std::memcpy(self->dataPayloadStart, encryptedData->begin(), payloadSize);
std::memcpy(self->dataPayloadStart, encryptedData.begin(), payloadSize);
return Void();
}
@ -767,13 +771,32 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
copyToBuffer(self, (uint8_t*)&self->fileVersion, sizeof(self->fileVersion));
// write options struct
self->options.configurableEncryptionEnabled = CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION;
Value serialized =
ObjectWriter::toValue(self->options, IncludeVersion(ProtocolVersion::withEncryptedSnapshotBackupFile()));
appendStringRefWithLenToBuffer(self, &serialized);
// calculate encryption header size
uint32_t headerSize = 0;
if (self->options.configurableEncryptionEnabled) {
EncryptAuthTokenMode authTokenMode =
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE);
EncryptAuthTokenAlgo authTokenAlgo = getAuthTokenAlgoFromMode(authTokenMode);
headerSize = BlobCipherEncryptHeaderRef::getHeaderSize(
CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION,
getEncryptCurrentAlgoHeaderVersion(authTokenMode, authTokenAlgo),
ENCRYPT_CIPHER_MODE_AES_256_CTR,
authTokenMode,
authTokenAlgo);
} else {
headerSize = BlobCipherEncryptHeader::headerSize;
}
ASSERT(headerSize > 0);
// write header size to buffer
copyToBuffer(self, (uint8_t*)&headerSize, sizeof(headerSize));
// leave space for encryption header
self->encryptHeader = (BlobCipherEncryptHeader*)self->wPtr;
self->wPtr += BlobCipherEncryptHeader::headerSize;
self->encryptHeader = StringRef(self->wPtr, headerSize);
self->wPtr += headerSize;
self->dataPayloadStart = self->wPtr;
// If this is NOT the first block then write duplicate stuff needed from last block
@ -913,7 +936,7 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter {
private:
Standalone<StringRef> buffer;
uint8_t* wPtr;
BlobCipherEncryptHeader* encryptHeader;
StringRef encryptHeader;
uint8_t* dataPayloadStart;
int64_t blockEnd;
uint32_t fileVersion;
@ -1050,7 +1073,7 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
Standalone<VectorRef<KeyValueRef>>* results,
bool encryptedBlock,
EncryptionAtRestMode encryptMode,
Optional<BlobCipherEncryptHeader> encryptHeader,
Optional<int64_t> blockDomainId,
Optional<Reference<TenantEntryCache<Void>>> tenantCache) {
// Read begin key, if this fails then block was invalid.
state uint32_t kLen = reader->consumeNetworkUInt32();
@ -1068,9 +1091,9 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
// make sure that all keys in a block belong to exactly one tenant,
// unless its the last key in which case it can be a truncated (different) tenant prefix
if (encryptedBlock && g_network && g_network->isSimulated() &&
!isReservedEncryptDomain(encryptHeader.get().cipherTextDetails.encryptDomainId)) {
ASSERT(encryptHeader.present());
ASSERT(!encryptedBlock || blockDomainId.present());
if (CLIENT_KNOBS->SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS && encryptedBlock && g_network &&
g_network->isSimulated() && !isReservedEncryptDomain(blockDomainId.get())) {
state KeyRef curKey = KeyRef(k, kLen);
if (!prevDomainId.present()) {
EncryptCipherDomainId domainId =
@ -1088,7 +1111,7 @@ ACTOR static Future<Void> decodeKVPairs(StringRefReader* reader,
}
// make sure that all keys (except possibly the last key) in a block are encrypted using the correct key
if (!prevKey.empty()) {
ASSERT(prevDomainId.get() == encryptHeader.get().cipherTextDetails.encryptDomainId);
ASSERT(prevDomainId.get() == blockDomainId.get());
}
prevKey = curKey;
prevDomainId = curDomainId;
@ -1152,15 +1175,14 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
wait(tenantCache.get()->init());
}
state EncryptionAtRestMode encryptMode = config.encryptionAtRestMode;
state int64_t blockTenantId = TenantInfo::INVALID_TENANT;
state int64_t blockDomainId = TenantInfo::INVALID_TENANT;
try {
// Read header, currently only decoding BACKUP_AGENT_SNAPSHOT_FILE_VERSION or
// BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION
int32_t file_version = reader.consume<int32_t>();
if (file_version == BACKUP_AGENT_SNAPSHOT_FILE_VERSION) {
wait(
decodeKVPairs(&reader, &results, false, encryptMode, Optional<BlobCipherEncryptHeader>(), tenantCache));
wait(decodeKVPairs(&reader, &results, false, encryptMode, Optional<int64_t>(), tenantCache));
} else if (file_version == BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION) {
CODE_PROBE(true, "decoding encrypted block");
// decode options struct
@ -1169,39 +1191,50 @@ ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<
StringRef optionsStringRef = StringRef(o, optionsLen);
EncryptedRangeFileWriter::Options options =
ObjectReader::fromStringRef<EncryptedRangeFileWriter::Options>(optionsStringRef, IncludeVersion());
ASSERT(!options.compressionEnabled);
// read header size
state uint32_t headerLen = reader.consume<uint32_t>();
// read the encryption header
state const uint8_t* headerStart = reader.consume(headerLen);
StringRef headerS = StringRef(headerStart, headerLen);
state std::variant<BlobCipherEncryptHeaderRef, BlobCipherEncryptHeader> encryptHeader;
if (options.configurableEncryptionEnabled) {
encryptHeader = BlobCipherEncryptHeaderRef::fromStringRef(headerS);
blockDomainId = std::get<BlobCipherEncryptHeaderRef>(encryptHeader)
.getCipherDetails()
.textCipherDetails.encryptDomainId;
} else {
encryptHeader = BlobCipherEncryptHeader::fromStringRef(headerS);
blockDomainId = std::get<BlobCipherEncryptHeader>(encryptHeader).cipherTextDetails.encryptDomainId;
}
// read encryption header
state const uint8_t* headerStart = reader.consume(BlobCipherEncryptHeader::headerSize);
StringRef headerS = StringRef(headerStart, BlobCipherEncryptHeader::headerSize);
state BlobCipherEncryptHeader header = BlobCipherEncryptHeader::fromStringRef(headerS);
blockTenantId = header.cipherTextDetails.encryptDomainId;
if (config.tenantMode == TenantMode::REQUIRED && !isReservedEncryptDomain(blockTenantId)) {
if (config.tenantMode == TenantMode::REQUIRED && !isReservedEncryptDomain(blockDomainId)) {
ASSERT(tenantCache.present());
Optional<TenantEntryCachePayload<Void>> payload = wait(tenantCache.get()->getById(blockTenantId));
Optional<TenantEntryCachePayload<Void>> payload = wait(tenantCache.get()->getById(blockDomainId));
if (!payload.present()) {
throw tenant_not_found();
}
}
const uint8_t* dataPayloadStart = headerStart + BlobCipherEncryptHeader::headerSize;
const uint8_t* dataPayloadStart = headerStart + headerLen;
// calculate the total bytes read up to (and including) the header
int64_t bytesRead = sizeof(int32_t) + sizeof(uint32_t) + optionsLen + BlobCipherEncryptHeader::headerSize;
int64_t bytesRead = sizeof(int32_t) + sizeof(uint32_t) + sizeof(uint32_t) + optionsLen + headerLen;
// get the size of the encrypted payload and decrypt it
int64_t dataLen = len - bytesRead;
StringRef decryptedData =
wait(EncryptedRangeFileWriter::decrypt(cx, header, dataPayloadStart, dataLen, &results.arena()));
wait(EncryptedRangeFileWriter::decrypt(cx, encryptHeader, dataPayloadStart, dataLen, &results.arena()));
reader = StringRefReader(decryptedData, restore_corrupted_data());
wait(decodeKVPairs(&reader, &results, true, encryptMode, header, tenantCache));
wait(decodeKVPairs(&reader, &results, true, encryptMode, blockDomainId, tenantCache));
} else {
throw restore_unsupported_file_version();
}
return results;
} catch (Error& e) {
if (e.code() == error_code_encrypt_keys_fetch_failed) {
TraceEvent(SevWarnAlways, "SnapshotRestoreEncryptKeyFetchFailed").detail("TenantId", blockTenantId);
ASSERT(!isReservedEncryptDomain(blockDomainId));
TraceEvent(SevWarnAlways, "SnapshotRestoreEncryptKeyFetchFailed").detail("TenantId", blockDomainId);
CODE_PROBE(true, "Snapshot restore encrypt keys not found");
} else if (e.code() == error_code_tenant_not_found) {
TraceEvent(SevWarnAlways, "EncryptedSnapshotRestoreTenantNotFound").detail("TenantId", blockTenantId);
ASSERT(!isReservedEncryptDomain(blockDomainId));
TraceEvent(SevWarnAlways, "EncryptedSnapshotRestoreTenantNotFound").detail("TenantId", blockDomainId);
CODE_PROBE(true, "Encrypted Snapshot restore tenant not found");
}
TraceEvent(SevWarn, "FileRestoreDecodeRangeFileBlockFailed")

116
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -549,6 +549,21 @@ ThreadFuture<bool> DLTenant::blobbifyRange(const KeyRangeRef& keyRange) {
});
}
ThreadFuture<bool> DLTenant::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
if (!api->tenantBlobbifyRangeBlocking) {
return unsupported_operation();
}
FdbCApi::FDBFuture* f = api->tenantBlobbifyRangeBlocking(
tenant, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size());
return toThreadFuture<bool>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
FdbCApi::fdb_bool_t ret = false;
ASSERT(!api->futureGetBool(f, &ret));
return ret;
});
}
ThreadFuture<bool> DLTenant::unblobbifyRange(const KeyRangeRef& keyRange) {
if (!api->tenantUnblobbifyRange) {
return unsupported_operation();
@ -601,6 +616,28 @@ ThreadFuture<Version> DLTenant::verifyBlobRange(const KeyRangeRef& keyRange, Opt
});
}
ThreadFuture<bool> DLTenant::flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) {
if (!api->tenantFlushBlobRange) {
return unsupported_operation();
}
Version readVersion = version.present() ? version.get() : latestVersion;
FdbCApi::FDBFuture* f = api->tenantFlushBlobRange(tenant,
keyRange.begin.begin(),
keyRange.begin.size(),
keyRange.end.begin(),
keyRange.end.size(),
compact,
readVersion);
return toThreadFuture<bool>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
FdbCApi::fdb_bool_t ret = false;
ASSERT(!api->futureGetBool(f, &ret));
return ret;
});
}
// DLDatabase
DLDatabase::DLDatabase(Reference<FdbCApi> api, ThreadFuture<FdbCApi::FDBDatabase*> dbFuture) : api(api), db(nullptr) {
addref();
@ -768,6 +805,21 @@ ThreadFuture<bool> DLDatabase::blobbifyRange(const KeyRangeRef& keyRange) {
});
}
ThreadFuture<bool> DLDatabase::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
if (!api->databaseBlobbifyRangeBlocking) {
return unsupported_operation();
}
FdbCApi::FDBFuture* f = api->databaseBlobbifyRangeBlocking(
db, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size());
return toThreadFuture<bool>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
FdbCApi::fdb_bool_t ret = false;
ASSERT(!api->futureGetBool(f, &ret));
return ret;
});
}
ThreadFuture<bool> DLDatabase::unblobbifyRange(const KeyRangeRef& keyRange) {
if (!api->databaseUnblobbifyRange) {
return unsupported_operation();
@ -820,6 +872,28 @@ ThreadFuture<Version> DLDatabase::verifyBlobRange(const KeyRangeRef& keyRange, O
});
}
ThreadFuture<bool> DLDatabase::flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) {
if (!api->databaseFlushBlobRange) {
return unsupported_operation();
}
Version readVersion = version.present() ? version.get() : latestVersion;
FdbCApi::FDBFuture* f = api->databaseFlushBlobRange(db,
keyRange.begin.begin(),
keyRange.begin.size(),
keyRange.end.begin(),
keyRange.end.size(),
compact,
readVersion);
return toThreadFuture<bool>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
FdbCApi::fdb_bool_t ret = false;
ASSERT(!api->futureGetBool(f, &ret));
return ret;
});
}
ThreadFuture<Standalone<StringRef>> DLDatabase::getClientStatus() {
if (!api->databaseGetClientStatus) {
return unsupported_operation();
@ -931,6 +1005,11 @@ void DLApi::init() {
fdbCPath,
"fdb_database_blobbify_range",
headerVersion >= ApiVersion::withBlobRangeApi().version());
loadClientFunction(&api->databaseBlobbifyRangeBlocking,
lib,
fdbCPath,
"fdb_database_blobbify_range_blocking",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->databaseUnblobbifyRange,
lib,
fdbCPath,
@ -946,6 +1025,11 @@ void DLApi::init() {
fdbCPath,
"fdb_database_verify_blob_range",
headerVersion >= ApiVersion::withBlobRangeApi().version());
loadClientFunction(&api->databaseFlushBlobRange,
lib,
fdbCPath,
"fdb_database_flush_blob_range",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->databaseGetClientStatus,
lib,
fdbCPath,
@ -968,6 +1052,11 @@ void DLApi::init() {
fdbCPath,
"fdb_tenant_blobbify_range",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->tenantBlobbifyRangeBlocking,
lib,
fdbCPath,
"fdb_tenant_blobbify_range_blocking",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->tenantUnblobbifyRange,
lib,
fdbCPath,
@ -983,6 +1072,11 @@ void DLApi::init() {
fdbCPath,
"fdb_tenant_verify_blob_range",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->tenantFlushBlobRange,
lib,
fdbCPath,
"fdb_tenant_flush_blob_range",
headerVersion >= ApiVersion::withTenantBlobRangeApi().version());
loadClientFunction(&api->tenantGetId, lib, fdbCPath, "fdb_tenant_get_id", headerVersion >= 730);
loadClientFunction(&api->tenantDestroy, lib, fdbCPath, "fdb_tenant_destroy", headerVersion >= 710);
@ -1837,6 +1931,10 @@ ThreadFuture<bool> MultiVersionTenant::blobbifyRange(const KeyRangeRef& keyRange
return executeOperation(&ITenant::blobbifyRange, keyRange);
}
ThreadFuture<bool> MultiVersionTenant::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
return executeOperation(&ITenant::blobbifyRangeBlocking, keyRange);
}
ThreadFuture<bool> MultiVersionTenant::unblobbifyRange(const KeyRangeRef& keyRange) {
return executeOperation(&ITenant::unblobbifyRange, keyRange);
}
@ -1850,6 +1948,13 @@ ThreadFuture<Version> MultiVersionTenant::verifyBlobRange(const KeyRangeRef& key
return executeOperation(&ITenant::verifyBlobRange, keyRange, std::forward<Optional<Version>>(version));
}
ThreadFuture<bool> MultiVersionTenant::flushBlobRange(const KeyRangeRef& keyRange,
bool compact,
Optional<Version> version) {
return executeOperation(
&ITenant::flushBlobRange, keyRange, std::forward<bool>(compact), std::forward<Optional<Version>>(version));
}
MultiVersionTenant::TenantState::TenantState(Reference<MultiVersionDatabase> db, TenantNameRef tenantName)
: tenantVar(new ThreadSafeAsyncVar<Reference<ITenant>>(Reference<ITenant>(nullptr))), tenantName(tenantName), db(db),
closed(false) {
@ -2071,6 +2176,10 @@ ThreadFuture<bool> MultiVersionDatabase::blobbifyRange(const KeyRangeRef& keyRan
return executeOperation(&IDatabase::blobbifyRange, keyRange);
}
ThreadFuture<bool> MultiVersionDatabase::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
return executeOperation(&IDatabase::blobbifyRangeBlocking, keyRange);
}
ThreadFuture<bool> MultiVersionDatabase::unblobbifyRange(const KeyRangeRef& keyRange) {
return executeOperation(&IDatabase::unblobbifyRange, keyRange);
}
@ -2084,6 +2193,13 @@ ThreadFuture<Version> MultiVersionDatabase::verifyBlobRange(const KeyRangeRef& k
return executeOperation(&IDatabase::verifyBlobRange, keyRange, std::forward<Optional<Version>>(version));
}
ThreadFuture<bool> MultiVersionDatabase::flushBlobRange(const KeyRangeRef& keyRange,
bool compact,
Optional<Version> version) {
return executeOperation(
&IDatabase::flushBlobRange, keyRange, std::forward<bool>(compact), std::forward<Optional<Version>>(version));
}
// Returns the protocol version reported by the coordinator this client is connected to
// If an expected version is given, the future won't return until the protocol version is different than expected
// Note: this will never return if the server is running a protocol from FDB 5.0 or older

69
fdbclient/NativeAPI.actor.cpp
View File

@ -47,6 +47,7 @@
#include "fdbclient/AnnotateActor.h"
#include "fdbclient/Atomic.h"
#include "fdbclient/BlobGranuleCommon.h"
#include "fdbclient/BlobGranuleRequest.actor.h"
#include "fdbclient/ClusterInterface.h"
#include "fdbclient/ClusterConnectionFile.h"
#include "fdbclient/ClusterConnectionMemoryRecord.h"
@ -8526,6 +8527,41 @@ Future<Version> DatabaseContext::verifyBlobRange(const KeyRange& range,
return verifyBlobRangeActor(Reference<DatabaseContext>::addRef(this), range, version, tenant);
}
ACTOR Future<bool> flushBlobRangeActor(Reference<DatabaseContext> cx,
KeyRange range,
bool compact,
Optional<Version> version,
Optional<Reference<Tenant>> tenant) {
if (tenant.present()) {
wait(tenant.get()->ready());
range = range.withPrefix(tenant.get()->prefix());
}
state Database db(cx);
if (!version.present()) {
state Transaction tr(db);
Version _v = wait(tr.getReadVersion());
version = _v;
}
FlushGranuleRequest req(-1, range, version.get(), compact);
try {
wait(success(doBlobGranuleRequests(db, range, req, &BlobWorkerInterface::flushGranuleRequest)));
return true;
} catch (Error& e) {
if (e.code() == error_code_blob_granule_transaction_too_old) {
// can't flush data at this version, because no granules
return false;
}
throw e;
}
}
Future<bool> DatabaseContext::flushBlobRange(const KeyRange& range,
bool compact,
Optional<Version> version,
Optional<Reference<Tenant>> tenant) {
return flushBlobRangeActor(Reference<DatabaseContext>::addRef(this), range, compact, version, tenant);
}
ACTOR Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
bool primary,
bool use_system_priority) {
@ -10928,8 +10964,39 @@ ACTOR Future<bool> setBlobRangeActor(Reference<DatabaseContext> cx,
}
}
ACTOR Future<bool> blobbifyRangeActor(Reference<DatabaseContext> cx,
KeyRange range,
bool doWait,
Optional<Reference<Tenant>> tenant) {
if (BG_REQUEST_DEBUG) {
fmt::print("BlobbifyRange [{0} - {1}) ({2})\n", range.begin.printable(), range.end.printable(), doWait);
}
state bool result = wait(setBlobRangeActor(cx, range, true, tenant));
if (!doWait || !result) {
return result;
}
// FIXME: add blob worker verifyRange rpc call that just waits for granule to become readable at any version
loop {
Version verifyVersion = wait(cx->verifyBlobRange(range, latestVersion, tenant));
if (verifyVersion != invalidVersion) {
if (BG_REQUEST_DEBUG) {
fmt::print("BlobbifyRange [{0} - {1}) got complete @ {2}\n",
range.begin.printable(),
range.end.printable(),
verifyVersion);
}
return result;
}
wait(delay(0.1));
}
}
Future<bool> DatabaseContext::blobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant) {
return setBlobRangeActor(Reference<DatabaseContext>::addRef(this), range, true, tenant);
return blobbifyRangeActor(Reference<DatabaseContext>::addRef(this), range, false, tenant);
}
Future<bool> DatabaseContext::blobbifyRangeBlocking(KeyRange range, Optional<Reference<Tenant>> tenant) {
return blobbifyRangeActor(Reference<DatabaseContext>::addRef(this), range, true, tenant);
}
Future<bool> DatabaseContext::unblobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant) {

27
fdbclient/RESTClient.actor.cpp
View File

@ -40,6 +40,16 @@
#include "flow/actorcompiler.h" // always the last include
#define ENABLE_VERBOSE_DEBUG true
#define TRACE_REST_OP(opName, url, secure) \
do { \
if (ENABLE_VERBOSE_DEBUG) { \
const std::string urlStr = url.toString(); \
TraceEvent("RESTClientOp").detail("Op", #opName).detail("Url", urlStr).detail("IsSecure", secure); \
} \
} while (0);
json_spirit::mObject RESTClient::Stats::getJSON() {
json_spirit::mObject o;
@ -61,10 +71,13 @@ RESTClient::Stats RESTClient::Stats::operator-(const Stats& rhs) {
return r;
}
RESTClient::RESTClient() {}
RESTClient::RESTClient() {
conectionPool = makeReference<RESTConnectionPool>(knobs.connection_pool_size);
}
RESTClient::RESTClient(std::unordered_map<std::string, int>& knobSettings) {
knobs.set(knobSettings);
conectionPool = makeReference<RESTConnectionPool>(knobs.connection_pool_size);
}
void RESTClient::setKnobs(const std::unordered_map<std::string, int>& knobSettings) {
@ -83,6 +96,10 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<RESTClient> cli
state UnsentPacketQueue content;
state int contentLen = url->body.size();
if (ENABLE_VERBOSE_DEBUG) {
TraceEvent(SevDebug, "DoRequestImpl").detail("Url", url->toString());
}
if (url->body.size() > 0) {
PacketWriter pw(content.getWriteBuffer(url->body.size()), nullptr, Unversioned());
pw.serializeBytes(url->body);
@ -166,7 +183,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<RESTClient> cli
// But only if our previous attempt was not the last allowable try.
retryable = retryable && (thisTry < maxTries);
TraceEvent event(SevWarn, retryable ? "RESTClient_FailedRetryable" : "RESTClient_RequestFailed");
TraceEvent event(SevWarn, retryable ? "RESTClientFailedRetryable" : "RESTClientRequestFailed");
// Attach err to trace event if present, otherwise extract some stuff from the response
if (err.present()) {
@ -269,6 +286,7 @@ Future<Reference<HTTP::Response>> RESTClient::doPost(const std::string& fullUrl,
const std::string& requestBody,
Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, requestBody, knobs.secure_connection);
TRACE_REST_OP("DoPost", url, knobs.secure_connection);
return doPutOrPost(HTTP::HTTP_VERB_POST, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK });
}
@ -276,6 +294,7 @@ Future<Reference<HTTP::Response>> RESTClient::doPut(const std::string& fullUrl,
const std::string& requestBody,
Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, requestBody, knobs.secure_connection);
TRACE_REST_OP("DoPut", url, knobs.secure_connection);
return doPutOrPost(
HTTP::HTTP_VERB_PUT,
optHeaders,
@ -299,16 +318,19 @@ Future<Reference<HTTP::Response>> RESTClient::doGetHeadDeleteOrTrace(const std::
Future<Reference<HTTP::Response>> RESTClient::doGet(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, knobs.secure_connection);
TRACE_REST_OP("DoGet", url, knobs.secure_connection);
return doGetHeadDeleteOrTrace(HTTP::HTTP_VERB_GET, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK });
}
Future<Reference<HTTP::Response>> RESTClient::doHead(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, knobs.secure_connection);
TRACE_REST_OP("DoHead", url, knobs.secure_connection);
return doGetHeadDeleteOrTrace(HTTP::HTTP_VERB_HEAD, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK });
}
Future<Reference<HTTP::Response>> RESTClient::doDelete(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, knobs.secure_connection);
TRACE_REST_OP("DoDelete", url, knobs.secure_connection);
return doGetHeadDeleteOrTrace(
HTTP::HTTP_VERB_DELETE,
optHeaders,
@ -321,6 +343,7 @@ Future<Reference<HTTP::Response>> RESTClient::doDelete(const std::string& fullUr
Future<Reference<HTTP::Response>> RESTClient::doTrace(const std::string& fullUrl, Optional<HTTP::Headers> optHeaders) {
RESTUrl url(fullUrl, knobs.secure_connection);
TRACE_REST_OP("DoTrace", url, knobs.secure_connection);
return doGetHeadDeleteOrTrace(
HTTP::HTTP_VERB_TRACE, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK });
}

32
fdbclient/RESTUtils.actor.cpp
View File

@ -25,6 +25,7 @@
#include "flow/IConnection.h"
#include <boost/algorithm/string.hpp>
#include <queue>
#include "flow/actorcompiler.h" // always the last include
@ -66,12 +67,12 @@ RESTClientKnobs::RESTClientKnobs() {
}
void RESTClientKnobs::set(const std::unordered_map<std::string, int>& knobSettings) {
TraceEvent trace = TraceEvent("RESTClient_SetKnobs");
TraceEvent trace = TraceEvent("RESTClientSetKnobs");
for (const auto& itr : knobSettings) {
const auto& kItr = RESTClientKnobs::knobMap.find(itr.first);
if (kItr == RESTClientKnobs::knobMap.end()) {
trace.detail("RESTClient_InvalidKnobName", itr.first);
trace.detail("RESTClientInvalidKnobName", itr.first);
throw rest_invalid_rest_client_knob();
}
*(kItr->second) = itr.second;
@ -98,28 +99,37 @@ ACTOR Future<RESTConnectionPool::ReusableConnection> connect_impl(Reference<REST
bool isSecure,
int maxConnLife) {
auto poolItr = connectionPool->connectionPoolMap.find(connectKey);
if (poolItr == connectionPool->connectionPoolMap.end()) {
throw rest_connectpool_key_not_found();
}
while (!poolItr->second.empty()) {
while (poolItr != connectionPool->connectionPoolMap.end() && !poolItr->second.empty()) {
RESTConnectionPool::ReusableConnection rconn = poolItr->second.front();
poolItr->second.pop();
if (rconn.expirationTime > now()) {
TraceEvent("RESTClient_ReusableConnection")
TraceEvent("RESTClientReuseConn")
.suppressFor(60)
.detail("Host", connectKey.first)
.detail("Service", connectKey.second)
.detail("RemoteEndpoint", rconn.conn->getPeerAddress())
.detail("ExpireIn", rconn.expirationTime - now());
return rconn;
}
}
// No valid connection exists, create a new one
state Reference<IConnection> conn =
wait(INetworkConnections::net()->connect(connectKey.first, connectKey.second, isSecure));
wait(conn->connectHandshake());
return RESTConnectionPool::ReusableConnection({ conn, now() + maxConnLife });
RESTConnectionPool::ReusableConnection reusableConn =
RESTConnectionPool::ReusableConnection({ conn, now() + maxConnLife });
connectionPool->connectionPoolMap.insert(
{ connectKey, std::queue<RESTConnectionPool::ReusableConnection>({ reusableConn }) });
TraceEvent("RESTClientCreateNewConn")
.suppressFor(60)
.detail("Host", connectKey.first)
.detail("Service", connectKey.second)
.detail("RemoteEndpoint", conn->getPeerAddress());
return reusableConn;
}
Future<RESTConnectionPool::ReusableConnection> RESTConnectionPool::connect(RESTConnectionPoolKey connectKey,
@ -188,14 +198,14 @@ void RESTUrl::parseUrl(const std::string& fullUrl, const bool isSecure) {
host = h.toString();
service = hRef.eat().toString();
TraceEvent("RESTClient_ParseURI")
TraceEvent("RESTClientParseURI")
.detail("URI", fullUrl)
.detail("Host", host)
.detail("Service", service)
.detail("Resource", resource)
.detail("ReqParameters", reqParameters);
} catch (std::string& err) {
TraceEvent("RESTClient_ParseError").detail("URI", fullUrl).detail("Error", err);
TraceEvent("RESTClientParseError").detail("URI", fullUrl).detail("Error", err);
throw rest_invalid_uri();
}
}

8
fdbclient/ServerKnobs.cpp
View File

@ -302,7 +302,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD, 20 );
init( DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC, isSimulated ? 2 : 21 * 60 * 60 * 24 ); if(randomize && BUGGIFY) DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC = isSimulated ? 0: 120;
init( DD_TENANT_AWARENESS_ENABLED, false );
init( STORAGE_QUOTA_ENABLED, false ); if(isSimulated) STORAGE_QUOTA_ENABLED = deterministicRandom()->coinflip();
init( STORAGE_QUOTA_ENABLED, true ); if(isSimulated) STORAGE_QUOTA_ENABLED = deterministicRandom()->coinflip();
init( TENANT_CACHE_LIST_REFRESH_INTERVAL, 2 ); if( randomize && BUGGIFY ) TENANT_CACHE_LIST_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10);
init( TENANT_CACHE_STORAGE_USAGE_REFRESH_INTERVAL, 2 ); if( randomize && BUGGIFY ) TENANT_CACHE_STORAGE_USAGE_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10);
init( TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL, 10 ); if( randomize && BUGGIFY ) TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10);
@ -987,13 +987,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init ( CLUSTER_RECOVERY_EVENT_NAME_PREFIX, "Master" );
// Encryption
init( ENABLE_ENCRYPTION, false ); if ( randomize && BUGGIFY ) ENABLE_ENCRYPTION = !ENABLE_ENCRYPTION;
init( ENCRYPTION_MODE, "AES-256-CTR" );
init( SIM_KMS_MAX_KEYS, 4096 );
init( ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH, 100000 );
init( ENABLE_TLOG_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY && ENABLE_ENCRYPTION ) ENABLE_TLOG_ENCRYPTION = false;
init( ENABLE_STORAGE_SERVER_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY && ENABLE_ENCRYPTION) ENABLE_STORAGE_SERVER_ENCRYPTION = false;
init( ENABLE_BLOB_GRANULE_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY && ENABLE_ENCRYPTION) ENABLE_BLOB_GRANULE_ENCRYPTION = false;
// encrypt key proxy
init( ENABLE_BLOB_GRANULE_COMPRESSION, false ); if ( randomize && BUGGIFY ) { ENABLE_BLOB_GRANULE_COMPRESSION = deterministicRandom()->coinflip(); }
@ -1060,6 +1055,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( BLOB_RESTORE_MANIFEST_URL, isSimulated ? "file://simfdb/fdbblob/manifest" : "" );
init( BLOB_RESTORE_MANIFEST_FILE_MAX_SIZE, isSimulated ? 10000 : 10000000 );
init( BLOB_RESTORE_MANIFEST_RETENTION_MAX, 10 );
init( BLOB_RESTORE_MLOGS_RETENTION_SECS, isSimulated ? 120 : 3600 * 24 * 14 );
init( BGCC_TIMEOUT, isSimulated ? 10.0 : 120.0 );
init( BGCC_MIN_INTERVAL, isSimulated ? 1.0 : 10.0 );

1
fdbclient/SystemData.cpp
View File

@ -1742,6 +1742,7 @@ Standalone<BlobRestoreArg> decodeBlobRestoreArg(ValueRef const& value) {
}
const Key blobManifestVersionKey = "\xff\x02/blobManifestVersion"_sr;
const Key blobGranulesLastFlushKey = "\xff\x02/blobGranulesLastFlushTs"_sr;
const KeyRangeRef idempotencyIdKeys("\xff\x02/idmp/"_sr, "\xff\x02/idmp0"_sr);
const KeyRef idempotencyIdsExpiredVersion("\xff\x02/idmpExpiredVersion"_sr);

13
fdbclient/Tenant.cpp
View File

@ -55,6 +55,17 @@ int64_t prefixToId(KeyRef prefix, EnforceValidTenantId enforceValidTenantId) {
return id;
}
KeyRangeRef clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena) {
if (tenantInfo.hasTenant()) {
return KeyRangeRef(range.begin.startsWith(tenantInfo.prefix.get()) ? range.begin : tenantInfo.prefix.get(),
range.end.startsWith(tenantInfo.prefix.get())
? range.end
: allKeys.end.withPrefix(tenantInfo.prefix.get(), arena));
} else {
return range;
}
}
bool withinSingleTenant(KeyRangeRef const& range) {
if (range.begin >= "\x80"_sr || range.begin.size() < TenantAPI::PREFIX_SIZE) {
return false;
@ -63,7 +74,7 @@ bool withinSingleTenant(KeyRangeRef const& range) {
return tRange.contains(range);
}
}; // namespace TenantAPI
} // namespace TenantAPI
std::string TenantAPI::tenantStateToString(TenantState tenantState) {
switch (tenantState) {

42
fdbclient/ThreadSafeTransaction.cpp
View File

@ -167,6 +167,15 @@ ThreadFuture<bool> ThreadSafeDatabase::blobbifyRange(const KeyRangeRef& keyRange
});
}
ThreadFuture<bool> ThreadSafeDatabase::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
DatabaseContext* db = this->db;
KeyRange range = keyRange;
return onMainThread([=]() -> Future<bool> {
db->checkDeferredError();
return db->blobbifyRangeBlocking(range);
});
}
ThreadFuture<bool> ThreadSafeDatabase::unblobbifyRange(const KeyRangeRef& keyRange) {
DatabaseContext* db = this->db;
KeyRange range = keyRange;
@ -195,6 +204,17 @@ ThreadFuture<Version> ThreadSafeDatabase::verifyBlobRange(const KeyRangeRef& key
});
}
ThreadFuture<bool> ThreadSafeDatabase::flushBlobRange(const KeyRangeRef& keyRange,
bool compact,
Optional<Version> version) {
DatabaseContext* db = this->db;
KeyRange range = keyRange;
return onMainThread([=]() -> Future<bool> {
db->checkDeferredError();
return db->flushBlobRange(range, compact, version);
});
}
ThreadSafeDatabase::ThreadSafeDatabase(ConnectionRecordType connectionRecordType,
std::string connectionRecordString,
int apiVersion) {
@ -277,6 +297,16 @@ ThreadFuture<bool> ThreadSafeTenant::blobbifyRange(const KeyRangeRef& keyRange)
});
}
ThreadFuture<bool> ThreadSafeTenant::blobbifyRangeBlocking(const KeyRangeRef& keyRange) {
DatabaseContext* db = this->db->db;
KeyRange range = keyRange;
return onMainThread([=]() -> Future<bool> {
db->checkDeferredError();
db->addref();
return db->blobbifyRangeBlocking(range, Reference<Tenant>::addRef(tenant));
});
}
ThreadFuture<bool> ThreadSafeTenant::unblobbifyRange(const KeyRangeRef& keyRange) {
DatabaseContext* db = this->db->db;
KeyRange range = keyRange;
@ -308,6 +338,18 @@ ThreadFuture<Version> ThreadSafeTenant::verifyBlobRange(const KeyRangeRef& keyRa
});
}
ThreadFuture<bool> ThreadSafeTenant::flushBlobRange(const KeyRangeRef& keyRange,
bool compact,
Optional<Version> version) {
DatabaseContext* db = this->db->db;
KeyRange range = keyRange;
return onMainThread([=]() -> Future<bool> {
db->checkDeferredError();
db->addref();
return db->flushBlobRange(range, compact, version, Reference<Tenant>::addRef(tenant));
});
}
ThreadSafeTenant::~ThreadSafeTenant() {
Tenant* t = this->tenant;
if (t)

11
fdbclient/include/fdbclient/BlobCipher.h
View File

@ -154,6 +154,8 @@ private:
uint8_t* buffer;
};
class BlobCipherKey;
#pragma pack(push, 1) // exact fit - no padding
struct BlobCipherDetails {
constexpr static FileIdentifier file_identifier = 1945731;
@ -163,7 +165,7 @@ struct BlobCipherDetails {
// BaseCipher encryption key identifier
EncryptCipherBaseKeyId baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID;
// Random salt
EncryptCipherRandomSalt salt{};
EncryptCipherRandomSalt salt = INVALID_ENCRYPT_RANDOM_SALT;
static uint32_t getSize() {
return sizeof(EncryptCipherDomainId) + sizeof(EncryptCipherBaseKeyId) + sizeof(EncryptCipherRandomSalt);
@ -175,6 +177,8 @@ struct BlobCipherDetails {
const EncryptCipherRandomSalt& random)
: encryptDomainId(dId), baseCipherId(bId), salt(random) {}
void validateCipherDetailsWithCipherKey(Reference<BlobCipherKey> headerCipherKey);
bool operator==(const BlobCipherDetails& o) const {
return encryptDomainId == o.encryptDomainId && baseCipherId == o.baseCipherId && salt == o.salt;
}
@ -209,6 +213,7 @@ struct EncryptHeaderCipherDetails {
BlobCipherDetails textCipherDetails;
Optional<BlobCipherDetails> headerCipherDetails;
EncryptHeaderCipherDetails() = default;
EncryptHeaderCipherDetails(const BlobCipherDetails& tCipherDetails) : textCipherDetails(tCipherDetails) {}
EncryptHeaderCipherDetails(const BlobCipherDetails& tCipherDetails, const BlobCipherDetails& hCipherDetails)
: textCipherDetails(tCipherDetails), headerCipherDetails(hCipherDetails) {}
@ -489,6 +494,7 @@ struct BlobCipherEncryptHeaderRef {
const uint8_t* getIV() const;
const EncryptHeaderCipherDetails getCipherDetails() const;
EncryptAuthTokenMode getAuthTokenMode() const;
EncryptCipherDomainId getDomainId() const;
void validateEncryptionHeaderDetails(const BlobCipherDetails& textCipherDetails,
const BlobCipherDetails& headerCipherDetails,
@ -1005,5 +1011,6 @@ void computeAuthToken(const std::vector<std::pair<const uint8_t*, size_t>>& payl
unsigned int digestMaxBufSz);
EncryptAuthTokenMode getEncryptAuthTokenMode(const EncryptAuthTokenMode mode);
int getEncryptCurrentAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo);
#endif // FDBCLIENT_BLOB_CIPHER_H
#endif // FDBCLIENT_BLOB_CIPHER_H

23
fdbclient/include/fdbclient/BlobGranuleCommon.h
View File

@ -197,6 +197,7 @@ struct BlobFilePointerRef {
int64_t offset;
int64_t length;
int64_t fullFileLength;
Version fileVersion;
Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx;
// Non-serializable fields
@ -205,25 +206,34 @@ struct BlobFilePointerRef {
BlobFilePointerRef() {}
BlobFilePointerRef(Arena& to, const std::string& filename, int64_t offset, int64_t length, int64_t fullFileLength)
: filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength) {}
BlobFilePointerRef(Arena& to,
const std::string& filename,
int64_t offset,
int64_t length,
int64_t fullFileLength,
Version fileVersion)
: filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength),
fileVersion(fileVersion) {}
BlobFilePointerRef(Arena& to,
const std::string& filename,
int64_t offset,
int64_t length,
int64_t fullFileLength,
Version fileVersion,
Optional<BlobGranuleCipherKeysCtx> ciphKeysCtx)
: filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength),
cipherKeysCtx(ciphKeysCtx) {}
fileVersion(fileVersion), cipherKeysCtx(ciphKeysCtx) {}
BlobFilePointerRef(Arena& to,
const std::string& filename,
int64_t offset,
int64_t length,
int64_t fullFileLength,
Version fileVersion,
Optional<BlobGranuleCipherKeysMeta> ciphKeysMeta)
: filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength) {
: filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength),
fileVersion(fileVersion) {
if (ciphKeysMeta.present()) {
cipherKeysMetaRef = BlobGranuleCipherKeysMetaRef(to, ciphKeysMeta.get());
}
@ -231,12 +241,12 @@ struct BlobFilePointerRef {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, filename, offset, length, fullFileLength, cipherKeysCtx);
serializer(ar, filename, offset, length, fullFileLength, fileVersion, cipherKeysCtx);
}
std::string toString() const {
std::stringstream ss;
ss << filename.toString() << ":" << offset << ":" << length << ":" << fullFileLength;
ss << filename.toString() << ":" << offset << ":" << length << ":" << fullFileLength << "@" << fileVersion;
if (cipherKeysCtx.present()) {
ss << ":CipherKeysCtx:TextCipher:" << cipherKeysCtx.get().textCipherKey.encryptDomainId << ":"
<< cipherKeysCtx.get().textCipherKey.baseCipherId << ":" << cipherKeysCtx.get().textCipherKey.salt
@ -257,6 +267,7 @@ struct BlobGranuleChunkRef {
constexpr static FileIdentifier file_identifier = 865198;
KeyRangeRef keyRange;
Version includedVersion;
// FIXME: remove snapshotVersion, it is deprecated with fileVersion in BlobFilePointerRef
Version snapshotVersion;
Optional<BlobFilePointerRef> snapshotFile; // not set if it's an incremental read
VectorRef<BlobFilePointerRef> deltaFiles;

1
fdbclient/include/fdbclient/ClientKnobs.h
View File

@ -302,6 +302,7 @@ public:
// key_not_found errors for. If TenantInfo::INVALID_TENANT is contained within the list then no tenants will be
// dropped. This Knob should ONLY be used in simulation for testing purposes
std::string SIMULATION_EKP_TENANT_IDS_TO_DROP;
bool SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS;
bool ENABLE_CONFIGURABLE_ENCRYPTION;
int ENCRYPT_HEADER_FLAGS_VERSION;
int ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION;

112
fdbclient/include/fdbclient/CommitTransaction.h
View File

@ -30,6 +30,8 @@
#include "flow/EncryptUtils.h"
#include "flow/Knobs.h"
#include <unordered_set>
// The versioned message has wire format : -1, version, messages
static const int32_t VERSION_HEADER = -1;
@ -143,6 +145,38 @@ struct MutationRef {
return reinterpret_cast<const BlobCipherEncryptHeader*>(param1.begin());
}
const BlobCipherEncryptHeaderRef configurableEncryptionHeader() const {
ASSERT(isEncrypted());
return BlobCipherEncryptHeaderRef::fromStringRef(param1);
}
EncryptCipherDomainId encryptDomainId() const {
ASSERT(isEncrypted());
return CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION ? configurableEncryptionHeader().getDomainId()
: encryptionHeader()->cipherTextDetails.encryptDomainId;
}
void updateEncryptCipherDetails(std::unordered_set<BlobCipherDetails>& cipherDetails) {
ASSERT(isEncrypted());
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
EncryptHeaderCipherDetails details = header.getCipherDetails();
ASSERT(details.textCipherDetails.isValid());
cipherDetails.insert(details.textCipherDetails);
if (details.headerCipherDetails.present()) {
ASSERT(details.headerCipherDetails.get().isValid());
cipherDetails.insert(details.headerCipherDetails.get());
}
} else {
const BlobCipherEncryptHeader* header = encryptionHeader();
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
}
}
MutationRef encrypt(TextAndHeaderCipherKeys cipherKeys,
Arena& arena,
BlobCipherMetrics::UsageType usageType) const {
@ -150,6 +184,7 @@ struct MutationRef {
deterministicRandom()->randomBytes(iv, AES_256_IV_LENGTH);
BinaryWriter bw(AssumeVersion(ProtocolVersion::withEncryptionAtRest()));
bw << *this;
EncryptBlobCipherAes265Ctr cipher(
cipherKeys.cipherTextKey,
cipherKeys.cipherHeaderKey,
@ -157,11 +192,22 @@ struct MutationRef {
AES_256_IV_LENGTH,
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
usageType);
BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader;
StringRef headerRef(reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader));
StringRef payload =
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), header, arena)->toStringRef();
return MutationRef(Encrypted, headerRef, payload);
StringRef serializedHeader;
StringRef payload;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef header;
payload = cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), &header, arena);
Standalone<StringRef> headerStr = BlobCipherEncryptHeaderRef::toStringRef(header);
arena.dependsOn(headerStr.arena());
serializedHeader = headerStr;
} else {
BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader;
serializedHeader = StringRef(reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader));
payload =
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), header, arena)->toStringRef();
}
return MutationRef(Encrypted, serializedHeader, payload);
}
MutationRef encrypt(const std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>& cipherKeys,
@ -183,6 +229,7 @@ struct MutationRef {
deterministicRandom()->randomBytes(iv, AES_256_IV_LENGTH);
BinaryWriter bw(AssumeVersion(ProtocolVersion::withEncryptionAtRest()));
bw << *this;
EncryptBlobCipherAes265Ctr cipher(
textCipherKey,
headerCipherKey,
@ -190,11 +237,22 @@ struct MutationRef {
AES_256_IV_LENGTH,
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
usageType);
BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader;
StringRef headerRef(reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader));
StringRef payload =
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), header, arena)->toStringRef();
return MutationRef(Encrypted, headerRef, payload);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef header;
StringRef payload =
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), &header, arena);
Standalone<StringRef> serializedHeader = BlobCipherEncryptHeaderRef::toStringRef(header);
arena.dependsOn(serializedHeader.arena());
return MutationRef(Encrypted, serializedHeader, payload);
} else {
BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader;
StringRef serializedHeader =
StringRef(reinterpret_cast<const uint8_t*>(header), sizeof(BlobCipherEncryptHeader));
StringRef payload =
cipher.encrypt(static_cast<const uint8_t*>(bw.getData()), bw.getLength(), header, arena)->toStringRef();
return MutationRef(Encrypted, serializedHeader, payload);
}
}
MutationRef encryptMetadata(const std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>& cipherKeys,
@ -207,9 +265,18 @@ struct MutationRef {
Arena& arena,
BlobCipherMetrics::UsageType usageType,
StringRef* buf = nullptr) const {
const BlobCipherEncryptHeader* header = encryptionHeader();
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header->iv, usageType);
StringRef plaintext = cipher.decrypt(param2.begin(), param2.size(), *header, arena)->toStringRef();
StringRef plaintext;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
const BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.getIV(), usageType);
plaintext = cipher.decrypt(param2.begin(), param2.size(), header, arena);
} else {
const BlobCipherEncryptHeader* header = encryptionHeader();
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header->iv, usageType);
plaintext = cipher.decrypt(param2.begin(), param2.size(), *header, arena)->toStringRef();
}
if (buf != nullptr) {
*buf = plaintext;
}
@ -229,7 +296,6 @@ struct MutationRef {
TextAndHeaderCipherKeys getCipherKeys(
const std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>& cipherKeys) const {
const BlobCipherEncryptHeader* header = encryptionHeader();
auto getCipherKey = [&](const BlobCipherDetails& details) -> Reference<BlobCipherKey> {
if (!details.isValid()) {
return {};
@ -239,8 +305,22 @@ struct MutationRef {
return iter->second;
};
TextAndHeaderCipherKeys textAndHeaderKeys;
textAndHeaderKeys.cipherHeaderKey = getCipherKey(header->cipherHeaderDetails);
textAndHeaderKeys.cipherTextKey = getCipherKey(header->cipherTextDetails);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
const BlobCipherEncryptHeaderRef header = configurableEncryptionHeader();
EncryptHeaderCipherDetails cipherDetails = header.getCipherDetails();
ASSERT(cipherDetails.textCipherDetails.isValid());
textAndHeaderKeys.cipherTextKey = getCipherKey(cipherDetails.textCipherDetails);
if (cipherDetails.headerCipherDetails.present()) {
ASSERT(cipherDetails.headerCipherDetails.get().isValid());
textAndHeaderKeys.cipherHeaderKey = getCipherKey(cipherDetails.headerCipherDetails.get());
} else {
ASSERT(!FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED);
}
} else {
const BlobCipherEncryptHeader* header = encryptionHeader();
textAndHeaderKeys.cipherHeaderKey = getCipherKey(header->cipherHeaderDetails);
textAndHeaderKeys.cipherTextKey = getCipherKey(header->cipherTextDetails);
}
return textAndHeaderKeys;
}

5
fdbclient/include/fdbclient/DatabaseContext.h
View File

@ -406,6 +406,7 @@ public:
Future<Void> waitPurgeGranulesComplete(Key purgeKey);
Future<bool> blobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant = {});
Future<bool> blobbifyRangeBlocking(KeyRange range, Optional<Reference<Tenant>> tenant = {});
Future<bool> unblobbifyRange(KeyRange range, Optional<Reference<Tenant>> tenant = {});
Future<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(KeyRange range,
int rangeLimit,
@ -413,6 +414,10 @@ public:
Future<Version> verifyBlobRange(const KeyRange& range,
Optional<Version> version,
Optional<Reference<Tenant>> tenant = {});
Future<bool> flushBlobRange(const KeyRange& range,
bool compact,
Optional<Version> version,
Optional<Reference<Tenant>> tenant = {});
Future<bool> blobRestore(const KeyRange range, Optional<Version> version);
// private:

4
fdbclient/include/fdbclient/FDBTypes.h
View File

@ -1063,9 +1063,9 @@ struct StorageBytes {
constexpr static FileIdentifier file_identifier = 3928581;
// Free space on the filesystem
int64_t free;
// Total space on the filesystem
// Total capacity on the filesystem usable by non-privileged users.
int64_t total;
// Used by *this* store, not total - free
// Total size of all files owned by *this* storage instance, not total - free
int64_t used;
// Amount of space available for use by the store, which includes free space on the filesystem
// and internal free space within the store data that is immediately reusable.

24
fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h
View File

@ -56,7 +56,7 @@ Future<Void> onEncryptKeyProxyChange(Reference<AsyncVar<T> const> db) {
break;
}
}
TraceEvent("GetEncryptCipherKeys_EncryptKeyProxyChanged")
TraceEvent("GetEncryptCipherKeysEncryptKeyProxyChanged")
.detail("PreviousProxyId", previousProxyId.orDefault(UID()))
.detail("CurrentProxyId", currentProxyId.orDefault(UID()));
return Void();
@ -69,19 +69,19 @@ Future<EKPGetLatestBaseCipherKeysReply> getUncachedLatestEncryptCipherKeys(Refer
Optional<EncryptKeyProxyInterface> proxy = db->get().encryptKeyProxy;
if (!proxy.present()) {
// Wait for onEncryptKeyProxyChange.
TraceEvent("GetLatestEncryptCipherKeys_EncryptKeyProxyNotPresent").detail("UsageType", toString(usageType));
TraceEvent("GetLatestEncryptCipherKeysEncryptKeyProxyNotPresent").detail("UsageType", toString(usageType));
return Never();
}
request.reply.reset();
try {
EKPGetLatestBaseCipherKeysReply reply = wait(proxy.get().getLatestBaseCipherKeys.getReply(request));
if (reply.error.present()) {
TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_RequestFailed").error(reply.error.get());
TraceEvent(SevWarn, "GetLatestEncryptCipherKeysRequestFailed").error(reply.error.get());
throw encrypt_keys_fetch_failed();
}
return reply;
} catch (Error& e) {
TraceEvent("GetLatestEncryptCipherKeys_CaughtError").error(e);
TraceEvent("GetLatestEncryptCipherKeysCaughtError").error(e);
if (e.code() == error_code_broken_promise) {
// Wait for onEncryptKeyProxyChange.
return Never();
@ -103,7 +103,7 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
state EKPGetLatestBaseCipherKeysRequest request;
if (!db.isValid()) {
TraceEvent(SevError, "GetLatestEncryptCipherKeys_ServerDBInfoNotAvailable");
TraceEvent(SevError, "GetLatestEncryptCipherKeysServerDBInfoNotAvailable");
throw encrypt_ops_error();
}
@ -140,7 +140,7 @@ Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getL
// Check for any missing cipher keys.
for (auto domainId : request.encryptDomainIds) {
if (cipherKeys.count(domainId) == 0) {
TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_KeyMissing").detail("DomainId", domainId);
TraceEvent(SevWarn, "GetLatestEncryptCipherKeysKeyMissing").detail("DomainId", domainId);
throw encrypt_key_not_found();
}
}
@ -176,14 +176,14 @@ Future<EKPGetBaseCipherKeysByIdsReply> getUncachedEncryptCipherKeys(Reference<As
Optional<EncryptKeyProxyInterface> proxy = db->get().encryptKeyProxy;
if (!proxy.present()) {
// Wait for onEncryptKeyProxyChange.
TraceEvent("GetEncryptCipherKeys_EncryptKeyProxyNotPresent").detail("UsageType", toString(usageType));
TraceEvent("GetEncryptCipherKeysEncryptKeyProxyNotPresent").detail("UsageType", toString(usageType));
return Never();
}
request.reply.reset();
try {
EKPGetBaseCipherKeysByIdsReply reply = wait(proxy.get().getBaseCipherKeysByIds.getReply(request));
if (reply.error.present()) {
TraceEvent(SevWarn, "GetEncryptCipherKeys_RequestFailed").error(reply.error.get());
TraceEvent(SevWarn, "GetEncryptCipherKeysRequestFailed").error(reply.error.get());
throw encrypt_keys_fetch_failed();
}
if (g_network && g_network->isSimulated() && usageType == BlobCipherMetrics::RESTORE) {
@ -192,7 +192,7 @@ Future<EKPGetBaseCipherKeysByIdsReply> getUncachedEncryptCipherKeys(Reference<As
if (!tenantIdsToDrop.count(TenantInfo::INVALID_TENANT)) {
for (auto& baseCipherInfo : request.baseCipherInfos) {
if (tenantIdsToDrop.count(baseCipherInfo.domainId)) {
TraceEvent("GetEncryptCipherKeys_SimulatedError").detail("DomainId", baseCipherInfo.domainId);
TraceEvent("GetEncryptCipherKeysSimulatedError").detail("DomainId", baseCipherInfo.domainId);
throw encrypt_keys_fetch_failed();
}
}
@ -200,7 +200,7 @@ Future<EKPGetBaseCipherKeysByIdsReply> getUncachedEncryptCipherKeys(Reference<As
}
return reply;
} catch (Error& e) {
TraceEvent("GetEncryptCipherKeys_CaughtError").error(e);
TraceEvent("GetEncryptCipherKeysCaughtError").error(e);
if (e.code() == error_code_broken_promise) {
// Wait for onEncryptKeyProxyChange.
return Never();
@ -225,7 +225,7 @@ Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> getEncry
state EKPGetBaseCipherKeysByIdsRequest request;
if (!db.isValid()) {
TraceEvent(SevError, "GetEncryptCipherKeys_ServerDBInfoNotAvailable");
TraceEvent(SevError, "GetEncryptCipherKeysServerDBInfoNotAvailable");
throw encrypt_ops_error();
}
@ -262,7 +262,7 @@ Future<std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>>> getEncry
BaseCipherIndex baseIdx = std::make_pair(details.encryptDomainId, details.baseCipherId);
const auto& itr = baseCipherKeys.find(baseIdx);
if (itr == baseCipherKeys.end()) {
TraceEvent(SevError, "GetEncryptCipherKeys_KeyMissing")
TraceEvent(SevError, "GetEncryptCipherKeysKeyMissing")
.detail("DomainId", details.encryptDomainId)
.detail("BaseCipherId", details.baseCipherId);
throw encrypt_key_not_found();

4
fdbclient/include/fdbclient/IClientApi.h
View File

@ -153,11 +153,13 @@ public:
virtual ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) = 0;
virtual ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) = 0;
virtual ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) = 0;
virtual ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) = 0;
virtual void addref() = 0;
virtual void delref() = 0;
@ -200,11 +202,13 @@ public:
virtual ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) = 0;
virtual ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) = 0;
virtual ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) = 0;
virtual ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) = 0;
virtual ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) = 0;
// Interface to manage shared state across multiple connections to the same Database
virtual ThreadFuture<DatabaseSharedState*> createSharedState() = 0;

36
fdbclient/include/fdbclient/MultiVersionTransaction.h
View File

@ -187,6 +187,12 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
uint8_t const* end_key_name,
int end_key_name_length);
FDBFuture* (*databaseBlobbifyRangeBlocking)(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
FDBFuture* (*databaseUnblobbifyRange)(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
@ -207,6 +213,14 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
int end_key_name_length,
int64_t version);
FDBFuture* (*databaseFlushBlobRange)(FDBDatabase* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version);
FDBFuture* (*databaseGetClientStatus)(FDBDatabase* db);
// Tenant
@ -230,6 +244,12 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
uint8_t const* end_key_name,
int end_key_name_length);
FDBFuture* (*tenantBlobbifyRangeBlocking)(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
FDBFuture* (*tenantUnblobbifyRange)(FDBTenant* tenant,
uint8_t const* begin_key_name,
int begin_key_name_length,
@ -250,6 +270,14 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
int end_key_name_length,
int64_t version);
FDBFuture* (*tenantFlushBlobRange)(FDBTenant* db,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t compact,
int64_t version);
FDBFuture* (*tenantGetId)(FDBTenant* tenant);
void (*tenantDestroy)(FDBTenant* tenant);
@ -547,11 +575,13 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
void addref() override { ThreadSafeReferenceCounted<DLTenant>::addref(); }
void delref() override { ThreadSafeReferenceCounted<DLTenant>::delref(); }
@ -597,11 +627,13 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
ThreadFuture<DatabaseSharedState*> createSharedState() override;
void setSharedState(DatabaseSharedState* p) override;
@ -866,10 +898,12 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
void addref() override { ThreadSafeReferenceCounted<MultiVersionTenant>::addref(); }
void delref() override { ThreadSafeReferenceCounted<MultiVersionTenant>::delref(); }
@ -994,10 +1028,12 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
ThreadFuture<DatabaseSharedState*> createSharedState() override;
void setSharedState(DatabaseSharedState* p) override;

6
fdbclient/include/fdbclient/RESTUtils.h
View File

@ -27,6 +27,7 @@
#include "flow/FastRef.h"
#include "flow/Net2Packet.h"
#include <fmt/format.h>
#include <unordered_map>
#include <utility>
@ -108,6 +109,11 @@ public:
explicit RESTUrl(const std::string& fullUrl, const bool isSecure);
explicit RESTUrl(const std::string& fullUrl, const std::string& body, const bool isSecure);
std::string toString() const {
return fmt::format(
"Host {} Service {} Resource {} ReqParams {} Body {}", host, service, resource, reqParameters, body);
}
private:
void parseUrl(const std::string& fullUrl, bool isSecure);
};

6
fdbclient/include/fdbclient/ServerKnobs.h
View File

@ -962,13 +962,8 @@ public:
std::string CLUSTER_RECOVERY_EVENT_NAME_PREFIX;
// Encryption
bool ENABLE_ENCRYPTION;
std::string ENCRYPTION_MODE;
int SIM_KMS_MAX_KEYS;
int ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH;
bool ENABLE_TLOG_ENCRYPTION;
bool ENABLE_STORAGE_SERVER_ENCRYPTION; // Currently only Redwood engine supports encryption
bool ENABLE_BLOB_GRANULE_ENCRYPTION;
// Compression
bool ENABLE_BLOB_GRANULE_COMPRESSION;
@ -1039,6 +1034,7 @@ public:
std::string BLOB_RESTORE_MANIFEST_URL;
int BLOB_RESTORE_MANIFEST_FILE_MAX_SIZE;
int BLOB_RESTORE_MANIFEST_RETENTION_MAX;
int BLOB_RESTORE_MLOGS_RETENTION_SECS;
// Blob metadata
int64_t BLOB_METADATA_CACHE_TTL;

1
fdbclient/include/fdbclient/SystemData.h
View File

@ -733,6 +733,7 @@ const KeyRange decodeBlobRestoreArgKeyFor(const KeyRef key);
const Value blobRestoreArgValueFor(BlobRestoreArg args);
Standalone<BlobRestoreArg> decodeBlobRestoreArg(ValueRef const& value);
extern const Key blobManifestVersionKey;
extern const Key blobGranulesLastFlushKey;
extern const KeyRangeRef idempotencyIdKeys;
extern const KeyRef idempotencyIdsExpiredVersion;

1
fdbclient/include/fdbclient/Tenant.h
View File

@ -36,6 +36,7 @@ namespace TenantAPI {
KeyRef idToPrefix(Arena& p, int64_t id);
Key idToPrefix(int64_t id);
int64_t prefixToId(KeyRef prefix, EnforceValidTenantId = EnforceValidTenantId::True);
KeyRangeRef clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena);
// return true if begin and end has the same non-negative prefix id
bool withinSingleTenant(KeyRangeRef const&);

4
fdbclient/include/fdbclient/ThreadSafeTransaction.h
View File

@ -64,11 +64,13 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
ThreadFuture<DatabaseSharedState*> createSharedState() override;
void setSharedState(DatabaseSharedState* p) override;
@ -101,11 +103,13 @@ public:
ThreadFuture<Void> waitPurgeGranulesComplete(const KeyRef& purgeKey) override;
ThreadFuture<bool> blobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> blobbifyRangeBlocking(const KeyRangeRef& keyRange) override;
ThreadFuture<bool> unblobbifyRange(const KeyRangeRef& keyRange) override;
ThreadFuture<Standalone<VectorRef<KeyRangeRef>>> listBlobbifiedRanges(const KeyRangeRef& keyRange,
int rangeLimit) override;
ThreadFuture<Version> verifyBlobRange(const KeyRangeRef& keyRange, Optional<Version> version) override;
ThreadFuture<bool> flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional<Version> version) override;
void addref() override { ThreadSafeReferenceCounted<ThreadSafeTenant>::addref(); }
void delref() override { ThreadSafeReferenceCounted<ThreadSafeTenant>::delref(); }

2
fdbrpc/include/fdbrpc/TenantInfo.h
View File

@ -47,7 +47,7 @@ struct TenantInfo {
// Helper function for most endpoints that read/write data. This returns true iff
// the client is either a) a trusted peer or b) is accessing keyspace belonging to a tenant,
// for which it has a valid authorization token.
// NOTE: In a cluster where TenantMode is OPTIONAL or DISABLED, tenant name may be unset.
// NOTE: In a cluster where TenantMode is OPTIONAL or DISABLED, tenant ID may be invalid.
// In such case, the request containing such TenantInfo is valid iff the requesting peer is trusted.
bool isAuthorized() const { return trusted || tenantAuthorized; }
bool hasTenant() const { return tenantId != INVALID_TENANT; }

1
fdbserver/ApplyMetadataMutation.cpp
View File

@ -25,7 +25,6 @@
#include "fdbclient/Notified.h"
#include "fdbclient/SystemData.h"
#include "fdbserver/ApplyMetadataMutation.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/IKeyValueStore.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/LogProtocolMessage.h"

6
fdbserver/BackupWorker.actor.cpp
View File

@ -105,11 +105,7 @@ struct VersionedMessage {
ArenaReader reader(arena, message, AssumeVersion(ProtocolVersion::withEncryptionAtRest()));
MutationRef m;
reader >> m;
const BlobCipherEncryptHeader* header = m.encryptionHeader();
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
m.updateEncryptCipherDetails(cipherDetails);
}
}
};

3
fdbserver/BlobGranuleServerCommon.actor.cpp
View File

@ -209,6 +209,7 @@ void GranuleFiles::getFiles(Version beginVersion,
snapshotF->offset,
snapshotF->length,
snapshotF->fullFileLength,
snapshotF->version,
summarize ? Optional<BlobGranuleCipherKeysMeta>() : snapshotF->cipherKeysMeta);
lastIncluded = chunk.snapshotVersion;
} else {
@ -221,6 +222,7 @@ void GranuleFiles::getFiles(Version beginVersion,
deltaF->offset,
deltaF->length,
deltaF->fullFileLength,
deltaF->version,
summarize ? Optional<BlobGranuleCipherKeysMeta>() : deltaF->cipherKeysMeta);
deltaBytesCounter += deltaF->length;
ASSERT(lastIncluded < deltaF->version);
@ -235,6 +237,7 @@ void GranuleFiles::getFiles(Version beginVersion,
deltaF->offset,
deltaF->length,
deltaF->fullFileLength,
deltaF->version,
deltaF->cipherKeysMeta);
deltaBytesCounter += deltaF->length;
lastIncluded = deltaF->version;

95
fdbserver/BlobManager.actor.cpp
View File

@ -19,14 +19,20 @@
*/
#include <algorithm>
#include <ctime>
#include <limits>
#include <sstream>
#include <queue>
#include <vector>
#include <unordered_map>
#include "fdbclient/BackupContainer.h"
#include "fdbclient/ClientBooleanParams.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/ServerKnobs.h"
#include "fdbrpc/simulator.h"
#include "flow/CodeProbe.h"
#include "flow/serialize.h"
#include "fmt/format.h"
#include "fdbclient/BackupContainerFileSystem.h"
#include "fdbclient/BlobGranuleCommon.h"
@ -38,6 +44,7 @@
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/Tuple.h"
#include "fdbclient/BackupAgent.actor.h"
#include "fdbserver/BlobManagerInterface.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/BlobGranuleValidation.actor.h"
@ -5243,6 +5250,91 @@ ACTOR Future<Void> bgConsistencyCheck(Reference<BlobManagerData> bmData) {
}
}
ACTOR Future<Void> updateLastFlushTs(Database db) {
state Transaction tr(db);
loop {
try {
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
KeyBackedProperty<int64_t> lastFlushTs(blobGranulesLastFlushKey);
int64_t epochs = (int64_t)now();
lastFlushTs.set(&tr, epochs);
wait(tr.commit());
return Void();
} catch (Error& e) {
fmt::print("updateLastFlushTs {} \n", e.what());
wait(tr.onError(e));
}
}
}
ACTOR Future<int64_t> getLastFlushTs(Database db) {
state Transaction tr(db);
loop {
try {
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
KeyBackedProperty<int64_t> lastFlushTs(blobGranulesLastFlushKey);
state int64_t ret;
wait(store(ret, lastFlushTs.getD(&tr, Snapshot::False, 0)));
return ret;
} catch (Error& e) {
wait(tr.onError(e));
}
}
}
ACTOR Future<Void> maybeFlushAndTruncateMutationLogs(Reference<BlobManagerData> bmData) {
try {
int64_t lastFlushTs = wait(getLastFlushTs(bmData->db));
bool shouldFlush = (now() - lastFlushTs) > SERVER_KNOBS->BLOB_RESTORE_MLOGS_RETENTION_SECS;
if (!shouldFlush) {
TraceEvent("SkipBlobGranulesFlush").detail("LastFlushTs", lastFlushTs);
return Void();
}
state std::string mlogsUrl = wait(getMutationLogUrl());
state Reference<IBackupContainer> bc = IBackupContainer::openContainer(mlogsUrl, {}, {});
state BackupDescription desc = wait(bc->describeBackup(true));
if (!desc.contiguousLogEnd.present()) {
TraceEvent("SkipBlobGranulesFlush").detail("LogUrl", mlogsUrl);
return Void(); // skip truncation if no valid backup for mutation logs
}
state Version logEndVersion = desc.contiguousLogEnd.get();
// Force flush in-memory data of all blob workers until end of log
FlushGranuleRequest req(bmData->epoch, normalKeys, logEndVersion, false);
wait(success(doBlobGranuleRequests(bmData->db, normalKeys, req, &BlobWorkerInterface::flushGranuleRequest)));
wait(updateLastFlushTs(bmData->db));
// Truncate mutation logs to max retention period
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(bmData->db));
Optional<int64_t> logEndEpochs = wait(timeKeeperEpochsFromVersion(logEndVersion, tr));
if (!logEndEpochs.present()) {
TraceEvent("SkipMutationLogTruncation").detail("LogEndVersion", logEndVersion);
return Void(); // skip truncation if no timestamp about log end
}
// Find timestamp and version to truncate
int64_t epochs = logEndEpochs.get() - SERVER_KNOBS->BLOB_RESTORE_MLOGS_RETENTION_SECS;
if (epochs <= 0) {
TraceEvent("SkipMutationLogTruncation").detail("Epochs", epochs);
return Void();
}
state std::string timestamp = BackupAgentBase::formatTime(epochs);
state Version truncVersion = wait(timeKeeperVersionFromDatetime(timestamp, bmData->db));
wait(bc->expireData(truncVersion, true));
TraceEvent("TruncateMutationLogs").detail("Version", truncVersion).detail("Timestamp", timestamp);
CODE_PROBE(true, "Flush blob granules and truncate mutation logs");
} catch (Error& e) {
TraceEvent("TruncateMutationLogsError").error(e); // skip and retry next time
}
return Void();
}
ACTOR Future<Void> backupManifest(Reference<BlobManagerData> bmData) {
bmData->initBStore();
@ -5256,8 +5348,9 @@ ACTOR Future<Void> backupManifest(Reference<BlobManagerData> bmData) {
break;
}
}
if (activeRanges) {
if (activeRanges && SERVER_KNOBS->BLOB_MANIFEST_BACKUP) {
if (bmData->manifestStore.isValid()) {
wait(maybeFlushAndTruncateMutationLogs(bmData));
wait(dumpManifest(bmData->db, bmData->manifestStore, bmData->epoch, bmData->manifestDumperSeqNo));
bmData->manifestDumperSeqNo++;
} else {

14
fdbserver/BlobManifest.actor.cpp
View File

@ -260,6 +260,7 @@ public:
self->segmentNo_ = 1;
self->totalRows_ = 0;
self->logicalSize_ = 0;
self->totalBytes_ = 0;
wait(waitForAll(self->pendingFutures_));
self->pendingFutures_.clear();
self->deleteSegmentFiles();
@ -999,3 +1000,16 @@ ACTOR Future<Version> getManifestVersion(Database db) {
}
}
}
ACTOR Future<std::string> getMutationLogUrl() {
state std::string baseUrl = SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL;
if (baseUrl.starts_with("file://")) {
state std::vector<std::string> containers = wait(IBackupContainer::listContainers(baseUrl, {}));
if (containers.size() == 0) {
throw blob_restore_missing_logs();
}
return containers.back();
} else {
return baseUrl;
}
}

18
fdbserver/BlobMigrator.actor.cpp
View File

@ -329,27 +329,15 @@ private:
// Apply mutation logs to blob granules so that they reach to a consistent version for all blob granules
ACTOR static Future<Void> applyMutationLogs(Reference<BlobMigrator> self) {
// check last version in mutation logs
state std::string baseUrl = SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL;
state std::string mlogsUrl;
if (baseUrl.starts_with("file://")) {
state std::vector<std::string> containers = wait(IBackupContainer::listContainers(baseUrl, {}));
if (containers.size() == 0) {
TraceEvent("MissingMutationLogs", self->interf_.id()).detail("Url", baseUrl);
throw restore_missing_data();
}
mlogsUrl = containers.front();
} else {
mlogsUrl = baseUrl;
}
state std::string mlogsUrl = wait(getMutationLogUrl());
state Reference<IBackupContainer> bc = IBackupContainer::openContainer(mlogsUrl, {}, {});
BackupDescription desc = wait(bc->describeBackup());
if (!desc.contiguousLogEnd.present()) {
TraceEvent("InvalidMutationLogs").detail("Url", baseUrl);
TraceEvent("InvalidMutationLogs").detail("Url", SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL);
throw blob_restore_missing_logs();
}
if (!desc.minLogBegin.present()) {
TraceEvent("InvalidMutationLogs").detail("Url", baseUrl);
TraceEvent("InvalidMutationLogs").detail("Url", SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL);
throw blob_restore_missing_logs();
}
state Version minLogVersion = desc.minLogBegin.get();

3
fdbserver/BlobWorker.actor.cpp
View File

@ -37,7 +37,6 @@
#include "fdbclient/Notified.h"
#include "fdbserver/BlobGranuleServerCommon.actor.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbclient/GetEncryptCipherKeys.actor.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/MutationTracking.h"
@ -1401,6 +1400,7 @@ ACTOR Future<BlobFileIndex> compactFromBlob(Reference<BlobWorkerData> bwData,
snapshotF.offset,
snapshotF.length,
snapshotF.fullFileLength,
snapshotF.version,
snapCipherKeysCtx);
compactBytesRead += snapshotF.length;
@ -1432,6 +1432,7 @@ ACTOR Future<BlobFileIndex> compactFromBlob(Reference<BlobWorkerData> bwData,
deltaF.offset,
deltaF.length,
deltaF.fullFileLength,
deltaF.version,
deltaCipherKeysCtx);
compactBytesRead += deltaF.length;
lastDeltaVersion = files.deltaFiles[deltaIdx].version;

1
fdbserver/ClusterRecovery.actor.cpp
View File

@ -24,7 +24,6 @@
#include "fdbserver/ApplyMetadataMutation.h"
#include "fdbserver/BackupProgress.actor.h"
#include "fdbserver/ClusterRecovery.actor.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/MasterInterface.h"
#include "fdbserver/WaitFailure.h"

21
fdbserver/CommitProxyServer.actor.cpp
View File

@ -42,7 +42,6 @@
#include "fdbserver/ApplyMetadataMutation.h"
#include "fdbserver/ConflictSet.h"
#include "fdbserver/DataDistributorInterface.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/FDBExecHelper.actor.h"
#include "fdbclient/GetEncryptCipherKeys.actor.h"
#include "fdbserver/IKeyValueStore.h"
@ -1691,6 +1690,8 @@ ACTOR Future<WriteMutationRefVar> writeMutationEncryptedMutation(CommitBatchCont
Arena* arena) {
state MutationRef encryptedMutation = encryptedMutationOpt->get();
state const BlobCipherEncryptHeader* header;
state BlobCipherEncryptHeaderRef headerRef;
state MutationRef decryptedMutation;
static_assert(TenantInfo::INVALID_TENANT == INVALID_ENCRYPT_DOMAIN_ID);
ASSERT(self->pProxyCommitData->encryptMode.isEncryptionEnabled());
@ -1698,9 +1699,15 @@ ACTOR Future<WriteMutationRefVar> writeMutationEncryptedMutation(CommitBatchCont
ASSERT(encryptedMutation.isEncrypted());
Reference<AsyncVar<ServerDBInfo> const> dbInfo = self->pProxyCommitData->db;
header = encryptedMutation.encryptionHeader();
TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, *header, BlobCipherMetrics::TLOG));
MutationRef decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
headerRef = encryptedMutation.configurableEncryptionHeader();
TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, headerRef, BlobCipherMetrics::TLOG));
decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG);
} else {
header = encryptedMutation.encryptionHeader();
TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, *header, BlobCipherMetrics::TLOG));
decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG);
}
ASSERT(decryptedMutation.type == mutation->type);
ASSERT(decryptedMutation.param1 == mutation->param1);
@ -2668,7 +2675,7 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
ssis.push_back(it->interf);
maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id());
}
rep.results.emplace_back(r.range(), ssis);
rep.results.emplace_back(TenantAPI::clampRangeToTenant(r.range(), req.tenant, req.arena), ssis);
} else if (!req.reverse) {
int count = 0;
for (auto r = commitData->keyInfo.rangeContaining(req.begin);
@ -2680,7 +2687,7 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
ssis.push_back(it->interf);
maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id());
}
rep.results.emplace_back(r.range(), ssis);
rep.results.emplace_back(TenantAPI::clampRangeToTenant(r.range(), req.tenant, req.arena), ssis);
count++;
}
} else {
@ -2693,7 +2700,7 @@ ACTOR static Future<Void> doKeyServerLocationRequest(GetKeyServerLocationsReques
ssis.push_back(it->interf);
maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id());
}
rep.results.emplace_back(r.range(), ssis);
rep.results.emplace_back(TenantAPI::clampRangeToTenant(r.range(), req.tenant, req.arena), ssis);
if (r == commitData->keyInfo.ranges().begin()) {
break;
}

172
fdbserver/KeyValueStoreMemory.actor.cpp
View File

@ -35,6 +35,8 @@
#include "flow/actorcompiler.h" // This must be the last #include.
#define OP_DISK_OVERHEAD (sizeof(OpHeader) + 1)
#define ENCRYPTION_ENABLED_BIT 31
static_assert(sizeof(uint32_t) == 4);
template <typename Container>
class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable {
@ -308,7 +310,7 @@ private:
OpCommit, // only in log, not in queue
OpRollback, // only in log, not in queue
OpSnapshotItemDelta,
OpEncrypted
OpEncrypted_Deprecated // deprecated since we now store the encryption status in the first bit of the opType
};
struct OpRef {
@ -319,7 +321,7 @@ private:
size_t expectedSize() const { return p1.expectedSize() + p2.expectedSize(); }
};
struct OpHeader {
int op;
uint32_t op;
int len1, len2;
};
@ -462,39 +464,56 @@ private:
return total;
}
// Data format for normal operation:
// +-------------+-------------+-------------+--------+--------+
// | opType | len1 | len2 | param2 | param2 |
// | sizeof(int) | sizeof(int) | sizeof(int) | len1 | len2 |
// +-------------+-------------+-------------+--------+--------+
static bool isOpEncrypted(OpHeader* header) { return header->op >> ENCRYPTION_ENABLED_BIT == 1; }
static void setEncryptFlag(OpHeader* header, bool set) {
if (set) {
header->op |= (1UL << ENCRYPTION_ENABLED_BIT);
} else {
header->op &= ~(1UL << ENCRYPTION_ENABLED_BIT);
}
}
// NOTE: The first bit of opType indicates whether the entry is encrypted or not. This is fine for backwards
// compatability since the first bit was never used previously
//
// However, if the operation is encrypted:
// +-------------+-------------+-------------+---------------------------------+-------------+--------+--------+
// | OpEncrypted | len1 | len2 | BlobCipherEncryptHeader | opType | param1 | param2 |
// | sizeof(int) | sizeof(int) | sizeof(int) | sizeof(BlobCipherEncryptHeader) | sizeof(int) | len1 | len2 |
// +-------------+-------------+-------------+---------------------------------+-------------+--------+--------+
// | plaintext | encrypted |
// +-----------------------------------------------------------------------------------------------------------+
// Unencrypted data format:
// +-------------+-------------+-------------+--------+--------+-----------+
// | opType | len1 | len2 | param2 | param2 | \x01 |
// | sizeof(int) | sizeof(int) | sizeof(int) | len1 | len2 | 1 byte |
// +-------------+-------------+-------------+--------+--------+-----------+
//
// Encrypted data format:
// +-------------+-------+---------+------------+-----------------------------+--------+--------+------------+
// | opType |len1 | len2 | headerSize | BlobCipherEncryptHeader | param1 | param2 | \x01 |
// | s(uint32) | s(int)| s(int) | s(uint16) | s(BlobCipherEncryptHeader) | len1 | len2 | 1 byte |
// +-------------+-------+---------+------------+-----------------------------+--------+--------+------------+
// | plaintext | encrypted | |
// +--------------------------------------------------------------------------+-----------------+------------+
//
IDiskQueue::location log_op(OpType op, StringRef v1, StringRef v2) {
// Metadata op types to be excluded from encryption.
static std::unordered_set<OpType> metaOps = { OpSnapshotEnd, OpSnapshotAbort, OpCommit, OpRollback };
uint32_t opType = (uint32_t)op;
// Make sure the first bit of the optype is empty
ASSERT(opType >> ENCRYPTION_ENABLED_BIT == 0);
if (!enableEncryption || metaOps.count(op) > 0) {
OpHeader h = { (int)op, v1.size(), v2.size() };
OpHeader h = { opType, v1.size(), v2.size() };
log->push(StringRef((const uint8_t*)&h, sizeof(h)));
log->push(v1);
log->push(v2);
} else {
OpHeader h = { (int)OpEncrypted, v1.size(), v2.size() };
OpHeader h = { opType, v1.size(), v2.size() };
// Set the first bit of the header to 1 to indicate that the log entry is encrypted
setEncryptFlag(&h, true);
log->push(StringRef((const uint8_t*)&h, sizeof(h)));
uint8_t* plaintext = new uint8_t[sizeof(int) + v1.size() + v2.size()];
*(int*)plaintext = op;
uint8_t* plaintext = new uint8_t[v1.size() + v2.size()];
if (v1.size()) {
memcpy(plaintext + sizeof(int), v1.begin(), v1.size());
memcpy(plaintext, v1.begin(), v1.size());
}
if (v2.size()) {
memcpy(plaintext + sizeof(int) + v1.size(), v2.begin(), v2.size());
memcpy(plaintext + v1.size(), v2.begin(), v2.size());
}
ASSERT(cipherKeys.cipherTextKey.isValid());
@ -504,12 +523,28 @@ private:
cipherKeys.cipherHeaderKey,
getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
BlobCipherMetrics::KV_MEMORY);
BlobCipherEncryptHeader cipherHeader;
uint16_t encryptHeaderSize;
// TODO: If possible we want to avoid memcpy to the disk log by using the same arena used by IDiskQueue
Arena arena;
StringRef ciphertext =
cipher.encrypt(plaintext, sizeof(int) + v1.size() + v2.size(), &cipherHeader, arena)->toStringRef();
log->push(StringRef((const uint8_t*)&cipherHeader, BlobCipherEncryptHeader::headerSize));
log->push(ciphertext);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef;
StringRef cipherText = cipher.encrypt(plaintext, v1.size() + v2.size(), &headerRef, arena);
Standalone<StringRef> headerRefStr = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
encryptHeaderSize = headerRefStr.size();
ASSERT(encryptHeaderSize > 0);
log->push(StringRef((const uint8_t*)&encryptHeaderSize, sizeof(encryptHeaderSize)));
log->push(headerRefStr);
log->push(cipherText);
} else {
BlobCipherEncryptHeader cipherHeader;
StringRef ciphertext =
cipher.encrypt(plaintext, v1.size() + v2.size(), &cipherHeader, arena)->toStringRef();
encryptHeaderSize = BlobCipherEncryptHeader::headerSize;
ASSERT(encryptHeaderSize > 0);
log->push(StringRef((const uint8_t*)&encryptHeaderSize, sizeof(encryptHeaderSize)));
log->push(StringRef((const uint8_t*)&cipherHeader, encryptHeaderSize));
log->push(ciphertext);
}
}
return log->push("\x01"_sr); // Changes here should be reflected in OP_DISK_OVERHEAD
}
@ -517,19 +552,38 @@ private:
// In case the op data is not encrypted, simply read the operands and the zero fill flag.
// Otherwise, decrypt the op type and data.
ACTOR static Future<Standalone<StringRef>> readOpData(KeyValueStoreMemory* self,
OpHeader* h,
OpHeader h,
bool* isZeroFilled,
int* zeroFillSize) {
int* zeroFillSize,
bool encryptedOp) {
ASSERT(!isOpEncrypted(&h));
// Metadata op types to be excluded from encryption.
static std::unordered_set<OpType> metaOps = { OpSnapshotEnd, OpSnapshotAbort, OpCommit, OpRollback };
if (metaOps.count((OpType)h->op) == 0) {
if (metaOps.count((OpType)h.op) == 0) {
// It is not supported to open an encrypted store as unencrypted, or vice-versa.
ASSERT_EQ(h->op == OpEncrypted, self->enableEncryption);
ASSERT_EQ(encryptedOp, self->enableEncryption);
}
state int remainingBytes = h->len1 + h->len2 + 1;
if (h->op == OpEncrypted) {
// if encrypted op read the header size
state uint16_t encryptHeaderSize = 0;
if (encryptedOp) {
state Standalone<StringRef> headerSizeStr = wait(self->log->readNext(sizeof(encryptHeaderSize)));
ASSERT(headerSizeStr.size() <= sizeof(encryptHeaderSize));
// Partial read on the header size
memset(&encryptHeaderSize, 0, sizeof(encryptHeaderSize));
memcpy(&encryptHeaderSize, headerSizeStr.begin(), headerSizeStr.size());
if (headerSizeStr.size() < sizeof(encryptHeaderSize)) {
CODE_PROBE(true, "zero fill partial encryption header size", probe::decoration::rare);
*zeroFillSize =
(sizeof(encryptHeaderSize) - headerSizeStr.size()) + encryptHeaderSize + h.len1 + h.len2 + 1;
}
if (*zeroFillSize > 0) {
return headerSizeStr;
}
}
state int remainingBytes = h.len1 + h.len2 + 1;
if (encryptedOp) {
// encryption header, plus the real (encrypted) op type
remainingBytes += BlobCipherEncryptHeader::headerSize + sizeof(int);
remainingBytes += encryptHeaderSize;
}
state Standalone<StringRef> data = wait(self->log->readNext(remainingBytes));
ASSERT(data.size() <= remainingBytes);
@ -537,23 +591,31 @@ private:
if (*zeroFillSize == 0) {
*isZeroFilled = (data[data.size() - 1] == 0);
}
if (h->op != OpEncrypted || *zeroFillSize > 0 || *isZeroFilled) {
if (!encryptedOp || *zeroFillSize > 0 || *isZeroFilled) {
return data;
}
state BlobCipherEncryptHeader cipherHeader = *(BlobCipherEncryptHeader*)data.begin();
TextAndHeaderCipherKeys cipherKeys =
wait(getEncryptCipherKeys(self->db, cipherHeader, BlobCipherMetrics::KV_MEMORY));
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, cipherHeader.iv, BlobCipherMetrics::KV_MEMORY);
Arena arena;
StringRef plaintext = cipher
.decrypt(data.begin() + BlobCipherEncryptHeader::headerSize,
sizeof(int) + h->len1 + h->len2,
cipherHeader,
arena)
->toStringRef();
h->op = *(int*)plaintext.begin();
return Standalone<StringRef>(plaintext.substr(sizeof(int)), arena);
state Arena arena;
state StringRef plaintext;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
state BlobCipherEncryptHeaderRef cipherHeaderRef =
BlobCipherEncryptHeaderRef::fromStringRef(StringRef(data.begin(), encryptHeaderSize));
TextAndHeaderCipherKeys cipherKeys =
wait(getEncryptCipherKeys(self->db, cipherHeaderRef, BlobCipherMetrics::KV_MEMORY));
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey,
cipherKeys.cipherHeaderKey,
cipherHeaderRef.getIV(),
BlobCipherMetrics::KV_MEMORY);
plaintext = cipher.decrypt(data.begin() + encryptHeaderSize, h.len1 + h.len2, cipherHeaderRef, arena);
} else {
state BlobCipherEncryptHeader cipherHeader = *(BlobCipherEncryptHeader*)data.begin();
TextAndHeaderCipherKeys cipherKeys =
wait(getEncryptCipherKeys(self->db, cipherHeader, BlobCipherMetrics::KV_MEMORY));
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, cipherHeader.iv, BlobCipherMetrics::KV_MEMORY);
plaintext =
cipher.decrypt(data.begin() + encryptHeaderSize, h.len1 + h.len2, cipherHeader, arena)->toStringRef();
}
return Standalone<StringRef>(plaintext, arena);
}
ACTOR static Future<Void> recover(KeyValueStoreMemory* self, bool exactRecovery) {
@ -586,6 +648,7 @@ private:
try {
loop {
state bool encryptedOp = false;
{
Standalone<StringRef> data = wait(self->log->readNext(sizeof(OpHeader)));
if (data.size() != sizeof(OpHeader)) {
@ -595,9 +658,11 @@ private:
memset(&h, 0, sizeof(OpHeader));
memcpy(&h, data.begin(), data.size());
zeroFillSize = sizeof(OpHeader) - data.size() + h.len1 + h.len2 + 1;
if (h.op == OpEncrypted) {
// encryption header, plus the real (encrypted) op type
zeroFillSize += BlobCipherEncryptHeader::headerSize + sizeof(int);
if (isOpEncrypted(&h)) {
// encrypt header size + encryption header
// If it's a partial header we assume the header size is 0 (this is fine since we
// don't read the header in this case)
zeroFillSize += 0 + sizeof(uint16_t);
}
}
TraceEvent("KVSMemRecoveryComplete", self->id)
@ -609,8 +674,13 @@ private:
break;
}
h = *(OpHeader*)data.begin();
encryptedOp = isOpEncrypted(&h);
// Reset the first bit to 0 so the op can be read properly
setEncryptFlag(&h, false);
ASSERT(h.op != OpEncrypted_Deprecated);
}
state Standalone<StringRef> data = wait(readOpData(self, &h, &isZeroFilled, &zeroFillSize));
state Standalone<StringRef> data =
wait(readOpData(self, h, &isZeroFilled, &zeroFillSize, encryptedOp));
if (zeroFillSize > 0) {
TraceEvent("KVSMemRecoveryComplete", self->id)
.detail("Reason", "data specified by header does not exist")

4
fdbserver/RESTKmsConnector.actor.cpp
View File

@ -778,10 +778,10 @@ Future<T> kmsRequestImpl(Reference<RESTKmsConnectorCtx> ctx,
ACTOR Future<Void> fetchEncryptionKeysByKeyIds(Reference<RESTKmsConnectorCtx> ctx, KmsConnLookupEKsByKeyIdsReq req) {
state KmsConnLookupEKsByKeyIdsRep reply;
try {
bool refreshKmsUrls = shouldRefreshKmsUrls(ctx);
StringRef requestBodyRef = getEncryptKeysByKeyIdsRequestBody(ctx, req, refreshKmsUrls, req.arena);
std::function<Standalone<VectorRef<EncryptCipherKeyDetailsRef>>(Reference<RESTKmsConnectorCtx>,
Reference<HTTP::Response>)>
f = &parseEncryptCipherResponse;
@ -1060,11 +1060,13 @@ ACTOR Future<Void> procureValidationTokensFromFiles(Reference<RESTKmsConnectorCt
ACTOR Future<Void> procureValidationTokens(Reference<RESTKmsConnectorCtx> ctx) {
std::string_view mode{ SERVER_KNOBS->REST_KMS_CONNECTOR_VALIDATION_TOKEN_MODE };
TraceEvent("ProcureValidationTokensStart", ctx->uid);
if (mode.compare("file") == 0) {
wait(procureValidationTokensFromFiles(ctx, SERVER_KNOBS->REST_KMS_CONNECTOR_VALIDATION_TOKEN_DETAILS));
} else {
throw not_implemented();
}
TraceEvent("ProcureValidationTokensDone", ctx->uid);
return Void();
}

1
fdbserver/Resolver.actor.cpp
View File

@ -26,7 +26,6 @@
#include "fdbclient/SystemData.h"
#include "fdbserver/ApplyMetadataMutation.h"
#include "fdbserver/ConflictSet.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/IKeyValueStore.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/LogSystem.h"

7
fdbserver/RestoreLoader.actor.cpp
View File

@ -372,13 +372,10 @@ void handleRestoreSysInfoRequest(const RestoreSysInfoRequest& req, Reference<Res
ACTOR static Future<MutationRef> _decryptMutation(MutationRef mutation, Database cx, Arena* arena) {
ASSERT(mutation.isEncrypted());
Reference<AsyncVar<ClientDBInfo> const> dbInfo = cx->clientInfo;
state const BlobCipherEncryptHeader* header = mutation.encryptionHeader();
std::unordered_set<BlobCipherDetails> cipherDetails;
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
mutation.updateEncryptCipherDetails(cipherDetails);
std::unordered_map<BlobCipherDetails, Reference<BlobCipherKey>> getCipherKeysResult =
wait(getEncryptCipherKeys(dbInfo, cipherDetails, BlobCipherMetrics::BACKUP));
return mutation.decrypt(getCipherKeysResult, *arena, BlobCipherMetrics::BACKUP);

3
fdbserver/SimKmsConnector.actor.cpp
View File

@ -134,6 +134,9 @@ ACTOR Future<Void> ekLookupByIds(Reference<SimKmsConnectorContext> ctx,
getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId.get(), itr->first), "");
}
} else {
TraceEvent("SimKmsEKLookupByIdsKeyNotFound")
.detail("DomId", item.domainId)
.detail("BaseCipherId", item.baseCipherId);
success = false;
break;
}

110
fdbserver/SimulatedCluster.actor.cpp
View File

@ -332,9 +332,6 @@ class TestConfig : public BasicTestConfig {
if (attrib == "disableRemoteKVS") {
disableRemoteKVS = strcmp(value.c_str(), "true") == 0;
}
if (attrib == "disableEncryption") {
disableEncryption = strcmp(value.c_str(), "true") == 0;
}
if (attrib == "encryptModes") {
std::stringstream ss(value);
std::string token;
@ -410,9 +407,6 @@ public:
bool disableHostname = false;
// remote key value store is a child process spawned by the SS process to run the storage engine
bool disableRemoteKVS = false;
// 7.2 cannot be downgraded to 7.1 or below after enabling encryption-at-rest.
// TODO: Remove this bool once the encryption knobs are removed
bool disableEncryption = false;
// By default, encryption mode is set randomly (based on the tenant mode)
// If provided, set using EncryptionAtRestMode::fromString
std::vector<std::string> encryptModes;
@ -493,7 +487,6 @@ public:
.add("disableTss", &disableTss)
.add("disableHostname", &disableHostname)
.add("disableRemoteKVS", &disableRemoteKVS)
.add("disableEncryption", &disableEncryption)
.add("encryptModes", &encryptModes)
.add("simpleConfig", &simpleConfig)
.add("generateFearless", &generateFearless)
@ -561,6 +554,11 @@ public:
}
}
bool excludedStorageEngineType(int storageEngineType) const {
return std::find(storageEngineExcludeTypes.begin(), storageEngineExcludeTypes.end(), storageEngineType) !=
storageEngineExcludeTypes.end();
}
TestConfig() = default;
TestConfig(const BasicTestConfig& config) : BasicTestConfig(config) {}
};
@ -1296,14 +1294,6 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
g_knobs.setKnob("remote_kv_store", KnobValueRef::create(bool{ false }));
TraceEvent(SevDebug, "DisableRemoteKVS");
}
// TODO: Remove this code when encryption knobs are removed
if (testConfig->disableEncryption) {
g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_tlog_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_storage_server_encryption", KnobValueRef::create(bool{ false }));
g_knobs.setKnob("enable_blob_granule_encryption", KnobValueRef::create(bool{ false }));
TraceEvent(SevDebug, "DisableEncryption");
}
*pConnString = conn;
*pTesterCount = testerCount;
bool usingSSL = conn.toString().find(":tls") != std::string::npos || listenersPerProcess > 1;
@ -1596,35 +1586,54 @@ void SimulationConfig::setTenantMode(const TestConfig& testConfig) {
}
void SimulationConfig::setEncryptionAtRestMode(const TestConfig& testConfig) {
EncryptionAtRestMode encryptionMode = EncryptionAtRestMode::DISABLED;
// Only Redwood support encryption. Disable encryption if non-Redwood storage engine is explicitly specified.
bool disableEncryption = testConfig.disableEncryption ||
(testConfig.storageEngineType.present() && testConfig.storageEngineType.get() != 3);
// TODO: Remove check on the ENABLE_ENCRYPTION knob once the EKP can start using the db config
if (!disableEncryption && (SERVER_KNOBS->ENABLE_ENCRYPTION || !testConfig.encryptModes.empty())) {
TenantMode tenantMode = db.tenantMode;
if (!testConfig.encryptModes.empty()) {
std::vector<EncryptionAtRestMode> validEncryptModes;
// Get the subset of valid encrypt modes given the tenant mode
for (int i = 0; i < testConfig.encryptModes.size(); i++) {
EncryptionAtRestMode encryptMode = EncryptionAtRestMode::fromString(testConfig.encryptModes.at(i));
if (encryptMode != EncryptionAtRestMode::DOMAIN_AWARE || tenantMode == TenantMode::REQUIRED) {
validEncryptModes.push_back(encryptMode);
}
}
if (validEncryptModes.size() > 0) {
encryptionMode = deterministicRandom()->randomChoice(validEncryptModes);
}
} else {
// TODO: These cases should only trigger with probability (BUGGIFY) once the server knob is removed
if (tenantMode == TenantMode::DISABLED || tenantMode == TenantMode::OPTIONAL_TENANT || BUGGIFY) {
// optional and disabled tenant modes currently only support cluster aware encryption
encryptionMode = EncryptionAtRestMode::CLUSTER_AWARE;
} else {
encryptionMode = EncryptionAtRestMode::DOMAIN_AWARE;
}
std::vector<bool> available;
std::vector<double> probability;
if (!testConfig.encryptModes.empty()) {
// If encryptModes are specified explicitly, give them equal probability to be chosen.
available = std::vector<bool>(EncryptionAtRestMode::END, false);
probability = std::vector<double>(EncryptionAtRestMode::END, 0);
for (auto& mode : testConfig.encryptModes) {
available[EncryptionAtRestMode::fromString(mode).mode] = true;
probability[EncryptionAtRestMode::fromString(mode).mode] = 1.0 / testConfig.encryptModes.size();
}
} else {
// If encryptModes are not specified, give encryption higher chance to be enabled.
// The good thing is testing with encryption on doesn't loss test coverage for most of the other features.
available = std::vector<bool>(EncryptionAtRestMode::END, true);
probability = { 0.25, 0.5, 0.25 };
// Only Redwood support encryption. Disable encryption if Redwood is not available.
if ((testConfig.storageEngineType.present() && testConfig.storageEngineType != 3) ||
testConfig.excludedStorageEngineType(3)) {
available[(int)EncryptionAtRestMode::DOMAIN_AWARE] = false;
available[(int)EncryptionAtRestMode::CLUSTER_AWARE] = false;
}
}
// domain_aware mode is supported only with required tenant mode.
if (db.tenantMode != TenantMode::REQUIRED) {
available[(int)EncryptionAtRestMode::DOMAIN_AWARE] = false;
}
int lastAvailableMode = EncryptionAtRestMode::END;
double totalProbability = 0;
for (int mode = 0; mode < (int)EncryptionAtRestMode::END; mode++) {
if (available[mode]) {
lastAvailableMode = mode;
totalProbability += probability[mode];
}
}
ASSERT(lastAvailableMode != EncryptionAtRestMode::END); // At least one mode available
double r = deterministicRandom()->random01() * totalProbability;
EncryptionAtRestMode encryptionMode;
for (int mode = 0;; mode++) {
if (available[mode] && (r < probability[mode] || mode == lastAvailableMode)) {
encryptionMode = (EncryptionAtRestMode::Mode)mode;
break;
}
r -= probability[mode];
}
TraceEvent("SimulatedClusterEncryptionMode").detail("Mode", encryptionMode.toString());
CODE_PROBE(encryptionMode == EncryptionAtRestMode::DISABLED, "Disabled encryption in simulation");
CODE_PROBE(encryptionMode == EncryptionAtRestMode::CLUSTER_AWARE, "Enabled cluster-aware encryption in simulation");
CODE_PROBE(encryptionMode == EncryptionAtRestMode::DOMAIN_AWARE, "Enabled domain-aware encryption in simulation");
set_config("encryption_at_rest_mode=" + encryptionMode.toString());
}
@ -1640,9 +1649,7 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
storage_engine_type = testConfig.storageEngineType.get();
} else {
// Continuously re-pick the storage engine type if it's the one we want to exclude
while (std::find(testConfig.storageEngineExcludeTypes.begin(),
testConfig.storageEngineExcludeTypes.end(),
storage_engine_type) != testConfig.storageEngineExcludeTypes.end()) {
while (testConfig.excludedStorageEngineType(storage_engine_type)) {
storage_engine_type = deterministicRandom()->randomInt(0, 6);
}
}
@ -2132,19 +2139,6 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
simconfig.db.tLogWriteAntiQuorum = testConfig.logAntiQuorum;
}
// TODO: remove knob hanlding once we move off from encrypption knobs to db config
if (simconfig.db.encryptionAtRestMode.mode == EncryptionAtRestMode::DISABLED) {
g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ false }));
CODE_PROBE(true, "Disabled encryption in simulation");
} else {
g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ true }));
CODE_PROBE(simconfig.db.encryptionAtRestMode.mode == EncryptionAtRestMode::CLUSTER_AWARE,
"Enabled cluster-aware encryption in simulation");
CODE_PROBE(simconfig.db.encryptionAtRestMode.mode == EncryptionAtRestMode::DOMAIN_AWARE,
"Enabled domain-aware encryption in simulation");
}
TraceEvent("SimulatedClusterEncryptionMode").detail("Mode", simconfig.db.encryptionAtRestMode.toString());
g_simulator->blobGranulesEnabled = simconfig.db.blobGranulesEnabled;
StatusObject startingConfigJSON = simconfig.db.toJSON(true);

2
fdbserver/Status.actor.cpp
View File

@ -854,7 +854,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
roles.addRole("consistency_scan", db->get().consistencyScan.get());
}
if (SERVER_KNOBS->ENABLE_ENCRYPTION && db->get().encryptKeyProxy.present()) {
if (db->get().encryptKeyProxy.present()) {
roles.addRole("encrypt_key_proxy", db->get().encryptKeyProxy.get());
}

6
fdbserver/StorageCache.actor.cpp
View File

@ -1925,11 +1925,7 @@ ACTOR Future<Void> pullAsyncData(StorageCacheData* data) {
cloneReader >> msg;
if (msg.isEncrypted()) {
if (!cipherKeys.present()) {
const BlobCipherEncryptHeader* header = msg.encryptionHeader();
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
msg.updateEncryptCipherDetails(cipherDetails);
collectingCipherKeys = true;
} else {
msg = msg.decrypt(cipherKeys.get(), cloneReader.arena(), BlobCipherMetrics::TLOG);

10
fdbserver/TLogServer.actor.cpp
View File

@ -2437,15 +2437,7 @@ ACTOR Future<EncryptionAtRestMode> getEncryptionAtRestMode(TLogData* self) {
when(GetEncryptionAtRestModeResponse resp = wait(brokenPromiseToNever(
self->dbInfo->get().clusterInterface.getEncryptionAtRestMode.getReply(req)))) {
TraceEvent("GetEncryptionAtRestMode", self->dbgid).detail("Mode", resp.mode);
// TODO: TLOG_ENCTYPTION KNOB shall be removed and db-config check should be sufficient to
// determine tlog (and cluster) encryption status
if ((EncryptionAtRestMode::Mode)resp.mode != EncryptionAtRestMode::Mode::DISABLED &&
SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION) {
return EncryptionAtRestMode((EncryptionAtRestMode::Mode)resp.mode);
} else {
return EncryptionAtRestMode();
}
return (EncryptionAtRestMode::Mode)resp.mode;
}
}
} catch (Error& e) {

53
fdbserver/VersionedBTree.actor.cpp
View File

@ -2815,6 +2815,13 @@ public:
wait(self->keyProviderInitialized.getFuture());
ASSERT(self->keyProvider.isValid());
}
if (self->keyProvider->expectedEncodingType() != page->getEncodingType()) {
TraceEvent(SevWarnAlways, "RedwoodBTreeUnexpectedNodeEncoding")
.detail("PhysicalPageID", page->getPhysicalPageID())
.detail("EncodingTypeFound", page->getEncodingType())
.detail("EncodingTypeExpected", self->keyProvider->expectedEncodingType());
throw unexpected_encoding_type();
}
ArenaPage::EncryptionKey k = wait(self->keyProvider->getEncryptionKey(page->getEncodingHeader()));
page->encryptionKey = k;
}
@ -2883,6 +2890,17 @@ public:
try {
page->postReadHeader(pageIDs.front());
if (page->isEncrypted()) {
if (!self->keyProvider.isValid()) {
wait(self->keyProviderInitialized.getFuture());
ASSERT(self->keyProvider.isValid());
}
if (self->keyProvider->expectedEncodingType() != page->getEncodingType()) {
TraceEvent(SevWarnAlways, "RedwoodBTreeUnexpectedNodeEncoding")
.detail("PhysicalPageID", page->getPhysicalPageID())
.detail("EncodingTypeFound", page->getEncodingType())
.detail("EncodingTypeExpected", self->keyProvider->expectedEncodingType());
throw unexpected_encoding_type();
}
ArenaPage::EncryptionKey k = wait(self->keyProvider->getEncryptionKey(page->getEncodingHeader()));
page->encryptionKey = k;
}
@ -3595,7 +3613,7 @@ public:
// The next section explicitly cancels all pending operations held in the pager
debug_printf("DWALPager(%s) shutdown kill ioLock\n", self->filename.c_str());
self->ioLock->kill();
self->ioLock->halt();
debug_printf("DWALPager(%s) shutdown cancel recovery\n", self->filename.c_str());
self->recoverFuture.cancel();
@ -3657,7 +3675,14 @@ public:
} else {
g_network->getDiskBytes(parentDirectory(filename), free, total);
}
int64_t pagerSize = header.pageCount * physicalPageSize;
// Size of redwood data file. Note that filePageCountPending is used here instead of filePageCount. This is
// because it is always >= filePageCount and accounts for file size changes which will complete soon.
int64_t pagerPhysicalSize = filePageCountPending * physicalPageSize;
// Size of the pager, which can be less than the data file size. All pages within this size are either in use
// in a data structure or accounted for in one of the pager's free page lists.
int64_t pagerLogicalSize = header.pageCount * physicalPageSize;
// It is not exactly known how many pages on the delayed free list are usable as of right now. It could be
// known, if each commit delayed entries that were freeable were shuffled from the delayed free queue to the
@ -3668,12 +3693,17 @@ public:
// Amount of space taken up by the free list queues themselves, as if we were to pop and use
// items on the free lists the space the items are stored in would also become usable
int64_t reusableQueueSpace = (freeList.numPages + delayedFreeList.numPages) * physicalPageSize;
int64_t reusable = reusablePageSpace + reusableQueueSpace;
// Pager slack is the space at the end of the pager's logical size until the end of the pager file's size.
// These pages will be used if needed without growing the file size.
int64_t reusablePagerSlackSpace = pagerPhysicalSize - pagerLogicalSize;
int64_t reusable = reusablePageSpace + reusableQueueSpace + reusablePagerSlackSpace;
// Space currently in used by old page versions have have not yet been freed due to the remap cleanup window.
int64_t temp = remapQueue.numEntries * physicalPageSize;
return StorageBytes(free, total, pagerSize - reusable, free + reusable, temp);
return StorageBytes(free, total, pagerPhysicalSize, free + reusable, temp);
}
int64_t getPageCacheCount() override { return pageCache.getCount(); }
@ -5255,6 +5285,7 @@ public:
.detail("InstanceName", self->m_pager->getName())
.detail("UsingEncodingType", self->m_encodingType)
.detail("ExistingEncodingType", self->m_header.encodingType);
throw unexpected_encoding_type();
}
// Verify if encryption mode and encoding type in the header are consistent.
// This check can also fail in case of authentication mode mismatch.
@ -6231,18 +6262,6 @@ private:
metrics.pageRead += 1;
metrics.pageReadExt += (id.size() - 1);
// If BTree encryption is enabled, pages read must be encrypted using the desired encryption type
if (self->m_enforceEncodingType && (page->getEncodingType() != self->m_encodingType)) {
Error e = unexpected_encoding_type();
TraceEvent(SevWarnAlways, "RedwoodBTreeUnexpectedNodeEncoding")
.error(e)
.detail("PhysicalPageID", page->getPhysicalPageID())
.detail("IsEncrypted", page->isEncrypted())
.detail("EncodingTypeFound", page->getEncodingType())
.detail("EncodingTypeExpected", self->m_encodingType);
throw e;
}
return std::move(page);
}
@ -11448,8 +11467,8 @@ TEST_CASE("/redwood/correctness/EnforceEncodingType") {
{}, // encryptionMode
reopenEncodingType,
encryptionKeyProviders.at(reopenEncodingType));
wait(kvs->init());
try {
wait(kvs->init());
Optional<Value> v = wait(kvs->readValue("foo"_sr));
UNREACHABLE();
} catch (Error& e) {

13
fdbserver/fdbserver.actor.cpp
View File

@ -2255,14 +2255,6 @@ int main(int argc, char* argv[]) {
}
}
}
g_knobs.setKnob("enable_encryption",
KnobValue::create(ini.GetBoolValue("META", "enableEncryption", false)));
g_knobs.setKnob("enable_tlog_encryption",
KnobValue::create(ini.GetBoolValue("META", "enableTLogEncryption", false)));
g_knobs.setKnob("enable_storage_server_encryption",
KnobValue::create(ini.GetBoolValue("META", "enableStorageServerEncryption", false)));
g_knobs.setKnob("enable_blob_granule_encryption",
KnobValue::create(ini.GetBoolValue("META", "enableBlobGranuleEncryption", false)));
g_knobs.setKnob("enable_blob_granule_compression",
KnobValue::create(ini.GetBoolValue("META", "enableBlobGranuleEncryption", false)));
g_knobs.setKnob("encrypt_header_auth_token_enabled",
@ -2270,6 +2262,11 @@ int main(int argc, char* argv[]) {
g_knobs.setKnob("encrypt_header_auth_token_algo",
KnobValue::create((int)ini.GetLongValue(
"META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO)));
g_knobs.setKnob("enable_configurable_encryption",
KnobValue::create(ini.GetBoolValue("META",
"enableConfigurableEncryption",
CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION)));
g_knobs.setKnob(
"shard_encode_location_metadata",
KnobValue::create(ini.GetBoolValue("META", "enableShardEncodeLocationMetadata", false)));

1
fdbserver/include/fdbserver/BlobGranuleServerCommon.actor.h
View File

@ -171,6 +171,7 @@ ACTOR Future<Optional<BlobRestoreStatus>> getRestoreStatus(Database db, KeyRange
ACTOR Future<Optional<BlobRestoreArg>> getRestoreArg(Database db, KeyRangeRef range);
ACTOR Future<Version> getRestoreTargetVersion(Database db, KeyRangeRef range, Version defaultVersion);
ACTOR Future<Version> getManifestVersion(Database db);
ACTOR Future<std::string> getMutationLogUrl();
#include "flow/unactorcompiler.h"
#endif

51
fdbserver/include/fdbserver/EncryptionOpsUtils.h
View File

@ -1,51 +0,0 @@
/*
* EncryptionOpUtils.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBSERVER_ENCRYPTION_OPS_UTIL_H
#define FDBSERVER_ENCRYPTION_OPS_UTIL_H
#pragma once
#include "fdbserver/Knobs.h"
#include "fdbclient/CommitProxyInterface.h"
typedef enum { TLOG_ENCRYPTION = 0, STORAGE_SERVER_ENCRYPTION = 1, BLOB_GRANULE_ENCRYPTION = 2 } EncryptOperationType;
inline bool isEncryptionOpSupported(EncryptOperationType operation_type) {
// We would check against dbInfo.isEncryptionEnabled instead, but the dbInfo may not be available before
// ClusterController broadcast the dbInfo to workers. Before the broadcast encryption may appear to be disabled
// when it should be enabled. Moving the encryption switch to DB config could fix the issue.
if (!SERVER_KNOBS->ENABLE_ENCRYPTION) {
return false;
}
if (operation_type == TLOG_ENCRYPTION) {
return SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION;
} else if (operation_type == STORAGE_SERVER_ENCRYPTION) {
return SERVER_KNOBS->ENABLE_STORAGE_SERVER_ENCRYPTION;
} else if (operation_type == BLOB_GRANULE_ENCRYPTION) {
bool supported = SERVER_KNOBS->ENABLE_BLOB_GRANULE_ENCRYPTION && SERVER_KNOBS->BG_METADATA_SOURCE == "tenant";
ASSERT((supported && SERVER_KNOBS->ENABLE_ENCRYPTION) || !supported);
return supported;
} else {
return false;
}
}
#endif // FDBSERVER_ENCRYPTION_OPS_UTIL_H

81
fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h
View File

@ -18,6 +18,7 @@
* limitations under the License.
*/
#include "fdbclient/Knobs.h"
#include "fdbclient/TenantManagement.actor.h"
#include "fdbrpc/TenantInfo.h"
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_IPAGEENCRYPTIONKEYPROVIDER_ACTOR_G_H)
@ -31,7 +32,6 @@
#include "fdbclient/SystemData.h"
#include "fdbclient/Tenant.h"
#include "fdbserver/EncryptionOpsUtils.h"
#include "fdbserver/IPager.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/ServerDBInfo.h"
@ -44,6 +44,7 @@
#include <functional>
#include <limits>
#include <tuple>
#include <type_traits>
#include "flow/actorcompiler.h" // This must be the last #include.
@ -158,6 +159,22 @@ public:
uint8_t xorWith;
};
namespace {
template <EncodingType encodingType>
int64_t getEncryptionDomainIdFromAesEncryptionHeader(const void* encodingHeader) {
using Encoder = typename ArenaPage::AESEncryptionEncoder<encodingType>;
using EncodingHeader = typename Encoder::Header;
ASSERT(encodingHeader != nullptr);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
return headerRef.getCipherDetails().textCipherDetails.encryptDomainId;
} else {
const BlobCipherEncryptHeader& header = reinterpret_cast<const EncodingHeader*>(encodingHeader)->encryption;
return header.cipherTextDetails.encryptDomainId;
}
}
} // anonymous namespace
// Key provider to provider cipher keys randomly from a pre-generated pool. It does not maintain encryption domains.
// Use for testing.
template <EncodingType encodingType,
@ -189,22 +206,37 @@ public:
bool enableEncryptionDomain() const override { return mode > 0; }
Future<EncryptionKey> getEncryptionKey(const void* encodingHeader) override {
using Header = typename ArenaPage::AESEncryptionEncoder<encodingType>::Header;
const Header* h = reinterpret_cast<const Header*>(encodingHeader);
using Encoder = typename ArenaPage::AESEncryptionEncoder<encodingType>;
EncryptionKey s;
s.aesKey.cipherTextKey =
getCipherKey(h->encryption.cipherTextDetails.encryptDomainId, h->encryption.cipherTextDetails.baseCipherId);
s.aesKey.cipherHeaderKey = getCipherKey(h->encryption.cipherHeaderDetails.encryptDomainId,
h->encryption.cipherHeaderDetails.baseCipherId);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
const BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
EncryptHeaderCipherDetails details = headerRef.getCipherDetails();
ASSERT(details.textCipherDetails.isValid());
s.aesKey.cipherTextKey =
getCipherKey(details.textCipherDetails.encryptDomainId, details.textCipherDetails.baseCipherId);
if (details.headerCipherDetails.present()) {
ASSERT(details.headerCipherDetails.get().isValid());
s.aesKey.cipherHeaderKey = getCipherKey(details.headerCipherDetails.get().encryptDomainId,
details.headerCipherDetails.get().baseCipherId);
}
} else {
const typename Encoder::Header* h = reinterpret_cast<const typename Encoder::Header*>(encodingHeader);
s.aesKey.cipherTextKey = getCipherKey(h->encryption.cipherTextDetails.encryptDomainId,
h->encryption.cipherTextDetails.baseCipherId);
if (h->encryption.cipherHeaderDetails.isValid()) {
s.aesKey.cipherHeaderKey = getCipherKey(h->encryption.cipherHeaderDetails.encryptDomainId,
h->encryption.cipherHeaderDetails.baseCipherId);
}
}
return s;
}
Future<EncryptionKey> getLatestEncryptionKey(int64_t domainId) override {
domainId = checkDomainId(domainId);
EncryptionKey s;
s.aesKey.cipherTextKey = getCipherKey(domainId, deterministicRandom()->randomInt(0, NUM_CIPHER));
s.aesKey.cipherTextKey = getCipherKey(domainId, deterministicRandom()->randomInt(1, NUM_CIPHER + 1));
s.aesKey.cipherHeaderKey =
getCipherKey(ENCRYPT_HEADER_DOMAIN_ID, deterministicRandom()->randomInt(0, NUM_CIPHER));
getCipherKey(ENCRYPT_HEADER_DOMAIN_ID, deterministicRandom()->randomInt(1, NUM_CIPHER + 1));
return s;
}
@ -222,10 +254,7 @@ public:
}
int64_t getEncryptionDomainIdFromHeader(const void* encodingHeader) override {
ASSERT(encodingHeader != nullptr);
using Header = typename ArenaPage::AESEncryptionEncoder<encodingType>::Header;
const Header* h = reinterpret_cast<const Header*>(encodingHeader);
return h->encryption.cipherTextDetails.encryptDomainId;
return getEncryptionDomainIdFromAesEncryptionHeader<encodingType>(encodingHeader);
}
private:
@ -260,11 +289,12 @@ private:
Reference<BlobCipherKey> getCipherKey(EncryptCipherDomainId domainId, EncryptCipherBaseKeyId cipherId) {
// Create a new cipher key by replacing the domain id.
ASSERT(cipherId > 0 && cipherId <= NUM_CIPHER);
return makeReference<BlobCipherKey>(domainId,
cipherId,
cipherKeys[cipherId]->rawBaseCipher(),
cipherKeys[cipherId - 1]->rawBaseCipher(),
AES_256_KEY_LENGTH,
cipherKeys[cipherId]->getSalt(),
cipherKeys[cipherId - 1]->getSalt(),
std::numeric_limits<int64_t>::max() /* refreshAt */,
std::numeric_limits<int64_t>::max() /* expireAt */);
}
@ -282,7 +312,8 @@ template <EncodingType encodingType,
true>
class AESEncryptionKeyProvider : public IPageEncryptionKeyProvider {
public:
using EncodingHeader = typename ArenaPage::AESEncryptionEncoder<encodingType>::Header;
using Encoder = typename ArenaPage::AESEncryptionEncoder<encodingType>;
using EncodingHeader = typename Encoder::Header;
const StringRef systemKeysPrefix = systemKeys.begin;
@ -303,9 +334,17 @@ public:
}
ACTOR static Future<EncryptionKey> getEncryptionKey(AESEncryptionKeyProvider* self, const void* encodingHeader) {
const BlobCipherEncryptHeader& header = reinterpret_cast<const EncodingHeader*>(encodingHeader)->encryption;
TextAndHeaderCipherKeys cipherKeys =
wait(getEncryptCipherKeys(self->db, header, BlobCipherMetrics::KV_REDWOOD));
state TextAndHeaderCipherKeys cipherKeys;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader);
TextAndHeaderCipherKeys cks =
wait(getEncryptCipherKeys(self->db, headerRef, BlobCipherMetrics::KV_REDWOOD));
cipherKeys = cks;
} else {
const BlobCipherEncryptHeader& header = reinterpret_cast<const EncodingHeader*>(encodingHeader)->encryption;
TextAndHeaderCipherKeys cks = wait(getEncryptCipherKeys(self->db, header, BlobCipherMetrics::KV_REDWOOD));
cipherKeys = cks;
}
EncryptionKey encryptionKey;
encryptionKey.aesKey = cipherKeys;
return encryptionKey;
@ -355,9 +394,7 @@ public:
}
int64_t getEncryptionDomainIdFromHeader(const void* encodingHeader) override {
ASSERT(encodingHeader != nullptr);
const BlobCipherEncryptHeader& header = reinterpret_cast<const EncodingHeader*>(encodingHeader)->encryption;
return header.cipherTextDetails.encryptDomainId;
return getEncryptionDomainIdFromAesEncryptionHeader<encodingType>(encodingHeader);
}
private:

61
fdbserver/include/fdbserver/IPager.h
View File

@ -19,6 +19,7 @@
*/
#pragma once
#include "fdbclient/Knobs.h"
#ifndef FDBSERVER_IPAGER_H
#define FDBSERVER_IPAGER_H
@ -387,23 +388,40 @@ public:
// By default, xxhash is used to checksum the page. But ff authentication is enabled (such as when we are using
// aes256-ctr-hmac-sha256 encryption scheme), the auth tag plays the role of a checksum while assuring authenticity
// of the data. xxhash checksum is not needed in this case.
//
// To support configurable encryption, which may come with variable size encryption header, we assume the encryption
// header size is no larger than that of BlobCipherEncryptHeader. This is true for current supported encryption
// header format types. Moving forward, the plan is to make IPager support variable size encoding header, and let
// Redwood rebuild a page when it tries to in-place update the page, but the reserved buffer for the encoding header
// is not large enough.
// TODO(yiwu): Cleanup the old encryption header, and update headerSize to be the maximum size of the supported
// encryption header format type.
// TODO(yiwu): Support variable size encoding header.
template <EncodingType encodingType,
typename std::enable_if<encodingType == AESEncryption || encodingType == AESEncryptionWithAuth,
bool>::type = true>
struct AESEncryptionEncoder {
struct AESEncryptionEncodingHeader {
BlobCipherEncryptHeader encryption;
XXH64_hash_t checksum;
union {
BlobCipherEncryptHeader encryption;
uint8_t encryptionHeaderBuf[0]; // for configurable encryption
};
};
struct AESEncryptionWithAuthEncodingHeader {
BlobCipherEncryptHeader encryption;
union {
BlobCipherEncryptHeader encryption;
uint8_t encryptionHeaderBuf[0]; // for configurable encryption
};
};
using Header = typename std::conditional<encodingType == AESEncryption,
AESEncryptionEncodingHeader,
AESEncryptionWithAuthEncodingHeader>::type;
static constexpr size_t headerSize = sizeof(Header);
static void encode(void* header,
const TextAndHeaderCipherKeys& cipherKeys,
uint8_t* payload,
@ -415,7 +433,19 @@ public:
getEncryptAuthTokenMode(ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE),
BlobCipherMetrics::KV_REDWOOD);
Arena arena;
StringRef ciphertext = cipher.encrypt(payload, len, &h->encryption, arena)->toStringRef();
StringRef ciphertext;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef;
ciphertext = cipher.encrypt(payload, len, &headerRef, arena);
Standalone<StringRef> serializedHeader = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
ASSERT(serializedHeader.size() <= headerSize);
memcpy(h->encryptionHeaderBuf, serializedHeader.begin(), serializedHeader.size());
if (serializedHeader.size() < headerSize) {
memset(h->encryptionHeaderBuf + serializedHeader.size(), 0, headerSize - serializedHeader.size());
}
} else {
ciphertext = cipher.encrypt(payload, len, &h->encryption, arena)->toStringRef();
}
ASSERT_EQ(len, ciphertext.size());
memcpy(payload, ciphertext.begin(), len);
if constexpr (encodingType == AESEncryption) {
@ -423,6 +453,13 @@ public:
}
}
static BlobCipherEncryptHeaderRef getEncryptionHeaderRef(const void* header) {
ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
const Header* h = reinterpret_cast<const Header*>(header);
return BlobCipherEncryptHeaderRef::fromStringRef(
StringRef(h->encryptionHeaderBuf, headerSize - (h->encryptionHeaderBuf - (const uint8_t*)h)));
}
static void decode(void* header,
const TextAndHeaderCipherKeys& cipherKeys,
uint8_t* payload,
@ -434,10 +471,22 @@ public:
throw page_decoding_failed();
}
}
DecryptBlobCipherAes256Ctr cipher(
cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, h->encryption.iv, BlobCipherMetrics::KV_REDWOOD);
Arena arena;
StringRef plaintext = cipher.decrypt(payload, len, h->encryption, arena)->toStringRef();
StringRef plaintext;
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef = getEncryptionHeaderRef(header);
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey,
cipherKeys.cipherHeaderKey,
headerRef.getIV(),
BlobCipherMetrics::KV_REDWOOD);
plaintext = cipher.decrypt(payload, len, headerRef, arena);
} else {
DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey,
cipherKeys.cipherHeaderKey,
h->encryption.iv,
BlobCipherMetrics::KV_REDWOOD);
plaintext = cipher.decrypt(payload, len, h->encryption, arena)->toStringRef();
}
ASSERT_EQ(len, plaintext.size());
memcpy(payload, plaintext.begin(), len);
}

2
fdbserver/include/fdbserver/ProxyCommitData.actor.h
View File

@ -19,8 +19,6 @@
*/
#pragma once
#include "fdbserver/EncryptionOpsUtils.h"
#include <unordered_map>
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H)
#define FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H
#include "fdbserver/ProxyCommitData.actor.g.h"

91
fdbserver/storageserver.actor.cpp
View File

@ -159,6 +159,7 @@ bool canReplyWith(Error e) {
return false;
}
}
} // namespace
#define PERSIST_PREFIX "\xff\xff"
@ -854,7 +855,6 @@ public:
void clearTenants(StringRef startTenant, StringRef endTenant, Version version);
void checkTenantEntry(Version version, TenantInfo tenant);
KeyRangeRef clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena);
std::vector<StorageServerShard> getStorageServerShards(KeyRangeRef range);
@ -2078,6 +2078,7 @@ ACTOR Future<Version> waitForMinVersion(StorageServer* data, Version version) {
void StorageServer::checkTenantEntry(Version version, TenantInfo tenantInfo) {
if (tenantInfo.hasTenant()) {
ASSERT(version == latestVersion || (version >= tenantMap.oldestVersion && version <= this->version.get()));
auto view = tenantMap.at(version);
auto itr = view.find(tenantInfo.tenantId);
if (itr == view.end()) {
@ -3020,11 +3021,7 @@ ACTOR Future<std::pair<ChangeFeedStreamReply, bool>> getChangeFeedMutations(Stor
if (doFilterMutations || !req.encrypted) {
for (auto& m : decodedMutations.back().first) {
if (m.isEncrypted()) {
const BlobCipherEncryptHeader* header = m.encryptionHeader();
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
m.updateEncryptCipherDetails(cipherDetails);
}
}
}
@ -4038,17 +4035,6 @@ ACTOR Future<GetKeyValuesReply> readRange(StorageServer* data,
return result;
}
KeyRangeRef StorageServer::clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena) {
if (tenantInfo.hasTenant()) {
return KeyRangeRef(range.begin.startsWith(tenantInfo.prefix.get()) ? range.begin : tenantInfo.prefix.get(),
range.end.startsWith(tenantInfo.prefix.get())
? range.end
: allKeys.end.withPrefix(tenantInfo.prefix.get(), arena));
} else {
return range;
}
}
ACTOR Future<Key> findKey(StorageServer* data,
KeySelectorRef sel,
Version version,
@ -4250,7 +4236,7 @@ ACTOR Future<Void> getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req)
throw wrong_shard_server();
}
KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena);
KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena);
state int offset1 = 0;
state int offset2;
@ -5415,7 +5401,7 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
throw wrong_shard_server();
}
KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena);
KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena);
state int offset1 = 0;
state int offset2;
@ -5616,7 +5602,7 @@ ACTOR Future<Void> getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe
throw wrong_shard_server();
}
KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena);
KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena);
state int offset1 = 0;
state int offset2;
@ -5806,7 +5792,7 @@ ACTOR Future<Void> getKeyQ(StorageServer* data, GetKeyRequest req) {
state uint64_t changeCounter = data->shardChangeCounter;
KeyRange shard = getShardKeyRange(data, req.sel);
KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena);
KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena);
state int offset;
Key absoluteKey = wait(findKey(data, req.sel, version, searchRange, &offset, req.spanContext, req.options));
@ -6854,9 +6840,7 @@ ACTOR Future<Version> fetchChangeFeedApplier(StorageServer* data,
data->counters.feedBytesFetched += remoteResult.expectedSize();
data->fetchKeysBytesBudget -= remoteResult.expectedSize();
if (data->fetchKeysBytesBudget <= 0) {
data->fetchKeysBudgetUsed.set(true);
}
data->fetchKeysBudgetUsed.set(data->fetchKeysBytesBudget <= 0);
wait(yield());
}
} catch (Error& e) {
@ -7574,16 +7558,15 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
metricReporter.addFetchedBytes(expectedBlockSize, this_block.size());
// Write this_block to storage
state int sinceYield = 0;
state KeyValueRef* kvItr = this_block.begin();
for (; kvItr != this_block.end(); ++kvItr) {
data->storage.writeKeyValue(*kvItr);
wait(yield());
}
kvItr = this_block.begin();
for (; kvItr != this_block.end(); ++kvItr) {
data->byteSampleApplySet(*kvItr, invalidVersion);
wait(yield());
if (++sinceYield > 1000) {
wait(yield());
sinceYield = 0;
}
}
ASSERT(this_block.readThrough.present() || this_block.size());
@ -7592,9 +7575,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
this_block = RangeResult();
data->fetchKeysBytesBudget -= expectedBlockSize;
if (data->fetchKeysBytesBudget <= 0) {
data->fetchKeysBudgetUsed.set(true);
}
data->fetchKeysBudgetUsed.set(data->fetchKeysBytesBudget <= 0);
}
} catch (Error& e) {
if (e.code() != error_code_end_of_stream && e.code() != error_code_connection_failed &&
@ -9257,11 +9238,7 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
}
if (msg.isEncrypted()) {
if (!cipherKeys.present()) {
const BlobCipherEncryptHeader* header = msg.encryptionHeader();
cipherDetails.insert(header->cipherTextDetails);
if (header->cipherHeaderDetails.isValid()) {
cipherDetails.insert(header->cipherHeaderDetails);
}
msg.updateEncryptCipherDetails(cipherDetails);
collectingCipherKeys = true;
} else {
msg = msg.decrypt(cipherKeys.get(), eager.arena, BlobCipherMetrics::TLOG);
@ -9668,6 +9645,8 @@ ACTOR Future<Void> createCheckpoint(StorageServer* data, CheckpointMetaData meta
ACTOR Future<Void> updateStorage(StorageServer* data) {
state UnlimitedCommitBytes unlimitedCommitBytes = UnlimitedCommitBytes::False;
state Future<Void> durableDelay = Void();
loop {
unlimitedCommitBytes = UnlimitedCommitBytes::False;
ASSERT(data->durableVersion.get() == data->storageVersion());
@ -9678,7 +9657,19 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
wait(delay(endTime - now(), TaskPriority::UpdateStorage));
}
}
wait(data->desiredOldestVersion.whenAtLeast(data->storageVersion() + 1));
// If the fetch keys budget is not used up then we have already waited for the storage commit delay so
// wait for either a new mutation version or the budget to be used up.
// Otherwise, don't wait at all.
if (!data->fetchKeysBudgetUsed.get()) {
wait(data->desiredOldestVersion.whenAtLeast(data->storageVersion() + 1) ||
data->fetchKeysBudgetUsed.onChange());
}
// Yield to TaskPriority::UpdateStorage in case more mutations have arrived but were not processed yet.
// If the fetch keys budget has already been used up, then we likely arrived here without waiting the
// full post storage commit delay, so this will allow the update actor to process some mutations
// before we proceed.
wait(delay(0, TaskPriority::UpdateStorage));
state Promise<Void> durableInProgress;
@ -9783,6 +9774,17 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
break;
}
// Allow data fetch to use an additional bytesLeft but don't penalize fetch budget if bytesLeft is negative
if (bytesLeft > 0) {
data->fetchKeysBytesBudget += bytesLeft;
data->fetchKeysBudgetUsed.set(data->fetchKeysBytesBudget <= 0);
// Dependng on how negative the fetchKeys budget was it could still be used up
if (!data->fetchKeysBudgetUsed.get()) {
wait(durableDelay || data->fetchKeysBudgetUsed.onChange());
}
}
if (addedRanges) {
TraceEvent(SevVerbose, "SSAddKVSRangeMetaData", data->thisServerID)
.detail("NewDurableVersion", newOldestVersion)
@ -9882,11 +9884,10 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
wait(data->storage.canCommit());
state Future<Void> durable = data->storage.commit();
++data->counters.kvCommits;
state Future<Void> durableDelay = Void();
if (bytesLeft > 0) {
durableDelay = delay(SERVER_KNOBS->STORAGE_COMMIT_INTERVAL, TaskPriority::UpdateStorage);
}
// If the mutation bytes budget was not fully used then wait some time before the next commit
durableDelay =
(bytesLeft > 0) ? delay(SERVER_KNOBS->STORAGE_COMMIT_INTERVAL, TaskPriority::UpdateStorage) : Void();
wait(ioTimeoutError(durable, SERVER_KNOBS->MAX_STORAGE_COMMIT_TIME, "StorageCommit"));
data->storageCommitLatencyHistogram->sampleSeconds(now() - beforeStorageCommit);
@ -11711,7 +11712,7 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
// If the storage server dies while something that uses self is still on the stack,
// we want that actor to complete before we terminate and that memory goes out of scope
self.ssLock->kill();
self.ssLock->halt();
state Error err = e;
if (storageServerTerminated(self, persistentData, err)) {
@ -11804,7 +11805,7 @@ ACTOR Future<Void> storageServer(IKeyValueStore* persistentData,
throw internal_error();
} catch (Error& e) {
self.ssLock->kill();
self.ssLock->halt();
if (self.byteSampleRecovery.isValid()) {
self.byteSampleRecovery.cancel();

2
fdbserver/tester.actor.cpp
View File

@ -1388,8 +1388,6 @@ std::map<std::string, std::function<void(const std::string&)>> testSpecGlobalKey
[](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableHostname", ""); } },
{ "disableRemoteKVS",
[](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedRemoteKVS", ""); } },
{ "disableEncryption",
[](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedEncryption", ""); } },
{ "allowDefaultTenant",
[](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDefaultTenant", ""); } }
};

72
fdbserver/workloads/AuthzSecurity.actor.cpp
View File

@ -54,14 +54,15 @@ struct AuthzSecurityWorkload : TestWorkload {
WipedString signedTokenAnotherTenant;
Standalone<StringRef> tLogConfigKey;
PerfIntCounter crossTenantGetPositive, crossTenantGetNegative, crossTenantCommitPositive, crossTenantCommitNegative,
publicNonTenantRequestPositive, tLogReadNegative;
publicNonTenantRequestPositive, tLogReadNegative, keyLocationLeakNegative;
std::vector<std::function<Future<Void>(Database cx)>> testFunctions;
AuthzSecurityWorkload(WorkloadContext const& wcx)
: TestWorkload(wcx), crossTenantGetPositive("CrossTenantGetPositive"),
crossTenantGetNegative("CrossTenantGetNegative"), crossTenantCommitPositive("CrossTenantCommitPositive"),
crossTenantCommitNegative("CrossTenantCommitNegative"),
publicNonTenantRequestPositive("PublicNonTenantRequestPositive"), tLogReadNegative("TLogReadNegative") {
publicNonTenantRequestPositive("PublicNonTenantRequestPositive"), tLogReadNegative("TLogReadNegative"),
keyLocationLeakNegative("KeyLocationLeakNegative") {
testDuration = getOption(options, "testDuration"_sr, 10.0);
transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 500.0) / clientCount;
actorCount = getOption(options, "actorsPerClient"_sr, transactionsPerSecond / 5);
@ -81,6 +82,7 @@ struct AuthzSecurityWorkload : TestWorkload {
testFunctions.push_back(
[this](Database cx) { return testPublicNonTenantRequestsAllowedWithoutTokens(this, cx); });
testFunctions.push_back([this](Database cx) { return testTLogReadDisallowed(this, cx); });
testFunctions.push_back([this](Database cx) { return testKeyLocationLeakDisallowed(this, cx); });
}
Future<Void> setup(Database const& cx) override {
@ -108,7 +110,8 @@ struct AuthzSecurityWorkload : TestWorkload {
clients.clear();
return errors == 0 && crossTenantGetPositive.getValue() > 0 && crossTenantGetNegative.getValue() > 0 &&
crossTenantCommitPositive.getValue() > 0 && crossTenantCommitNegative.getValue() > 0 &&
publicNonTenantRequestPositive.getValue() > 0 && tLogReadNegative.getValue() > 0;
publicNonTenantRequestPositive.getValue() > 0 && tLogReadNegative.getValue() > 0 &&
keyLocationLeakNegative.getValue() > 0;
}
void getMetrics(std::vector<PerfMetric>& m) override {
@ -118,6 +121,7 @@ struct AuthzSecurityWorkload : TestWorkload {
m.push_back(crossTenantCommitNegative.getMetric());
m.push_back(publicNonTenantRequestPositive.getMetric());
m.push_back(tLogReadNegative.getMetric());
m.push_back(keyLocationLeakNegative.getMetric());
}
void setAuthToken(Transaction& tr, StringRef token) {
@ -400,6 +404,68 @@ struct AuthzSecurityWorkload : TestWorkload {
return Void();
}
ACTOR static Future<Void> testKeyLocationLeakDisallowed(AuthzSecurityWorkload* self, Database cx) {
state Key key = self->randomString();
state Value value = self->randomString();
state Version v1 =
wait(setAndCommitKeyValueAndGetVersion(self, cx, self->tenant, self->signedToken, key, value));
state Version v2 = wait(setAndCommitKeyValueAndGetVersion(
self, cx, self->anotherTenant, self->signedTokenAnotherTenant, key, value));
{
GetKeyServerLocationsReply rep =
wait(basicLoadBalance(cx->getCommitProxies(UseProvisionalProxies::False),
&CommitProxyInterface::getKeyServersLocations,
GetKeyServerLocationsRequest(SpanContext(),
TenantInfo(self->tenant->id(), self->signedToken),
key,
Optional<KeyRef>(),
100,
false,
v2,
Arena())));
for (auto const& [range, ssIfaces] : rep.results) {
if (!range.begin.startsWith(self->tenant->prefix())) {
TraceEvent(SevError, "AuthzSecurityKeyRangeLeak")
.detail("TenantId", self->tenant->id())
.detail("LeakingRangeBegin", range.begin.printable());
}
if (!range.end.startsWith(self->tenant->prefix())) {
TraceEvent(SevError, "AuthzSecurityKeyRangeLeak")
.detail("TenantId", self->tenant->id())
.detail("LeakingRangeEnd", range.end.printable());
}
}
}
{
GetKeyServerLocationsReply rep = wait(basicLoadBalance(
cx->getCommitProxies(UseProvisionalProxies::False),
&CommitProxyInterface::getKeyServersLocations,
GetKeyServerLocationsRequest(SpanContext(),
TenantInfo(self->anotherTenant->id(), self->signedTokenAnotherTenant),
key,
Optional<KeyRef>(),
100,
false,
v2,
Arena())));
for (auto const& [range, ssIfaces] : rep.results) {
if (!range.begin.startsWith(self->anotherTenant->prefix())) {
TraceEvent(SevError, "AuthzSecurityKeyRangeLeak")
.detail("TenantId", self->anotherTenant->id())
.detail("LeakingRangeBegin", range.begin.printable());
}
if (!range.end.startsWith(self->anotherTenant->prefix())) {
TraceEvent(SevError, "AuthzSecurityKeyRangeLeak")
.detail("TenantId", self->anotherTenant->id())
.detail("LeakingRangeEnd", range.end.printable());
}
}
}
++self->keyLocationLeakNegative;
return Void();
}
ACTOR static Future<Void> runTestClient(AuthzSecurityWorkload* self, Database cx) {
state double lastTime = now();
state double delay = self->actorCount / self->transactionsPerSecond;

16
fdbserver/workloads/BlobGranuleRangesWorkload.actor.cpp
View File

@ -553,8 +553,8 @@ struct BlobGranuleRangesWorkload : TestWorkload {
ASSERT(!fail6);
bool fail7 = wait(cx->blobbifyRange(KeyRangeRef(middleKey, activeRange.end), self->tenant));
ASSERT(!fail7);
ASSERT(!fail7);
bool fail8 = wait(cx->blobbifyRange(KeyRangeRef(middleKey, middleKey2), self->tenant));
ASSERT(!fail8);
@ -681,6 +681,15 @@ struct BlobGranuleRangesWorkload : TestWorkload {
return Void();
}
ACTOR Future<Void> blobbifyBlockingUnit(Database cx, BlobGranuleRangesWorkload* self, KeyRange range) {
bool setSuccess = wait(cx->blobbifyRangeBlocking(range, self->tenant));
ASSERT(setSuccess);
bool verifySuccess = wait(self->isRangeActive(cx, range, self->tenant));
ASSERT(verifySuccess);
return Void();
}
enum UnitTestTypes {
VERIFY_RANGE_UNIT,
VERIFY_RANGE_GAP_UNIT,
@ -688,7 +697,8 @@ struct BlobGranuleRangesWorkload : TestWorkload {
BLOBBIFY_IDEMPOTENT,
RE_BLOBBIFY,
ADJACENT_PURGE,
OP_COUNT = 6 /* keep this last */
BLOBBIFY_BLOCKING_UNIT,
OP_COUNT = 7 /* keep this last */
};
ACTOR Future<Void> blobGranuleRangesUnitTests(Database cx, BlobGranuleRangesWorkload* self) {
@ -732,6 +742,8 @@ struct BlobGranuleRangesWorkload : TestWorkload {
wait(self->reBlobbifyUnit(cx, self, range));
} else if (op == ADJACENT_PURGE) {
wait(self->adjacentPurge(cx, self, range));
} else if (op == BLOBBIFY_BLOCKING_UNIT) {
wait(self->blobbifyBlockingUnit(cx, self, range));
} else {
ASSERT(false);
}

22
fdbserver/workloads/BlobGranuleVerifier.actor.cpp
View File

@ -168,27 +168,13 @@ struct BlobGranuleVerifierWorkload : TestWorkload {
}
}
// FIXME: run the actual FDBCLI command instead of copy/pasting its implementation
// Sets the whole user keyspace to be blobified
ACTOR Future<Void> setUpBlobRange(Database cx) {
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr->set(blobRangeChangeKey, deterministicRandom()->randomUniqueID().toString());
wait(krmSetRange(tr, blobRangeKeys.begin, KeyRange(normalKeys), "1"_sr));
wait(tr->commit());
if (BGV_DEBUG) {
printf("Successfully set up blob granule range for normalKeys\n");
}
TraceEvent("BlobGranuleVerifierSetup");
return Void();
} catch (Error& e) {
wait(tr->onError(e));
}
}
bool success = wait(cx->blobbifyRange(normalKeys));
ASSERT(success);
return Void();
}
void disableFailureInjectionWorkloads(std::set<std::string>& out) const override { out.emplace("Attrition"); }
Future<Void> setup(Database const& cx) override { return _setup(cx, this); }

8
fdbserver/workloads/BlobRestoreWorkload.actor.cpp
View File

@ -25,7 +25,6 @@
#include "fdbclient/BackupContainer.h"
#include "fdbclient/BackupContainerFileSystem.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/Knobs.h"
#include "fdbclient/SystemData.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/BlobGranuleServerCommon.actor.h"
@ -73,6 +72,10 @@ struct BlobRestoreWorkload : TestWorkload {
if (self->performRestore_) {
fmt::print("Perform blob restore\n");
// disable manifest backup and log truncation
KnobValueRef knobFalse = KnobValueRef::create(bool{ false });
IKnobCollection::getMutableGlobalKnobCollection().setKnob("blob_manifest_backup", knobFalse);
wait(store(result, self->extraDb_->blobRestore(normalKeys, {})));
state std::vector<Future<Void>> futures;
@ -178,8 +181,9 @@ struct BlobRestoreWorkload : TestWorkload {
if (src[i].value != dest[i].value) {
fmt::print("Value mismatch at {}\n", i);
TraceEvent(SevError, "TestFailure")
.detail("Reason", "Key Mismatch")
.detail("Reason", "Value Mismatch")
.detail("Index", i)
.detail("Key", src[i].key.printable())
.detail("SrcValue", src[i].value.printable())
.detail("DestValue", dest[i].value.printable());
return false;

26
fdbserver/workloads/EncryptionOps.actor.cpp
View File

@ -120,7 +120,6 @@ struct EncryptionOpsWorkload : TestWorkload {
std::unique_ptr<uint8_t[]> buff;
int enableTTLTest;
Arena arena;
std::unique_ptr<WorkloadMetrics> metrics;
EncryptCipherDomainId minDomainId;
@ -279,7 +278,8 @@ struct EncryptionOpsWorkload : TestWorkload {
int len,
const EncryptAuthTokenMode authMode,
const EncryptAuthTokenAlgo authAlgo,
BlobCipherEncryptHeader* header) {
BlobCipherEncryptHeader* header,
Arena& arena) {
uint8_t iv[AES_256_IV_LENGTH];
deterministicRandom()->randomBytes(&iv[0], AES_256_IV_LENGTH);
EncryptBlobCipherAes265Ctr encryptor(
@ -304,7 +304,8 @@ struct EncryptionOpsWorkload : TestWorkload {
int len,
const EncryptAuthTokenMode authMode,
const EncryptAuthTokenAlgo authAlgo,
BlobCipherEncryptHeaderRef* headerRef) {
BlobCipherEncryptHeaderRef* headerRef,
Arena& arena) {
uint8_t iv[AES_256_IV_LENGTH];
deterministicRandom()->randomBytes(&iv[0], AES_256_IV_LENGTH);
EncryptBlobCipherAes265Ctr encryptor(
@ -345,7 +346,8 @@ struct EncryptionOpsWorkload : TestWorkload {
int len,
const BlobCipherEncryptHeader& header,
uint8_t* originalPayload,
Reference<BlobCipherKey> orgCipherKey) {
Reference<BlobCipherKey> orgCipherKey,
Arena& arena) {
ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION);
ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR);
@ -379,7 +381,8 @@ struct EncryptionOpsWorkload : TestWorkload {
int len,
const Standalone<StringRef>& headerStr,
uint8_t* originalPayload,
Reference<BlobCipherKey> orgCipherKey) {
Reference<BlobCipherKey> orgCipherKey,
Arena& arena) {
BlobCipherEncryptHeaderRef headerRef = BlobCipherEncryptHeaderRef::fromStringRef(headerStr);
ASSERT_EQ(headerRef.flagsVersion(), CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION);
@ -436,6 +439,7 @@ struct EncryptionOpsWorkload : TestWorkload {
setupCipherEssentials();
for (int i = 0; i < numIterations; i++) {
Arena tmpArena;
bool updateBaseCipher = deterministicRandom()->randomInt(1, 100) < 5;
// Step-1: Encryption key derivation, caching the cipher for later use
@ -482,23 +486,23 @@ struct EncryptionOpsWorkload : TestWorkload {
try {
BlobCipherEncryptHeader header;
Reference<EncryptBuf> encrypted =
doEncryption(cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &header);
Reference<EncryptBuf> encrypted = doEncryption(
cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &header, tmpArena);
// Decrypt the payload - parses the BlobCipherEncryptHeader, fetch corresponding cipherKey and
// decrypt
doDecryption(encrypted, dataLen, header, buff.get(), cipherKey);
doDecryption(encrypted, dataLen, header, buff.get(), cipherKey, tmpArena);
if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) {
BlobCipherEncryptHeaderRef headerRef;
StringRef encrypted =
doEncryption(cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &headerRef);
StringRef encrypted = doEncryption(
cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &headerRef, tmpArena);
// simulate 'header' on-disk read, serialize buffer and deserialize on decryption
Standalone<StringRef> serHeader = BlobCipherEncryptHeaderRef::toStringRef(headerRef);
// Decrypt the payload - parses the BlobCipherEncryptHeader, fetch corresponding cipherKey and
// decrypt
doDecryption(encrypted, dataLen, serHeader, buff.get(), cipherKey);
doDecryption(encrypted, dataLen, serHeader, buff.get(), cipherKey, tmpArena);
}
} catch (Error& e) {
TraceEvent("Failed")

8
fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp
View File

@ -299,22 +299,18 @@ struct MetaclusterManagementWorkload : TestWorkload {
} catch (Error& e) {
if (e.code() == error_code_conflicting_restore) {
ASSERT(retried);
CODE_PROBE(true, "MetaclusterRestore: 2 restores on the same cluster simultaneously");
CODE_PROBE(true, "MetaclusterManagementWorkload: timed out restore conflicts with retried restore");
continue;
}
throw;
}
}
ASSERT(dataDb->registered);
if (!dryRun) {
dataDb->detached = false;
}
} catch (Error& e) {
if (e.code() == error_code_cluster_not_found) {
ASSERT(!dataDb->registered);
return Void();
}
TraceEvent(SevError, "RestoreClusterFailure").error(e).detail("ClusterName", clusterName);
ASSERT(false);
}

7
fdbserver/workloads/SaveAndKill.actor.cpp
View File

@ -79,13 +79,8 @@ struct SaveAndKillWorkload : TestWorkload {
if (cx->defaultTenant.present()) {
ini.SetValue("META", "defaultTenant", cx->defaultTenant.get().toString().c_str());
}
ini.SetBoolValue("META", "enableEncryption", SERVER_KNOBS->ENABLE_ENCRYPTION);
ini.SetBoolValue("META", "enableTLogEncryption", SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION);
ini.SetBoolValue("META", "enableStorageServerEncryption", SERVER_KNOBS->ENABLE_STORAGE_SERVER_ENCRYPTION);
ini.SetBoolValue("META", "enableBlobGranuleEncryption", SERVER_KNOBS->ENABLE_BLOB_GRANULE_ENCRYPTION);
ini.SetBoolValue("META", "enableShardEncodeLocationMetadata", SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA);
ini.SetBoolValue("META", "enableConfigurableEncryption", CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION);
ini.SetBoolValue("META", "encryptHeaderAuthTokenEnabled", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED);
ini.SetLongValue("META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO);

6
flow/Arena.cpp
View File

@ -22,6 +22,8 @@
#include "flow/UnitTest.h"
#include "flow/config.h"
// We don't align memory properly, and we need to tell lsan about that.
extern "C" const char* __lsan_default_options(void) {
return "use_unaligned=1";
@ -998,7 +1000,7 @@ TEST_CASE("/flow/Arena/OptionalMap") {
}
TEST_CASE("/flow/Arena/Secure") {
#ifndef ADDRESS_SANITIZER
#ifndef USE_SANITIZER
// Note: Assumptions underlying this unit test are speculative.
// Disable for a build configuration or entirely if deemed flaky.
// As of writing, below equivalency of (buf == newBuf) holds except for ASAN builds.
@ -1051,6 +1053,6 @@ TEST_CASE("/flow/Arena/Secure") {
}
}
fmt::print("Total iterations: {}, # of times check passed: {}\n", totalIters, samePtrCount);
#endif // ADDRESS_SANITIZER
#endif // USE_SANITIZER
return Void();
}

6
flow/Platform.actor.cpp
View File

@ -651,7 +651,11 @@ void getDiskBytes(std::string const& directory, int64_t& free, int64_t& total) {
#endif
free = std::min((uint64_t)std::numeric_limits<int64_t>::max(), buf.f_bavail * blockSize);
total = std::min((uint64_t)std::numeric_limits<int64_t>::max(), buf.f_blocks * blockSize);
// f_blocks is the total fs space but (f_bfree - f_bavail) is space only available to privileged users
// so that amount will be subtracted from the reported total since FDB can't use it.
total = std::min((uint64_t)std::numeric_limits<int64_t>::max(),
(buf.f_blocks - (buf.f_bfree - buf.f_bavail)) * blockSize);
#elif defined(_WIN32)
std::string fullPath = abspath(directory);

36
flow/include/flow/PriorityMultiLock.actor.h
View File

@ -88,7 +88,7 @@ public:
: PriorityMultiLock(concurrency, parseStringToVector<int>(weights, ',')) {}
PriorityMultiLock(int concurrency, std::vector<int> weightsByPriority)
: concurrency(concurrency), available(concurrency), waiting(0), totalPendingWeights(0) {
: concurrency(concurrency), available(concurrency), waiting(0), totalPendingWeights(0), killed(false) {
priorities.resize(weightsByPriority.size());
for (int i = 0; i < priorities.size(); ++i) {
@ -102,6 +102,9 @@ public:
~PriorityMultiLock() { kill(); }
Future<Lock> lock(int priority = 0) {
if (killed)
throw broken_promise();
Priority& p = priorities[priority];
Queue& q = p.queue;
@ -135,17 +138,33 @@ public:
return w.lockPromise.getFuture();
}
void kill() {
pml_debug_printf("kill %s\n", toString().c_str());
// Halt stops the PML from handing out any new locks but leaves waiters and runners alone.
// Existing and new waiters will not see an error, they will just never get a lock.
// Can be safely called multiple times.
void halt() {
pml_debug_printf("halt %s\n", toString().c_str());
brokenOnDestruct.reset();
// handleRelease will not free up any execution slots when it ends via cancel
fRunner.cancel();
available = 0;
if (fRunner.isValid()) {
fRunner.cancel();
// Adjust available and concurrency so that if all runners finish the available
available -= concurrency;
concurrency = 0;
}
waitingPriorities.clear();
for (auto& p : priorities) {
p.queue.clear();
}
// Halt, then make existing and new waiters get a broken_promise error.
// Can be safely called multiple times.
void kill() {
if (!killed) {
// Killed must be set first because waiters which ignore exceptions could call wait again immediately.
killed = true;
halt();
for (auto& p : priorities) {
p.queue.clear();
}
}
}
@ -231,6 +250,7 @@ private:
Future<Void> fRunner;
AsyncTrigger wakeRunner;
Promise<Void> brokenOnDestruct;
bool killed;
ACTOR static void handleRelease(Reference<PriorityMultiLock> self, Priority* priority, Future<Void> holder) {
pml_debug_printf("%f handleRelease self=%p start\n", now(), self.getPtr());

2
tests/BlobGranuleFileUnit.toml
View File

@ -2,8 +2,6 @@
encryptModes = ['domain_aware', 'cluster_aware']
[[knobs]]
enable_encryption = true
enable_blob_file_encryption = true
enable_blob_file_compression = true
[[test]]

9
tests/CMakeLists.txt
View File

@ -333,15 +333,15 @@ if(WITH_PYTHON)
add_fdb_test(
TEST_FILES restarting/from_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-1.toml
restarting/from_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-2.toml)
add_fdb_test(
TEST_FILES restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-1.toml
restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-2.toml)
add_fdb_test(
TEST_FILES restarting/from_7.2.0_until_7.3.0/VersionVectorDisableRestart-1.toml
restarting/from_7.2.0_until_7.3.0/VersionVectorDisableRestart-2.toml)
add_fdb_test(
TEST_FILES restarting/from_7.2.0_until_7.3.0/VersionVectorEnableRestart-1.toml
restarting/from_7.2.0_until_7.3.0/VersionVectorEnableRestart-2.toml)
add_fdb_test(
TEST_FILES restarting/from_7.2.0/DrUpgradeRestart-1.toml
restarting/from_7.2.0/DrUpgradeRestart-2.toml)
add_fdb_test(
TEST_FILES restarting/from_7.2.4_until_7.3.0/UpgradeAndBackupRestore-1.toml
restarting/from_7.2.4_until_7.3.0/UpgradeAndBackupRestore-2.toml)
@ -351,6 +351,9 @@ if(WITH_PYTHON)
add_fdb_test(
TEST_FILES restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-1.toml
restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-2.toml)
add_fdb_test(
TEST_FILES restarting/from_7.3.0/DrUpgradeRestart-1.toml
restarting/from_7.3.0/DrUpgradeRestart-2.toml)
add_fdb_test(
TEST_FILES restarting/from_7.3.0/UpgradeAndBackupRestore-1.toml
restarting/from_7.3.0/UpgradeAndBackupRestore-2.toml)

5
tests/TestRunner/local_cluster.py
View File

@ -114,6 +114,7 @@ logdir = {logdir}
{bg_knob_line}
{encrypt_knob_line1}
{encrypt_knob_line2}
{encrypt_knob_line3}
{tls_config}
{authz_public_key_config}
{custom_config}
@ -256,13 +257,14 @@ logdir = {logdir}
bg_knob_line = ""
encrypt_knob_line1 = ""
encrypt_knob_line2 = ""
encrypt_knob_line3 = ""
if self.use_legacy_conf_syntax:
conf_template = conf_template.replace("-", "_")
if self.blob_granules_enabled:
bg_knob_line = "knob_bg_url=file://" + str(self.data) + "/fdbblob/"
if self.enable_encryption_at_rest:
encrypt_knob_line1 = "knob_enable_encryption=true"
encrypt_knob_line2 = "knob_kms_connector_type=FDBPerfKmsConnector"
encrypt_knob_line3 = "knob_enable_configurable_encryption=true"
f.write(
conf_template.format(
etcdir=self.etc,
@ -273,6 +275,7 @@ logdir = {logdir}
bg_knob_line=bg_knob_line,
encrypt_knob_line1=encrypt_knob_line1,
encrypt_knob_line2=encrypt_knob_line2,
encrypt_knob_line3=encrypt_knob_line3,
tls_config=self.tls_conf_string(),
authz_public_key_config=self.authz_public_key_conf_string(),
optional_tls=":tls" if self.tls_config is not None else "",

3
tests/fast/BackupCorrectnessWithEKPKeyFetchFailures.toml
View File

@ -4,9 +4,6 @@ tenantModes = ['required']
allowCreatingTenants = false
encryptModes = ['domain_aware']
[[knobs]]
enable_encryption = true
[[test]]
testTitle = 'BackupAndRestoreWithEKPKeyFetchFailures'
clearAfterTest = false

3
tests/fast/BackupCorrectnessWithTenantDeletion.toml
View File

@ -3,6 +3,9 @@ allowDefaultTenant = false
tenantModes = ['required']
allowCreatingTenants = false
[[knobs]]
simulation_enable_snapshot_encryption_checks = false
[[test]]
testTitle = 'BackupAndRestoreWithTenantDeletion'
clearAfterTest = false

3
tests/fast/EncryptKeyProxyTest.toml
View File

@ -2,9 +2,6 @@
testClass = "Encryption"
encryptModes = ['domain_aware', 'cluster_aware']
[[knobs]]
enable_encryption = true
[[test]]
testTitle = 'EncryptKeyProxy'

4
tests/fast/EncryptedBackupCorrectness.toml
View File

@ -5,10 +5,6 @@ tenantModes = ['required']
encryptModes = ['domain_aware']
[[knobs]]
enable_encryption = true
enable_tlog_encryption = true
enable_storage_server_encryption = false
enable_blob_granule_encryption = true
max_write_transaction_life_versions = 5000000
[[test]]

2
tests/fast/EncryptionOps.toml
View File

@ -1,7 +1,7 @@
[configuration]
buggify = false
testClass = "Encryption"
disableEncryption = true
encryptModes = ['disabled']
[[knobs]]
enable_configurable_encryption = true

3
tests/fast/IncrementalBackupWithEKPKeyFetchFailures.toml
View File

@ -4,9 +4,6 @@ tenantModes = ['required']
allowCreatingTenants = false
encryptModes = ['domain_aware']
[[knobs]]
enable_encryption = true
[[test]]
testTitle = 'SubmitBackup'
simBackupAgents = 'BackupToFile'

3
tests/fast/IncrementalBackupWithTenantDeletion.toml
View File

@ -3,6 +3,9 @@ allowDefaultTenant = false
tenantModes = ['required']
allowCreatingTenants = false
[[knobs]]
simulation_enable_snapshot_encryption_checks = false
[[test]]
testTitle = 'SubmitBackup'
simBackupAgents = 'BackupToFile'

0
tests/restarting/from_7.2.0/DrUpgradeRestart-1.toml → tests/restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-1.toml
View File

0
tests/restarting/from_7.2.0/DrUpgradeRestart-2.toml → tests/restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-2.toml
View File

28
tests/restarting/from_7.3.0/DrUpgradeRestart-1.toml Normal file
View File

@ -0,0 +1,28 @@
[configuration]
extraDatabaseMode = "Local"
# In 7.2, DR is not supported in required tenant mode
allowDefaultTenant = false
encryptModes = ['disabled']
[[test]]
testTitle = "DrUpgrade"
clearAfterTest = false
simBackupAgents = "BackupToDB"
[[test.workload]]
testName = "Cycle"
nodeCount = 30000
transactionsPerSecond = 2500.0
testDuration = 30.0
expectedRate = 0
[[test.workload]]
testName = "BackupToDBUpgrade"
backupAfter = 10.0
stopDifferentialAfter = 50.0
backupRangesCount = -1
[[test.workload]]
testName = "SaveAndKill"
restartInfoLocation = "simfdb/restartInfo.ini"
testDuration = 40.0

Some files were not shown because too many files have changed in this diff Show More