diff --git a/CMakeLists.txt b/CMakeLists.txt index 3652be013c..ecb5df6a43 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,7 +97,10 @@ else() set(FDB_VERSION ${PROJECT_VERSION}) endif() if (NOT FDB_RELEASE) - string(TIMESTAMP FDB_BUILDTIME %Y%m%d%H%M%S) + string(TIMESTAMP FDB_BUILD_TIMESTMAP %Y%m%d%H%M%S) + # Adding support to pass custom fdb_build timestamp, + # required to achieve uniform naming across different builds + set(FDB_BUILDTIME "${FDB_BUILD_TIMESTMAP}" CACHE STRING "A timestamp for packages") set(FDB_BUILDTIME_STRING ".${FDB_BUILDTIME}") set(PRERELEASE_TAG "prerelease") endif() diff --git a/bindings/bindingtester/tests/api.py b/bindings/bindingtester/tests/api.py index 8cdf0a971b..bda718496c 100644 --- a/bindings/bindingtester/tests/api.py +++ b/bindings/bindingtester/tests/api.py @@ -33,23 +33,32 @@ fdb.api_version(FDB_API_VERSION) def matches_op(op, target_op): - return op == target_op or op == target_op + '_SNAPSHOT' or op == target_op + '_DATABASE' or op == target_op + '_TENANT' + return ( + op == target_op + or op == target_op + "_SNAPSHOT" + or op == target_op + "_DATABASE" + or op == target_op + "_TENANT" + ) def is_non_transaction_op(op): - return op.endswith('_DATABASE') or op.endswith('_TENANT') + return op.endswith("_DATABASE") or op.endswith("_TENANT") class ApiTest(Test): def __init__(self, subspace): super(ApiTest, self).__init__(subspace) - self.workspace = self.subspace['workspace'] # The keys and values here must match between subsequent runs of the same test - self.scratch = self.subspace['scratch'] # The keys and values here can differ between runs - self.stack_subspace = self.subspace['stack'] + self.workspace = self.subspace[ + "workspace" + ] # The keys and values here must match between subsequent runs of the same test + self.scratch = self.subspace[ + "scratch" + ] # The keys and values here can differ between runs + self.stack_subspace = self.subspace["stack"] - self.versionstamped_values = self.scratch['versionstamped_values'] - self.versionstamped_values_2 = self.scratch['versionstamped_values_2'] - self.versionstamped_keys = self.scratch['versionstamped_keys'] + self.versionstamped_values = self.scratch["versionstamped_values"] + self.versionstamped_values_2 = self.scratch["versionstamped_values_2"] + self.versionstamped_keys = self.scratch["versionstamped_keys"] def setup(self, args): self.stack_size = 0 @@ -64,7 +73,9 @@ class ApiTest(Test): self.generated_keys = [] self.outstanding_ops = [] - self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version, args.types) + self.random = test_util.RandomGenerator( + args.max_int_bits, args.api_version, args.types + ) self.api_version = args.api_version self.allocated_tenants = set() @@ -88,7 +99,9 @@ class ApiTest(Test): self.string_depth = max(0, self.string_depth - num) self.key_depth = max(0, self.key_depth - num) - self.outstanding_ops = [i for i in self.outstanding_ops if i[0] <= self.stack_size] + self.outstanding_ops = [ + i for i in self.outstanding_ops if i[0] <= self.stack_size + ] def ensure_string(self, instructions, num): while self.string_depth < num: @@ -101,7 +114,7 @@ class ApiTest(Test): if random.random() < float(len(self.generated_keys)) / self.max_keys: tup = random.choice(self.generated_keys) if random.random() < 0.3: - return self.workspace.pack(tup[0:random.randint(0, len(tup))]) + return self.workspace.pack(tup[0 : random.randint(0, len(tup))]) return self.workspace.pack(tup) @@ -119,7 +132,9 @@ class ApiTest(Test): def ensure_key_value(self, instructions): if self.string_depth == 0: - instructions.push_args(self.choose_key(), self.random.random_string(random.randint(0, 100))) + instructions.push_args( + self.choose_key(), self.random.random_string(random.randint(0, 100)) + ) elif self.string_depth == 1 or self.key_depth == 0: self.ensure_key(instructions, 1) @@ -131,7 +146,7 @@ class ApiTest(Test): def preload_database(self, instructions, num): for i in range(num): self.ensure_key_value(instructions) - instructions.append('SET') + instructions.append("SET") if i % 100 == 99: test_util.blocking_commit(instructions) @@ -140,42 +155,81 @@ class ApiTest(Test): self.add_stack_items(1) def wait_for_reads(self, instructions): - while len(self.outstanding_ops) > 0 and self.outstanding_ops[-1][0] <= self.stack_size: + while ( + len(self.outstanding_ops) > 0 + and self.outstanding_ops[-1][0] <= self.stack_size + ): read = self.outstanding_ops.pop() # print '%d. waiting for read at instruction %r' % (len(instructions), read) test_util.to_front(instructions, self.stack_size - read[0]) - instructions.append('WAIT_FUTURE') + instructions.append("WAIT_FUTURE") def choose_tenant(self, new_tenant_probability): if len(self.allocated_tenants) == 0 or random.random() < new_tenant_probability: return self.random.random_string(random.randint(0, 30)) else: - return random.choice(list(self.allocated_tenants)) + tenant_list = list(self.allocated_tenants) + # sort to ensure deterministic selection of a tenant + tenant_list.sort() + return random.choice(tenant_list) def generate(self, args, thread_number): instructions = InstructionSet() - op_choices = ['NEW_TRANSACTION', 'COMMIT'] + op_choices = ["NEW_TRANSACTION", "COMMIT"] - reads = ['GET', 'GET_KEY', 'GET_RANGE', 'GET_RANGE_STARTS_WITH', 'GET_RANGE_SELECTOR'] - mutations = ['SET', 'CLEAR', 'CLEAR_RANGE', 'CLEAR_RANGE_STARTS_WITH', 'ATOMIC_OP'] - snapshot_reads = [x + '_SNAPSHOT' for x in reads] - database_reads = [x + '_DATABASE' for x in reads] - database_mutations = [x + '_DATABASE' for x in mutations] - tenant_reads = [x + '_TENANT' for x in reads] - tenant_mutations = [x + '_TENANT' for x in mutations] - mutations += ['VERSIONSTAMP'] - versions = ['GET_READ_VERSION', 'SET_READ_VERSION', 'GET_COMMITTED_VERSION'] - snapshot_versions = ['GET_READ_VERSION_SNAPSHOT'] - tuples = ['TUPLE_PACK', 'TUPLE_UNPACK', 'TUPLE_RANGE', 'TUPLE_SORT', 'SUB', 'ENCODE_FLOAT', 'ENCODE_DOUBLE', 'DECODE_DOUBLE', 'DECODE_FLOAT'] - if 'versionstamp' in args.types: - tuples.append('TUPLE_PACK_WITH_VERSIONSTAMP') - resets = ['ON_ERROR', 'RESET', 'CANCEL'] - read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY'] - write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT'] - txn_sizes = ['GET_APPROXIMATE_SIZE'] - storage_metrics = ['GET_ESTIMATED_RANGE_SIZE', 'GET_RANGE_SPLIT_POINTS'] - tenants = ['TENANT_CREATE', 'TENANT_DELETE', 'TENANT_SET_ACTIVE', 'TENANT_CLEAR_ACTIVE', 'TENANT_LIST', 'TENANT_GET_ID'] + reads = [ + "GET", + "GET_KEY", + "GET_RANGE", + "GET_RANGE_STARTS_WITH", + "GET_RANGE_SELECTOR", + ] + mutations = [ + "SET", + "CLEAR", + "CLEAR_RANGE", + "CLEAR_RANGE_STARTS_WITH", + "ATOMIC_OP", + ] + snapshot_reads = [x + "_SNAPSHOT" for x in reads] + database_reads = [x + "_DATABASE" for x in reads] + database_mutations = [x + "_DATABASE" for x in mutations] + tenant_reads = [x + "_TENANT" for x in reads] + tenant_mutations = [x + "_TENANT" for x in mutations] + mutations += ["VERSIONSTAMP"] + versions = ["GET_READ_VERSION", "SET_READ_VERSION", "GET_COMMITTED_VERSION"] + snapshot_versions = ["GET_READ_VERSION_SNAPSHOT"] + tuples = [ + "TUPLE_PACK", + "TUPLE_UNPACK", + "TUPLE_RANGE", + "TUPLE_SORT", + "SUB", + "ENCODE_FLOAT", + "ENCODE_DOUBLE", + "DECODE_DOUBLE", + "DECODE_FLOAT", + ] + if "versionstamp" in args.types: + tuples.append("TUPLE_PACK_WITH_VERSIONSTAMP") + resets = ["ON_ERROR", "RESET", "CANCEL"] + read_conflicts = ["READ_CONFLICT_RANGE", "READ_CONFLICT_KEY"] + write_conflicts = [ + "WRITE_CONFLICT_RANGE", + "WRITE_CONFLICT_KEY", + "DISABLE_WRITE_CONFLICT", + ] + txn_sizes = ["GET_APPROXIMATE_SIZE"] + storage_metrics = ["GET_ESTIMATED_RANGE_SIZE", "GET_RANGE_SPLIT_POINTS"] + tenants = [ + "TENANT_CREATE", + "TENANT_DELETE", + "TENANT_SET_ACTIVE", + "TENANT_CLEAR_ACTIVE", + "TENANT_LIST", + "TENANT_GET_ID", + ] op_choices += reads op_choices += mutations @@ -196,16 +250,23 @@ class ApiTest(Test): op_choices += tenant_reads op_choices += tenant_mutations - idempotent_atomic_ops = ['BIT_AND', 'BIT_OR', 'MAX', 'MIN', 'BYTE_MIN', 'BYTE_MAX'] - atomic_ops = idempotent_atomic_ops + ['ADD', 'BIT_XOR', 'APPEND_IF_FITS'] + idempotent_atomic_ops = [ + "BIT_AND", + "BIT_OR", + "MAX", + "MIN", + "BYTE_MIN", + "BYTE_MAX", + ] + atomic_ops = idempotent_atomic_ops + ["ADD", "BIT_XOR", "APPEND_IF_FITS"] if args.concurrency > 1: self.max_keys = random.randint(100, 1000) else: self.max_keys = random.randint(100, 10000) - instructions.append('NEW_TRANSACTION') - instructions.append('GET_READ_VERSION') + instructions.append("NEW_TRANSACTION") + instructions.append("GET_READ_VERSION") self.preload_database(instructions, self.max_keys) @@ -218,25 +279,29 @@ class ApiTest(Test): # print 'Adding instruction %s at %d' % (op, index) - if args.concurrency == 1 and (op in database_mutations or op in tenant_mutations or op in ['TENANT_CREATE', 'TENANT_DELETE']): + if args.concurrency == 1 and ( + op in database_mutations + or op in tenant_mutations + or op in ["TENANT_CREATE", "TENANT_DELETE"] + ): self.wait_for_reads(instructions) test_util.blocking_commit(instructions) self.can_get_commit_version = False self.add_stack_items(1) - if op in resets or op == 'NEW_TRANSACTION': + if op in resets or op == "NEW_TRANSACTION": if args.concurrency == 1: self.wait_for_reads(instructions) self.outstanding_ops = [] - if op == 'NEW_TRANSACTION': + if op == "NEW_TRANSACTION": instructions.append(op) self.can_get_commit_version = True self.can_set_version = True self.can_use_key_selectors = True - elif op == 'ON_ERROR': + elif op == "ON_ERROR": instructions.push_args(random.randint(0, 5000)) instructions.append(op) @@ -244,20 +309,20 @@ class ApiTest(Test): if args.concurrency == 1: self.wait_for_reads(instructions) - instructions.append('NEW_TRANSACTION') + instructions.append("NEW_TRANSACTION") self.can_get_commit_version = True self.can_set_version = True self.can_use_key_selectors = True self.add_strings(1) - elif matches_op(op, 'GET'): + elif matches_op(op, "GET"): self.ensure_key(instructions, 1) instructions.append(op) self.add_strings(1) self.can_set_version = False read_performed = True - elif matches_op(op, 'GET_KEY'): + elif matches_op(op, "GET_KEY"): if is_non_transaction_op(op) or self.can_use_key_selectors: self.ensure_key(instructions, 1) instructions.push_args(self.workspace.key()) @@ -270,7 +335,7 @@ class ApiTest(Test): self.can_set_version = False read_performed = True - elif matches_op(op, 'GET_RANGE'): + elif matches_op(op, "GET_RANGE"): self.ensure_key(instructions, 2) range_params = self.random.random_range_params() instructions.push_args(*range_params) @@ -278,7 +343,9 @@ class ApiTest(Test): test_util.to_front(instructions, 4) instructions.append(op) - if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large + if ( + range_params[0] >= 1 and range_params[0] <= 1000 + ): # avoid adding a string if the limit is large self.add_strings(1) else: self.add_stack_items(1) @@ -286,7 +353,7 @@ class ApiTest(Test): self.can_set_version = False read_performed = True - elif matches_op(op, 'GET_RANGE_STARTS_WITH'): + elif matches_op(op, "GET_RANGE_STARTS_WITH"): # TODO: not tested well self.ensure_key(instructions, 1) range_params = self.random.random_range_params() @@ -294,7 +361,9 @@ class ApiTest(Test): test_util.to_front(instructions, 3) instructions.append(op) - if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large + if ( + range_params[0] >= 1 and range_params[0] <= 1000 + ): # avoid adding a string if the limit is large self.add_strings(1) else: self.add_stack_items(1) @@ -302,7 +371,7 @@ class ApiTest(Test): self.can_set_version = False read_performed = True - elif matches_op(op, 'GET_RANGE_SELECTOR'): + elif matches_op(op, "GET_RANGE_SELECTOR"): if is_non_transaction_op(op) or self.can_use_key_selectors: self.ensure_key(instructions, 2) instructions.push_args(self.workspace.key()) @@ -314,7 +383,9 @@ class ApiTest(Test): test_util.to_front(instructions, 9) instructions.append(op) - if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large + if ( + range_params[0] >= 1 and range_params[0] <= 1000 + ): # avoid adding a string if the limit is large self.add_strings(1) else: self.add_stack_items(1) @@ -322,29 +393,29 @@ class ApiTest(Test): self.can_set_version = False read_performed = True - elif matches_op(op, 'GET_READ_VERSION'): + elif matches_op(op, "GET_READ_VERSION"): instructions.append(op) self.has_version = self.can_set_version self.add_strings(1) - elif matches_op(op, 'SET'): + elif matches_op(op, "SET"): self.ensure_key_value(instructions) instructions.append(op) if is_non_transaction_op(op): self.add_stack_items(1) - elif op == 'SET_READ_VERSION': + elif op == "SET_READ_VERSION": if self.has_version and self.can_set_version: instructions.append(op) self.can_set_version = False - elif matches_op(op, 'CLEAR'): + elif matches_op(op, "CLEAR"): self.ensure_key(instructions, 1) instructions.append(op) if is_non_transaction_op(op): self.add_stack_items(1) - elif matches_op(op, 'CLEAR_RANGE'): + elif matches_op(op, "CLEAR_RANGE"): # Protect against inverted range key1 = self.workspace.pack(self.random.random_tuple(5)) key2 = self.workspace.pack(self.random.random_tuple(5)) @@ -358,13 +429,13 @@ class ApiTest(Test): if is_non_transaction_op(op): self.add_stack_items(1) - elif matches_op(op, 'CLEAR_RANGE_STARTS_WITH'): + elif matches_op(op, "CLEAR_RANGE_STARTS_WITH"): self.ensure_key(instructions, 1) instructions.append(op) if is_non_transaction_op(op): self.add_stack_items(1) - elif matches_op(op, 'ATOMIC_OP'): + elif matches_op(op, "ATOMIC_OP"): self.ensure_key_value(instructions) if is_non_transaction_op(op) and args.concurrency == 1: instructions.push_args(random.choice(idempotent_atomic_ops)) @@ -375,50 +446,58 @@ class ApiTest(Test): if is_non_transaction_op(op): self.add_stack_items(1) - elif op == 'VERSIONSTAMP': + elif op == "VERSIONSTAMP": rand_str1 = self.random.random_string(100) key1 = self.versionstamped_values.pack((rand_str1,)) key2 = self.versionstamped_values_2.pack((rand_str1,)) split = random.randint(0, 70) prefix = self.random.random_string(20 + split) - if prefix.endswith(b'\xff'): + if prefix.endswith(b"\xff"): # Necessary to make sure that the SET_VERSIONSTAMPED_VALUE check # correctly finds where the version is supposed to fit in. - prefix += b'\x00' + prefix += b"\x00" suffix = self.random.random_string(70 - split) rand_str2 = prefix + fdb.tuple.Versionstamp._UNSET_TR_VERSION + suffix key3 = self.versionstamped_keys.pack() + rand_str2 index = len(self.versionstamped_keys.pack()) + len(prefix) key3 = self.versionstamp_key(key3, index) - instructions.push_args('SET_VERSIONSTAMPED_VALUE', - key1, - self.versionstamp_value(fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2)) - instructions.append('ATOMIC_OP') + instructions.push_args( + "SET_VERSIONSTAMPED_VALUE", + key1, + self.versionstamp_value( + fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2 + ), + ) + instructions.append("ATOMIC_OP") if args.api_version >= 520: - instructions.push_args('SET_VERSIONSTAMPED_VALUE', key2, self.versionstamp_value(rand_str2, len(prefix))) - instructions.append('ATOMIC_OP') + instructions.push_args( + "SET_VERSIONSTAMPED_VALUE", + key2, + self.versionstamp_value(rand_str2, len(prefix)), + ) + instructions.append("ATOMIC_OP") - instructions.push_args('SET_VERSIONSTAMPED_KEY', key3, rand_str1) - instructions.append('ATOMIC_OP') + instructions.push_args("SET_VERSIONSTAMPED_KEY", key3, rand_str1) + instructions.append("ATOMIC_OP") self.can_use_key_selectors = False - elif op == 'READ_CONFLICT_RANGE' or op == 'WRITE_CONFLICT_RANGE': + elif op == "READ_CONFLICT_RANGE" or op == "WRITE_CONFLICT_RANGE": self.ensure_key(instructions, 2) instructions.append(op) self.add_strings(1) - elif op == 'READ_CONFLICT_KEY' or op == 'WRITE_CONFLICT_KEY': + elif op == "READ_CONFLICT_KEY" or op == "WRITE_CONFLICT_KEY": self.ensure_key(instructions, 1) instructions.append(op) self.add_strings(1) - elif op == 'DISABLE_WRITE_CONFLICT': + elif op == "DISABLE_WRITE_CONFLICT": instructions.append(op) - elif op == 'COMMIT': + elif op == "COMMIT": if args.concurrency == 1 or i < self.max_keys or random.random() < 0.9: if args.concurrency == 1: self.wait_for_reads(instructions) @@ -431,23 +510,23 @@ class ApiTest(Test): instructions.append(op) self.add_strings(1) - elif op == 'RESET': + elif op == "RESET": instructions.append(op) self.can_get_commit_version = False self.can_set_version = True self.can_use_key_selectors = True - elif op == 'CANCEL': + elif op == "CANCEL": instructions.append(op) self.can_set_version = False - elif op == 'GET_COMMITTED_VERSION': + elif op == "GET_COMMITTED_VERSION": if self.can_get_commit_version: do_commit = random.random() < 0.5 if do_commit: - instructions.append('COMMIT') - instructions.append('WAIT_FUTURE') + instructions.append("COMMIT") + instructions.append("WAIT_FUTURE") self.add_stack_items(1) instructions.append(op) @@ -456,35 +535,47 @@ class ApiTest(Test): self.add_strings(1) if do_commit: - instructions.append('RESET') + instructions.append("RESET") self.can_get_commit_version = False self.can_set_version = True self.can_use_key_selectors = True - elif op == 'GET_APPROXIMATE_SIZE': + elif op == "GET_APPROXIMATE_SIZE": instructions.append(op) self.add_strings(1) - elif op == 'TUPLE_PACK' or op == 'TUPLE_RANGE': + elif op == "TUPLE_PACK" or op == "TUPLE_RANGE": tup = self.random.random_tuple(10) instructions.push_args(len(tup), *tup) instructions.append(op) - if op == 'TUPLE_PACK': + if op == "TUPLE_PACK": self.add_strings(1) else: self.add_strings(2) - elif op == 'TUPLE_PACK_WITH_VERSIONSTAMP': - tup = (self.random.random_string(20),) + self.random.random_tuple(10, incomplete_versionstamps=True) + elif op == "TUPLE_PACK_WITH_VERSIONSTAMP": + tup = (self.random.random_string(20),) + self.random.random_tuple( + 10, incomplete_versionstamps=True + ) prefix = self.versionstamped_keys.pack() instructions.push_args(prefix, len(tup), *tup) instructions.append(op) self.add_strings(1) versionstamp_param = prefix + fdb.tuple.pack(tup) - first_incomplete = versionstamp_param.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION) - second_incomplete = -1 if first_incomplete < 0 else \ - versionstamp_param.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION, first_incomplete + len(fdb.tuple.Versionstamp._UNSET_TR_VERSION) + 1) + first_incomplete = versionstamp_param.find( + fdb.tuple.Versionstamp._UNSET_TR_VERSION + ) + second_incomplete = ( + -1 + if first_incomplete < 0 + else versionstamp_param.find( + fdb.tuple.Versionstamp._UNSET_TR_VERSION, + first_incomplete + + len(fdb.tuple.Versionstamp._UNSET_TR_VERSION) + + 1, + ) + ) # If there is exactly one incomplete versionstamp, perform the versionstamp operation. if first_incomplete >= 0 and second_incomplete < 0: @@ -492,76 +583,90 @@ class ApiTest(Test): instructions.push_args(rand_str) test_util.to_front(instructions, 1) - instructions.push_args('SET_VERSIONSTAMPED_KEY') - instructions.append('ATOMIC_OP') + instructions.push_args("SET_VERSIONSTAMPED_KEY") + instructions.append("ATOMIC_OP") if self.api_version >= 520: - version_value_key_2 = self.versionstamped_values_2.pack((rand_str,)) - versionstamped_value = self.versionstamp_value(fdb.tuple.pack(tup), first_incomplete - len(prefix)) - instructions.push_args('SET_VERSIONSTAMPED_VALUE', version_value_key_2, versionstamped_value) - instructions.append('ATOMIC_OP') + version_value_key_2 = self.versionstamped_values_2.pack( + (rand_str,) + ) + versionstamped_value = self.versionstamp_value( + fdb.tuple.pack(tup), first_incomplete - len(prefix) + ) + instructions.push_args( + "SET_VERSIONSTAMPED_VALUE", + version_value_key_2, + versionstamped_value, + ) + instructions.append("ATOMIC_OP") version_value_key = self.versionstamped_values.pack((rand_str,)) - instructions.push_args('SET_VERSIONSTAMPED_VALUE', version_value_key, - self.versionstamp_value(fdb.tuple.Versionstamp._UNSET_TR_VERSION + fdb.tuple.pack(tup))) - instructions.append('ATOMIC_OP') + instructions.push_args( + "SET_VERSIONSTAMPED_VALUE", + version_value_key, + self.versionstamp_value( + fdb.tuple.Versionstamp._UNSET_TR_VERSION + + fdb.tuple.pack(tup) + ), + ) + instructions.append("ATOMIC_OP") self.can_use_key_selectors = False - elif op == 'TUPLE_UNPACK': + elif op == "TUPLE_UNPACK": tup = self.random.random_tuple(10) instructions.push_args(len(tup), *tup) - instructions.append('TUPLE_PACK') + instructions.append("TUPLE_PACK") instructions.append(op) self.add_strings(len(tup)) - elif op == 'TUPLE_SORT': + elif op == "TUPLE_SORT": tups = self.random.random_tuple_list(10, 30) for tup in tups: instructions.push_args(len(tup), *tup) - instructions.append('TUPLE_PACK') + instructions.append("TUPLE_PACK") instructions.push_args(len(tups)) instructions.append(op) self.add_strings(len(tups)) # Use SUB to test if integers are correctly unpacked - elif op == 'SUB': + elif op == "SUB": a = self.random.random_int() // 2 b = self.random.random_int() // 2 instructions.push_args(0, a, b) instructions.append(op) instructions.push_args(1) - instructions.append('SWAP') + instructions.append("SWAP") instructions.append(op) instructions.push_args(1) - instructions.append('TUPLE_PACK') + instructions.append("TUPLE_PACK") self.add_stack_items(1) - elif op == 'ENCODE_FLOAT': + elif op == "ENCODE_FLOAT": f = self.random.random_float(8) - f_bytes = struct.pack('>f', f) + f_bytes = struct.pack(">f", f) instructions.push_args(f_bytes) instructions.append(op) self.add_stack_items(1) - elif op == 'ENCODE_DOUBLE': + elif op == "ENCODE_DOUBLE": d = self.random.random_float(11) - d_bytes = struct.pack('>d', d) + d_bytes = struct.pack(">d", d) instructions.push_args(d_bytes) instructions.append(op) self.add_stack_items(1) - elif op == 'DECODE_FLOAT': + elif op == "DECODE_FLOAT": f = self.random.random_float(8) instructions.push_args(fdb.tuple.SingleFloat(f)) instructions.append(op) self.add_strings(1) - elif op == 'DECODE_DOUBLE': + elif op == "DECODE_DOUBLE": d = self.random.random_float(11) instructions.push_args(d) instructions.append(op) self.add_strings(1) - elif op == 'GET_ESTIMATED_RANGE_SIZE': + elif op == "GET_ESTIMATED_RANGE_SIZE": # Protect against inverted range and identical keys key1 = self.workspace.pack(self.random.random_tuple(1)) key2 = self.workspace.pack(self.random.random_tuple(1)) @@ -576,7 +681,7 @@ class ApiTest(Test): instructions.push_args(key1, key2) instructions.append(op) self.add_strings(1) - elif op == 'GET_RANGE_SPLIT_POINTS': + elif op == "GET_RANGE_SPLIT_POINTS": # Protect against inverted range and identical keys key1 = self.workspace.pack(self.random.random_tuple(1)) key2 = self.workspace.pack(self.random.random_tuple(1)) @@ -593,27 +698,27 @@ class ApiTest(Test): instructions.push_args(key1, key2, chunkSize) instructions.append(op) self.add_strings(1) - elif op == 'TENANT_CREATE': + elif op == "TENANT_CREATE": tenant_name = self.choose_tenant(0.8) self.allocated_tenants.add(tenant_name) instructions.push_args(tenant_name) instructions.append(op) self.add_strings(1) - elif op == 'TENANT_DELETE': + elif op == "TENANT_DELETE": tenant_name = self.choose_tenant(0.2) if tenant_name in self.allocated_tenants: self.allocated_tenants.remove(tenant_name) instructions.push_args(tenant_name) instructions.append(op) self.add_strings(1) - elif op == 'TENANT_SET_ACTIVE': + elif op == "TENANT_SET_ACTIVE": tenant_name = self.choose_tenant(0.8) instructions.push_args(tenant_name) instructions.append(op) self.add_strings(1) - elif op == 'TENANT_CLEAR_ACTIVE': + elif op == "TENANT_CLEAR_ACTIVE": instructions.append(op) - elif op == 'TENANT_LIST': + elif op == "TENANT_LIST": self.ensure_string(instructions, 2) instructions.push_args(self.random.random_int()) test_util.to_front(instructions, 2) @@ -624,27 +729,33 @@ class ApiTest(Test): instructions.append(op) self.add_strings(1) else: - assert False, 'Unknown operation: ' + op + assert False, "Unknown operation: " + op if read_performed and op not in database_reads and op not in tenant_reads: self.outstanding_ops.append((self.stack_size, len(instructions) - 1)) - if args.concurrency == 1 and (op in database_reads or op in database_mutations or op in tenant_reads or op in tenant_mutations or op in ['TENANT_CREATE', 'TENANT_DELETE']): - instructions.append('WAIT_FUTURE') + if args.concurrency == 1 and ( + op in database_reads + or op in database_mutations + or op in tenant_reads + or op in tenant_mutations + or op in ["TENANT_CREATE", "TENANT_DELETE"] + ): + instructions.append("WAIT_FUTURE") instructions.begin_finalization() if not args.no_tenants: - instructions.append('TENANT_CLEAR_ACTIVE') + instructions.append("TENANT_CLEAR_ACTIVE") if args.concurrency == 1: self.wait_for_reads(instructions) test_util.blocking_commit(instructions) self.add_stack_items(1) - instructions.append('NEW_TRANSACTION') + instructions.append("NEW_TRANSACTION") instructions.push_args(self.stack_subspace.key()) - instructions.append('LOG_STACK') + instructions.append("LOG_STACK") test_util.blocking_commit(instructions) @@ -654,22 +765,30 @@ class ApiTest(Test): def check_versionstamps(self, tr, begin_key, limit): next_begin = None incorrect_versionstamps = 0 - for k, v in tr.get_range(begin_key, self.versionstamped_values.range().stop, limit=limit): - next_begin = k + b'\x00' + for k, v in tr.get_range( + begin_key, self.versionstamped_values.range().stop, limit=limit + ): + next_begin = k + b"\x00" random_id = self.versionstamped_values.unpack(k)[0] - versioned_value = v[10:].replace(fdb.tuple.Versionstamp._UNSET_TR_VERSION, v[:10], 1) + versioned_value = v[10:].replace( + fdb.tuple.Versionstamp._UNSET_TR_VERSION, v[:10], 1 + ) versioned_key = self.versionstamped_keys.pack() + versioned_value if tr[versioned_key] != random_id: - util.get_logger().error(' INCORRECT VERSIONSTAMP:') - util.get_logger().error(' %s != %s', repr(tr[versioned_key]), repr(random_id)) + util.get_logger().error(" INCORRECT VERSIONSTAMP:") + util.get_logger().error( + " %s != %s", repr(tr[versioned_key]), repr(random_id) + ) incorrect_versionstamps += 1 if self.api_version >= 520: k2 = self.versionstamped_values_2.pack((random_id,)) if tr[k2] != versioned_value: - util.get_logger().error(' INCORRECT VERSIONSTAMP:') - util.get_logger().error(' %s != %s', repr(tr[k2]), repr(versioned_value)) + util.get_logger().error(" INCORRECT VERSIONSTAMP:") + util.get_logger().error( + " %s != %s", repr(tr[k2]), repr(versioned_value) + ) incorrect_versionstamps += 1 return (next_begin, incorrect_versionstamps) @@ -681,16 +800,26 @@ class ApiTest(Test): incorrect_versionstamps = 0 while begin is not None: - (begin, current_incorrect_versionstamps) = self.check_versionstamps(db, begin, 100) + (begin, current_incorrect_versionstamps) = self.check_versionstamps( + db, begin, 100 + ) incorrect_versionstamps += current_incorrect_versionstamps if incorrect_versionstamps > 0: - errors.append('There were %d failed version stamp operations' % incorrect_versionstamps) + errors.append( + "There were %d failed version stamp operations" + % incorrect_versionstamps + ) return errors def get_result_specifications(self): return [ ResultSpecification(self.workspace, global_error_filter=[1007, 1009, 1021]), - ResultSpecification(self.stack_subspace, key_start_index=1, ordering_index=1, global_error_filter=[1007, 1009, 1021]) + ResultSpecification( + self.stack_subspace, + key_start_index=1, + ordering_index=1, + global_error_filter=[1007, 1009, 1021], + ), ] diff --git a/bindings/bindingtester/tests/test_util.py b/bindings/bindingtester/tests/test_util.py index 0e6069e5b6..13d33c3370 100644 --- a/bindings/bindingtester/tests/test_util.py +++ b/bindings/bindingtester/tests/test_util.py @@ -33,16 +33,20 @@ from bindingtester.known_testers import COMMON_TYPES class RandomGenerator(object): - def __init__(self, max_int_bits=64, api_version=FDB_API_VERSION, types=COMMON_TYPES): + def __init__( + self, max_int_bits=64, api_version=FDB_API_VERSION, types=COMMON_TYPES + ): self.max_int_bits = max_int_bits self.api_version = api_version self.types = list(types) def random_unicode_str(self, length): - return ''.join(self.random_unicode_char() for i in range(0, length)) + return "".join(self.random_unicode_char() for i in range(0, length)) def random_int(self): - num_bits = random.randint(0, self.max_int_bits) # This way, we test small numbers with higher probability + num_bits = random.randint( + 0, self.max_int_bits + ) # This way, we test small numbers with higher probability max_value = (1 << num_bits) - 1 min_value = -max_value - 1 @@ -54,11 +58,15 @@ class RandomGenerator(object): def random_float(self, exp_bits): if random.random() < 0.05: # Choose a special value. - return random.choice([float('-nan'), float('-inf'), -0.0, 0.0, float('inf'), float('nan')]) + return random.choice( + [float("-nan"), float("-inf"), -0.0, 0.0, float("inf"), float("nan")] + ) else: # Choose a value from all over the range of acceptable floats for this precision. sign = -1 if random.random() < 0.5 else 1 - exponent = random.randint(-(1 << (exp_bits - 1)) - 10, (1 << (exp_bits - 1) - 1)) + exponent = random.randint( + -(1 << (exp_bits - 1)) - 10, (1 << (exp_bits - 1) - 1) + ) mantissa = random.random() result = sign * math.pow(2, exponent) * mantissa @@ -73,38 +81,38 @@ class RandomGenerator(object): for i in range(size): choice = random.choice(self.types) - if choice == 'int': + if choice == "int": tup.append(self.random_int()) - elif choice == 'null': + elif choice == "null": tup.append(None) - elif choice == 'bytes': + elif choice == "bytes": tup.append(self.random_string(random.randint(0, 100))) - elif choice == 'string': + elif choice == "string": tup.append(self.random_unicode_str(random.randint(0, 100))) - elif choice == 'uuid': - tup.append(uuid.uuid4()) - elif choice == 'bool': + elif choice == "uuid": + tup.append(uuid.UUID(int=random.getrandbits(128))) + elif choice == "bool": b = random.random() < 0.5 if self.api_version < 500: tup.append(int(b)) else: tup.append(b) - elif choice == 'float': + elif choice == "float": tup.append(fdb.tuple.SingleFloat(self.random_float(8))) - elif choice == 'double': + elif choice == "double": tup.append(self.random_float(11)) - elif choice == 'tuple': + elif choice == "tuple": length = random.randint(0, max_size - size) if length == 0: tup.append(()) else: tup.append(self.random_tuple(length)) - elif choice == 'versionstamp': + elif choice == "versionstamp": if incomplete_versionstamps and random.random() < 0.5: tr_version = fdb.tuple.Versionstamp._UNSET_TR_VERSION else: tr_version = self.random_string(10) - user_version = random.randint(0, 0xffff) + user_version = random.randint(0, 0xFFFF) tup.append(fdb.tuple.Versionstamp(tr_version, user_version)) else: assert False @@ -123,12 +131,19 @@ class RandomGenerator(object): smaller_size = random.randint(1, len(to_add)) tuples.append(to_add[:smaller_size]) else: - non_empty = [x for x in enumerate(to_add) if (isinstance(x[1], list) or isinstance(x[1], tuple)) and len(x[1]) > 0] + non_empty = [ + x + for x in enumerate(to_add) + if (isinstance(x[1], list) or isinstance(x[1], tuple)) + and len(x[1]) > 0 + ] if len(non_empty) > 0 and random.random() < 0.25: # Add a smaller list to test prefixes of nested structures. idx, choice = random.choice(non_empty) smaller_size = random.randint(0, len(to_add[idx])) - tuples.append(to_add[:idx] + (choice[:smaller_size],) + to_add[idx + 1:]) + tuples.append( + to_add[:idx] + (choice[:smaller_size],) + to_add[idx + 1 :] + ) random.shuffle(tuples) return tuples @@ -153,30 +168,40 @@ class RandomGenerator(object): def random_string(self, length): if length == 0: - return b'' + return b"" - return bytes([random.randint(0, 254)] + [random.randint(0, 255) for i in range(0, length - 1)]) + return bytes( + [random.randint(0, 254)] + + [random.randint(0, 255) for i in range(0, length - 1)] + ) def random_unicode_char(self): while True: if random.random() < 0.05: # Choose one of these special character sequences. - specials = ['\U0001f4a9', '\U0001f63c', '\U0001f3f3\ufe0f\u200d\U0001f308', '\U0001f1f5\U0001f1f2', '\uf8ff', - '\U0002a2b2', '\u05e9\u05dc\u05d5\u05dd'] + specials = [ + "\U0001f4a9", + "\U0001f63c", + "\U0001f3f3\ufe0f\u200d\U0001f308", + "\U0001f1f5\U0001f1f2", + "\uf8ff", + "\U0002a2b2", + "\u05e9\u05dc\u05d5\u05dd", + ] return random.choice(specials) - c = random.randint(0, 0xffff) - if unicodedata.category(chr(c))[0] in 'LMNPSZ': + c = random.randint(0, 0xFFFF) + if unicodedata.category(chr(c))[0] in "LMNPSZ": return chr(c) def error_string(error_code): - return fdb.tuple.pack((b'ERROR', bytes(str(error_code), 'utf-8'))) + return fdb.tuple.pack((b"ERROR", bytes(str(error_code), "utf-8"))) def blocking_commit(instructions): - instructions.append('COMMIT') - instructions.append('WAIT_FUTURE') - instructions.append('RESET') + instructions.append("COMMIT") + instructions.append("WAIT_FUTURE") + instructions.append("RESET") def to_front(instructions, index): @@ -184,19 +209,19 @@ def to_front(instructions, index): pass elif index == 1: instructions.push_args(1) - instructions.append('SWAP') + instructions.append("SWAP") elif index == 2: instructions.push_args(index - 1) - instructions.append('SWAP') + instructions.append("SWAP") instructions.push_args(index) - instructions.append('SWAP') + instructions.append("SWAP") else: instructions.push_args(index - 1) - instructions.append('SWAP') + instructions.append("SWAP") instructions.push_args(index) - instructions.append('SWAP') + instructions.append("SWAP") instructions.push_args(index - 1) - instructions.append('SWAP') + instructions.append("SWAP") to_front(instructions, index - 1) diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index d44c1e75b8..e5f777c71d 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -386,8 +386,7 @@ void setBlobFilePointer(FDBBGFilePointer* dest, const BlobFilePointerRef& source dest->file_offset = source.offset; dest->file_length = source.length; dest->full_file_length = source.fullFileLength; - // FIXME: add version info to each source file pointer - dest->file_version = 0; + dest->file_version = source.fileVersion; // handle encryption if (source.cipherKeysCtx.present()) { @@ -717,6 +716,17 @@ extern "C" DLLEXPORT FDBFuture* fdb_database_blobbify_range(FDBDatabase* db, .extractPtr()); } +extern "C" DLLEXPORT FDBFuture* fdb_database_blobbify_range_blocking(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length) { + return (FDBFuture*)(DB(db) + ->blobbifyRangeBlocking(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length), + StringRef(end_key_name, end_key_name_length))) + .extractPtr()); +} + extern "C" DLLEXPORT FDBFuture* fdb_database_unblobbify_range(FDBDatabase* db, uint8_t const* begin_key_name, int begin_key_name_length, @@ -758,6 +768,25 @@ extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_verify_blob_rang .extractPtr()); } +extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_flush_blob_range(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + fdb_bool_t compact, + int64_t version) { + Optional rv; + if (version != latestVersion) { + rv = version; + } + return (FDBFuture*)(DB(db) + ->flushBlobRange(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length), + StringRef(end_key_name, end_key_name_length)), + compact, + rv) + .extractPtr()); +} + extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_client_status(FDBDatabase* db) { return (FDBFuture*)(DB(db)->getClientStatus().extractPtr()); } @@ -799,6 +828,17 @@ extern "C" DLLEXPORT FDBFuture* fdb_tenant_blobbify_range(FDBTenant* tenant, .extractPtr()); } +extern "C" DLLEXPORT FDBFuture* fdb_tenant_blobbify_range_blocking(FDBTenant* tenant, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length) { + return (FDBFuture*)(TENANT(tenant) + ->blobbifyRangeBlocking(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length), + StringRef(end_key_name, end_key_name_length))) + .extractPtr()); +} + extern "C" DLLEXPORT FDBFuture* fdb_tenant_unblobbify_range(FDBTenant* tenant, uint8_t const* begin_key_name, int begin_key_name_length, @@ -840,6 +880,25 @@ extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_verify_blob_range( .extractPtr()); } +extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_flush_blob_range(FDBTenant* tenant, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + fdb_bool_t compact, + int64_t version) { + Optional rv; + if (version != latestVersion) { + rv = version; + } + return (FDBFuture*)(TENANT(tenant) + ->flushBlobRange(KeyRangeRef(StringRef(begin_key_name, begin_key_name_length), + StringRef(end_key_name, end_key_name_length)), + compact, + rv) + .extractPtr()); +} + extern "C" DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_get_id(FDBTenant* tenant) { return (FDBFuture*)(TENANT(tenant)->getId().extractPtr()); } diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index 6280fdb680..b43dcc27c0 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -417,6 +417,12 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_blobbify_range(FDBDatabase* uint8_t const* end_key_name, int end_key_name_length); +DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_blobbify_range_blocking(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length); + DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_unblobbify_range(FDBDatabase* db, uint8_t const* begin_key_name, int begin_key_name_length, @@ -437,6 +443,14 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_verify_blob_range(FDBDataba int end_key_name_length, int64_t version); +DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_flush_blob_range(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + fdb_bool_t compact, + int64_t version); + DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_client_status(FDBDatabase* db); DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_tenant_create_transaction(FDBTenant* tenant, @@ -460,6 +474,12 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_blobbify_range(FDBTenant* ten uint8_t const* end_key_name, int end_key_name_length); +DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_blobbify_range_blocking(FDBTenant* tenant, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length); + DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_unblobbify_range(FDBTenant* tenant, uint8_t const* begin_key_name, int begin_key_name_length, @@ -487,6 +507,14 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_verify_blob_range(FDBTenant* int end_key_name_length, int64_t version); +DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_flush_blob_range(FDBTenant* tenant, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + fdb_bool_t compact, + int64_t version); + DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_tenant_get_id(FDBTenant* tenant); DLLEXPORT void fdb_tenant_destroy(FDBTenant* tenant); diff --git a/bindings/c/test/apitester/TesterApiWorkload.cpp b/bindings/c/test/apitester/TesterApiWorkload.cpp index 778849489b..5190b5433e 100644 --- a/bindings/c/test/apitester/TesterApiWorkload.cpp +++ b/bindings/c/test/apitester/TesterApiWorkload.cpp @@ -380,7 +380,6 @@ void ApiWorkload::setupBlobGranules(TTaskFct cont) { void ApiWorkload::blobbifyTenant(std::optional tenantId, std::shared_ptr> blobbifiedCount, TTaskFct cont) { - auto retBlobbifyRange = std::make_shared(false); execOperation( [=](auto ctx) { fdb::Key begin(1, '\x00'); @@ -388,48 +387,17 @@ void ApiWorkload::blobbifyTenant(std::optional tenantId, info(fmt::format("setup: blobbifying {}: [\\x00 - \\xff)\n", debugTenantStr(tenantId))); - fdb::Future f = ctx->dbOps()->blobbifyRange(begin, end).eraseType(); - ctx->continueAfter(f, [ctx, retBlobbifyRange, f]() { - *retBlobbifyRange = f.get(); + // wait for blobbification before returning + fdb::Future f = ctx->dbOps()->blobbifyRangeBlocking(begin, end).eraseType(); + ctx->continueAfter(f, [ctx, f]() { + bool success = f.get(); + ASSERT(success); ctx->done(); }); }, [=]() { - if (!*retBlobbifyRange) { - schedule([=]() { blobbifyTenant(tenantId, blobbifiedCount, cont); }); - } else { - schedule([=]() { verifyTenant(tenantId, blobbifiedCount, cont); }); - } - }, - /*tenant=*/getTenant(tenantId), - /* failOnError = */ false); -} - -void ApiWorkload::verifyTenant(std::optional tenantId, - std::shared_ptr> blobbifiedCount, - TTaskFct cont) { - auto retVerifyVersion = std::make_shared(-1); - - execOperation( - [=](auto ctx) { - fdb::Key begin(1, '\x00'); - fdb::Key end(1, '\xff'); - - info(fmt::format("setup: verifying {}: [\\x00 - \\xff)\n", debugTenantStr(tenantId))); - - fdb::Future f = ctx->dbOps()->verifyBlobRange(begin, end, /*latest_version*/ -2).eraseType(); - ctx->continueAfter(f, [ctx, retVerifyVersion, f]() { - *retVerifyVersion = f.get(); - ctx->done(); - }); - }, - [=]() { - if (*retVerifyVersion == -1) { - schedule([=]() { verifyTenant(tenantId, blobbifiedCount, cont); }); - } else { - if (blobbifiedCount->fetch_sub(1) == 1) { - schedule(cont); - } + if (blobbifiedCount->fetch_sub(1) == 1) { + schedule(cont); } }, /*tenant=*/getTenant(tenantId), diff --git a/bindings/c/test/apitester/TesterApiWorkload.h b/bindings/c/test/apitester/TesterApiWorkload.h index a750c0f162..0d13aa859c 100644 --- a/bindings/c/test/apitester/TesterApiWorkload.h +++ b/bindings/c/test/apitester/TesterApiWorkload.h @@ -135,7 +135,6 @@ protected: // Generic BlobGranules setup. void setupBlobGranules(TTaskFct cont); void blobbifyTenant(std::optional tenantId, std::shared_ptr> blobbifiedCount, TTaskFct cont); - void verifyTenant(std::optional tenantId, std::shared_ptr> blobbifiedCount, TTaskFct cont); private: void populateDataTx(TTaskFct cont, std::optional tenantId); diff --git a/bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp b/bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp index 54cf54d372..a344bd1f38 100644 --- a/bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp +++ b/bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp @@ -37,6 +37,9 @@ public: if (Random::get().randomInt(0, 1) == 0) { excludedOpTypes.push_back(OP_CLEAR_RANGE); } + if (Random::get().randomInt(0, 1) == 0) { + excludedOpTypes.push_back(OP_FLUSH); + } } private: @@ -51,7 +54,8 @@ private: OP_GET_BLOB_RANGES, OP_VERIFY, OP_READ_DESC, - OP_LAST = OP_READ_DESC + OP_FLUSH, + OP_LAST = OP_FLUSH }; std::vector excludedOpTypes; @@ -303,7 +307,10 @@ private: fdb::native::FDBReadBlobGranuleContext& bgCtx, fdb::GranuleFilePointer snapshotFile, fdb::KeyRange keyRange, - fdb::native::FDBBGTenantPrefix const* tenantPrefix) { + fdb::native::FDBBGTenantPrefix const* tenantPrefix, + int64_t& prevFileVersion) { + ASSERT(snapshotFile.fileVersion > prevFileVersion); + prevFileVersion = snapshotFile.fileVersion; if (validatedFiles.contains(snapshotFile.filename)) { return; } @@ -339,7 +346,10 @@ private: fdb::GranuleFilePointer deltaFile, fdb::KeyRange keyRange, fdb::native::FDBBGTenantPrefix const* tenantPrefix, - int64_t& lastDFMaxVersion) { + int64_t& lastDFMaxVersion, + int64_t& prevFileVersion) { + ASSERT(deltaFile.fileVersion > prevFileVersion); + prevFileVersion = deltaFile.fileVersion; if (validatedFiles.contains(deltaFile.filename)) { return; } @@ -380,6 +390,9 @@ private: } lastDFMaxVersion = std::max(lastDFMaxVersion, thisDFMaxVersion); + // can be higher due to empty versions but must not be lower + ASSERT(lastDFMaxVersion <= prevFileVersion); + // TODO have delta mutations update map } @@ -392,22 +405,28 @@ private: ASSERT(desc.keyRange.beginKey < desc.keyRange.endKey); ASSERT(tenantId.has_value() == desc.tenantPrefix.present); // beginVersion of zero means snapshot present + int64_t prevFileVersion = 0; // validate snapshot file ASSERT(desc.snapshotFile.has_value()); if (BG_API_DEBUG_VERBOSE) { info(fmt::format("Loading snapshot file {0}\n", fdb::toCharsRef(desc.snapshotFile->filename))); } - validateSnapshotData(ctx, bgCtx, *desc.snapshotFile, desc.keyRange, &desc.tenantPrefix); + validateSnapshotData(ctx, bgCtx, *desc.snapshotFile, desc.keyRange, &desc.tenantPrefix, prevFileVersion); // validate delta files int64_t lastDFMaxVersion = 0; for (int i = 0; i < desc.deltaFiles.size(); i++) { - validateDeltaData(ctx, bgCtx, desc.deltaFiles[i], desc.keyRange, &desc.tenantPrefix, lastDFMaxVersion); + validateDeltaData( + ctx, bgCtx, desc.deltaFiles[i], desc.keyRange, &desc.tenantPrefix, lastDFMaxVersion, prevFileVersion); } // validate memory mutations - int64_t lastVersion = 0; + if (desc.memoryMutations.size()) { + ASSERT(desc.memoryMutations.front().version > lastDFMaxVersion); + ASSERT(desc.memoryMutations.front().version > prevFileVersion); + } + int64_t lastVersion = prevFileVersion; for (int i = 0; i < desc.memoryMutations.size(); i++) { fdb::GranuleMutation& m = desc.memoryMutations[i]; ASSERT(m.type == 0 || m.type == 1); @@ -494,6 +513,33 @@ private: getTenant(tenantId)); } + void randomFlushOp(TTaskFct cont, std::optional tenantId) { + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); + fdb::native::fdb_bool_t compact = Random::get().randomBool(0.5); + + auto result = std::make_shared(false); + + debugOp(compact ? "Flush" : "Compact", keyRange, tenantId, "starting"); + execOperation( + [keyRange, compact, result](auto ctx) { + fdb::Future f = + ctx->dbOps() + ->flushBlobRange(keyRange.beginKey, keyRange.endKey, compact, -2 /* latest version*/) + .eraseType(); + ctx->continueAfter(f, [ctx, result, f]() { + *result = f.get(); + ctx->done(); + }); + }, + [this, keyRange, compact, result, tenantId, cont]() { + ASSERT(*result); + debugOp(compact ? "Flush " : "Compact ", keyRange, tenantId, "Complete"); + schedule(cont); + }, + getTenant(tenantId), + /* failOnError = */ false); + } + void randomOperation(TTaskFct cont) override { std::optional tenantId = randomTenant(); @@ -530,6 +576,13 @@ private: case OP_READ_DESC: randomReadDescription(cont, tenantId); break; + case OP_FLUSH: + randomFlushOp(cont, tenantId); + // don't do too many flushes because they're expensive + if (Random::get().randomInt(0, 1) == 0) { + excludedOpTypes.push_back(OP_FLUSH); + } + break; } } }; diff --git a/bindings/c/test/apitester/TesterBlobGranuleErrorsWorkload.cpp b/bindings/c/test/apitester/TesterBlobGranuleErrorsWorkload.cpp index d4a0383119..cf52212b74 100644 --- a/bindings/c/test/apitester/TesterBlobGranuleErrorsWorkload.cpp +++ b/bindings/c/test/apitester/TesterBlobGranuleErrorsWorkload.cpp @@ -44,7 +44,9 @@ private: OP_CANCEL_BLOBBIFY, OP_CANCEL_UNBLOBBIFY, OP_CANCEL_PURGE, - OP_LAST = OP_CANCEL_PURGE + OP_CANCEL_FLUSH, + OP_FLUSH_TOO_OLD, + OP_LAST = OP_FLUSH_TOO_OLD }; void setup(TTaskFct cont) override { setupBlobGranules(cont); } @@ -122,14 +124,10 @@ private: void randomPurgeUnalignedOp(TTaskFct cont) { // blobbify/unblobbify need to be aligned to blob range boundaries, so this should always fail - fdb::Key begin = randomKeyName(); - fdb::Key end = randomKeyName(); - if (begin > end) { - std::swap(begin, end); - } + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); execOperation( - [this, begin, end](auto ctx) { - fdb::Future f = ctx->db().purgeBlobGranules(begin, end, -2, false).eraseType(); + [this, keyRange](auto ctx) { + fdb::Future f = ctx->db().purgeBlobGranules(keyRange.beginKey, keyRange.endKey, -2, false).eraseType(); ctx->continueAfter( f, [this, ctx, f]() { @@ -144,16 +142,12 @@ private: void randomBlobbifyUnalignedOp(bool blobbify, TTaskFct cont) { // blobbify/unblobbify need to be aligned to blob range boundaries, so this should always return false - fdb::Key begin = randomKeyName(); - fdb::Key end = randomKeyName(); - if (begin > end) { - std::swap(begin, end); - } + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); auto success = std::make_shared(false); execOperation( - [begin, end, blobbify, success](auto ctx) { - fdb::Future f = blobbify ? ctx->db().blobbifyRange(begin, end).eraseType() - : ctx->db().unblobbifyRange(begin, end).eraseType(); + [keyRange, blobbify, success](auto ctx) { + fdb::Future f = blobbify ? ctx->db().blobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType() + : ctx->db().unblobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType(); ctx->continueAfter( f, [ctx, f, success]() { @@ -169,103 +163,116 @@ private: } void randomCancelGetGranulesOp(TTaskFct cont) { - fdb::Key begin = randomKeyName(); - fdb::Key end = randomKeyName(); - if (begin > end) { - std::swap(begin, end); - } + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); execTransaction( - [begin, end](auto ctx) { - fdb::Future f = ctx->tx().getBlobGranuleRanges(begin, end, 1000).eraseType(); + [keyRange](auto ctx) { + fdb::Future f = ctx->tx().getBlobGranuleRanges(keyRange.beginKey, keyRange.endKey, 1000).eraseType(); ctx->done(); }, [this, cont]() { schedule(cont); }); } void randomCancelGetRangesOp(TTaskFct cont) { - fdb::Key begin = randomKeyName(); - fdb::Key end = randomKeyName(); - if (begin > end) { - std::swap(begin, end); - } + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); execOperation( - [begin, end](auto ctx) { - fdb::Future f = ctx->db().listBlobbifiedRanges(begin, end, 1000).eraseType(); + [keyRange](auto ctx) { + fdb::Future f = ctx->db().listBlobbifiedRanges(keyRange.beginKey, keyRange.endKey, 1000).eraseType(); ctx->done(); }, [this, cont]() { schedule(cont); }); } void randomCancelVerifyOp(TTaskFct cont) { - fdb::Key begin = randomKeyName(); - fdb::Key end = randomKeyName(); - if (begin > end) { - std::swap(begin, end); - } + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); execOperation( - [begin, end](auto ctx) { - fdb::Future f = ctx->db().verifyBlobRange(begin, end, -2 /* latest version*/).eraseType(); + [keyRange](auto ctx) { + fdb::Future f = + ctx->db().verifyBlobRange(keyRange.beginKey, keyRange.endKey, -2 /* latest version*/).eraseType(); ctx->done(); }, [this, cont]() { schedule(cont); }); } void randomCancelSummarizeOp(TTaskFct cont) { - fdb::Key begin = randomKeyName(); - fdb::Key end = randomKeyName(); - if (begin > end) { - std::swap(begin, end); - } + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); execTransaction( - [begin, end](auto ctx) { - fdb::Future f = ctx->tx().summarizeBlobGranules(begin, end, -2, 1000).eraseType(); + [keyRange](auto ctx) { + fdb::Future f = + ctx->tx().summarizeBlobGranules(keyRange.beginKey, keyRange.endKey, -2, 1000).eraseType(); ctx->done(); }, [this, cont]() { schedule(cont); }); } void randomCancelBlobbifyOp(TTaskFct cont) { - fdb::Key begin = randomKeyName(); - fdb::Key end = randomKeyName(); - if (begin > end) { - std::swap(begin, end); - } + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); execOperation( - [begin, end](auto ctx) { - fdb::Future f = ctx->db().blobbifyRange(begin, end).eraseType(); + [keyRange](auto ctx) { + fdb::Future f; + if (Random::get().randomBool(0.5)) { + f = ctx->db().blobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType(); + } else { + f = ctx->db().blobbifyRangeBlocking(keyRange.beginKey, keyRange.endKey).eraseType(); + } ctx->done(); }, [this, cont]() { schedule(cont); }); } void randomCancelUnblobbifyOp(TTaskFct cont) { - fdb::Key begin = randomKeyName(); - fdb::Key end = randomKeyName(); - if (begin > end) { - std::swap(begin, end); - } + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); execOperation( - [begin, end](auto ctx) { - fdb::Future f = ctx->db().unblobbifyRange(begin, end).eraseType(); + [keyRange](auto ctx) { + fdb::Future f = ctx->db().unblobbifyRange(keyRange.beginKey, keyRange.endKey).eraseType(); ctx->done(); }, [this, cont]() { schedule(cont); }); } void randomCancelPurgeOp(TTaskFct cont) { - fdb::Key begin = randomKeyName(); - fdb::Key end = randomKeyName(); - if (begin > end) { - std::swap(begin, end); - } + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); execOperation( - [begin, end](auto ctx) { - fdb::Future f = ctx->db().purgeBlobGranules(begin, end, -2, false).eraseType(); + [keyRange](auto ctx) { + fdb::Future f = ctx->db().purgeBlobGranules(keyRange.beginKey, keyRange.endKey, -2, false).eraseType(); ctx->done(); }, [this, cont]() { schedule(cont); }); } + void randomCancelFlushOp(TTaskFct cont) { + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); + fdb::native::fdb_bool_t compact = Random::get().randomBool(0.5); + execOperation( + [keyRange, compact](auto ctx) { + fdb::Future f = ctx->db().flushBlobRange(keyRange.beginKey, keyRange.endKey, compact, -2).eraseType(); + ctx->done(); + }, + [this, cont]() { schedule(cont); }); + } + + void randomFlushTooOldOp(TTaskFct cont) { + fdb::KeyRange keyRange = randomNonEmptyKeyRange(); + fdb::native::fdb_bool_t compact = Random::get().randomBool(0.5); + + auto success = std::make_shared(false); + execOperation( + [keyRange, compact, success](auto ctx) { + // version of 1 should be too old + fdb::Future f = ctx->db().flushBlobRange(keyRange.beginKey, keyRange.endKey, compact, 1).eraseType(); + ctx->continueAfter( + f, + [ctx, f, success]() { + *success = f.get(); + ctx->done(); + }, + true); + }, + [this, cont, success]() { + ASSERT(!(*success)); + schedule(cont); + }); + } + void randomOperation(TTaskFct cont) override { OpType txType = (OpType)Random::get().randomInt(0, OP_LAST); switch (txType) { @@ -309,6 +316,12 @@ private: case OP_CANCEL_PURGE: randomCancelPurgeOp(cont); break; + case OP_CANCEL_FLUSH: + randomCancelPurgeOp(cont); + break; + case OP_FLUSH_TOO_OLD: + randomCancelPurgeOp(cont); + break; } } }; diff --git a/bindings/c/test/fdb_api.hpp b/bindings/c/test/fdb_api.hpp index d0397a238b..85c59e1086 100644 --- a/bindings/c/test/fdb_api.hpp +++ b/bindings/c/test/fdb_api.hpp @@ -86,6 +86,7 @@ struct GranuleFilePointer { int64_t offset; int64_t length; int64_t fullFileLength; + int64_t fileVersion; // just keep raw data structures to pass to callbacks native::FDBBGEncryptionCtx encryptionCtx; @@ -95,6 +96,7 @@ struct GranuleFilePointer { offset = nativePointer.file_offset; length = nativePointer.file_length; fullFileLength = nativePointer.full_file_length; + fileVersion = nativePointer.file_version; encryptionCtx = nativePointer.encryption_ctx; } }; @@ -829,11 +831,13 @@ public: virtual Transaction createTransaction() = 0; virtual TypedFuture blobbifyRange(KeyRef begin, KeyRef end) = 0; + virtual TypedFuture blobbifyRangeBlocking(KeyRef begin, KeyRef end) = 0; virtual TypedFuture unblobbifyRange(KeyRef begin, KeyRef end) = 0; virtual TypedFuture listBlobbifiedRanges(KeyRef begin, KeyRef end, int rangeLimit) = 0; virtual TypedFuture verifyBlobRange(KeyRef begin, KeyRef end, int64_t version) = 0; + virtual TypedFuture flushBlobRange(KeyRef begin, KeyRef end, bool compact, int64_t version) = 0; virtual TypedFuture purgeBlobGranules(KeyRef begin, KeyRef end, int64_t version, @@ -901,6 +905,13 @@ public: return native::fdb_tenant_blobbify_range(tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end)); } + TypedFuture blobbifyRangeBlocking(KeyRef begin, KeyRef end) override { + if (!tenant) + throw std::runtime_error("blobbifyRangeBlocking() from null tenant"); + return native::fdb_tenant_blobbify_range_blocking( + tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end)); + } + TypedFuture unblobbifyRange(KeyRef begin, KeyRef end) override { if (!tenant) throw std::runtime_error("unblobbifyRange() from null tenant"); @@ -922,6 +933,13 @@ public: tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end), version); } + TypedFuture flushBlobRange(KeyRef begin, KeyRef end, bool compact, int64_t version) override { + if (!tenant) + throw std::runtime_error("flushBlobRange() from null tenant"); + return native::fdb_tenant_flush_blob_range( + tenant.get(), begin.data(), intSize(begin), end.data(), intSize(end), compact, version); + } + TypedFuture getId() { if (!tenant) throw std::runtime_error("getId() from null tenant"); @@ -1026,12 +1044,26 @@ public: db.get(), begin.data(), intSize(begin), end.data(), intSize(end), version); } + TypedFuture flushBlobRange(KeyRef begin, KeyRef end, bool compact, int64_t version) override { + if (!db) + throw std::runtime_error("flushBlobRange from null database"); + return native::fdb_database_flush_blob_range( + db.get(), begin.data(), intSize(begin), end.data(), intSize(end), compact, version); + } + TypedFuture blobbifyRange(KeyRef begin, KeyRef end) override { if (!db) throw std::runtime_error("blobbifyRange from null database"); return native::fdb_database_blobbify_range(db.get(), begin.data(), intSize(begin), end.data(), intSize(end)); } + TypedFuture blobbifyRangeBlocking(KeyRef begin, KeyRef end) override { + if (!db) + throw std::runtime_error("blobbifyRangeBlocking from null database"); + return native::fdb_database_blobbify_range_blocking( + db.get(), begin.data(), intSize(begin), end.data(), intSize(end)); + } + TypedFuture unblobbifyRange(KeyRef begin, KeyRef end) override { if (!db) throw std::runtime_error("unblobbifyRange from null database"); diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index 8be5d629e0..632d82d660 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -942,6 +942,41 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1blobbi return (jlong)f; } +JNIEXPORT jlong JNICALL +Java_com_apple_foundationdb_FDBDatabase_Database_1blobbifyRangeBlocking(JNIEnv* jenv, + jobject, + jlong dbPtr, + jbyteArray beginKeyBytes, + jbyteArray endKeyBytes) { + if (!dbPtr || !beginKeyBytes || !endKeyBytes) { + throwParamNotNull(jenv); + return 0; + } + + FDBDatabase* database = (FDBDatabase*)dbPtr; + + uint8_t* beginKeyArr = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL); + if (!beginKeyArr) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + uint8_t* endKeyArr = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); + if (!endKeyArr) { + jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + FDBFuture* f = fdb_database_blobbify_range_blocking( + database, beginKeyArr, jenv->GetArrayLength(beginKeyBytes), endKeyArr, jenv->GetArrayLength(endKeyBytes)); + jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT); + jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKeyArr, JNI_ABORT); + return (jlong)f; +} + JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1unblobbifyRange(JNIEnv* jenv, jobject, jlong dbPtr, @@ -1058,6 +1093,46 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1getCli return (jlong)f; } +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1flushBlobRange(JNIEnv* jenv, + jobject, + jlong dbPtr, + jbyteArray beginKeyBytes, + jbyteArray endKeyBytes, + jboolean compact, + jlong version) { + if (!dbPtr || !beginKeyBytes || !endKeyBytes) { + throwParamNotNull(jenv); + return 0; + } + FDBDatabase* tr = (FDBDatabase*)dbPtr; + + uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL); + if (!startKey) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); + if (!endKey) { + jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + FDBFuture* f = fdb_database_flush_blob_range(tr, + startKey, + jenv->GetArrayLength(beginKeyBytes), + endKey, + jenv->GetArrayLength(endKeyBytes), + (fdb_bool_t)compact, + version); + jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT); + jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT); + return (jlong)f; +} + JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv* jenv, jobject, jint predicate, @@ -1212,6 +1287,39 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1blobbifyRa return (jlong)f; } +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1blobbifyRangeBlocking(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray beginKeyBytes, + jbyteArray endKeyBytes) { + if (!tPtr || !beginKeyBytes || !endKeyBytes) { + throwParamNotNull(jenv); + return 0; + } + FDBTenant* tenant = (FDBTenant*)tPtr; + + uint8_t* beginKeyArr = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL); + if (!beginKeyArr) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + uint8_t* endKeyArr = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); + if (!endKeyArr) { + jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + FDBFuture* f = fdb_tenant_blobbify_range_blocking( + tenant, beginKeyArr, jenv->GetArrayLength(beginKeyBytes), endKeyArr, jenv->GetArrayLength(endKeyBytes)); + jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT); + jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKeyArr, JNI_ABORT); + return (jlong)f; +} + JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1unblobbifyRange(JNIEnv* jenv, jobject, jlong tPtr, @@ -1313,6 +1421,46 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1verifyBlob return (jlong)f; } +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1flushBlobRange(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray beginKeyBytes, + jbyteArray endKeyBytes, + jboolean compact, + jlong version) { + if (!tPtr || !beginKeyBytes || !endKeyBytes) { + throwParamNotNull(jenv); + return 0; + } + FDBTenant* tenant = (FDBTenant*)tPtr; + + uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL); + if (!startKey) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); + if (!endKey) { + jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + FDBFuture* f = fdb_tenant_flush_blob_range(tenant, + startKey, + jenv->GetArrayLength(beginKeyBytes), + endKey, + jenv->GetArrayLength(endKeyBytes), + (fdb_bool_t)compact, + version); + jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT); + jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT); + return (jlong)f; +} + JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTenant_Tenant_1getId(JNIEnv* jenv, jobject, jlong tPtr) { if (!tPtr) { throwParamNotNull(jenv); diff --git a/bindings/java/src/integration/com/apple/foundationdb/BlobGranuleIntegrationTest.java b/bindings/java/src/integration/com/apple/foundationdb/BlobGranuleIntegrationTest.java index 25baf80407..170b138def 100644 --- a/bindings/java/src/integration/com/apple/foundationdb/BlobGranuleIntegrationTest.java +++ b/bindings/java/src/integration/com/apple/foundationdb/BlobGranuleIntegrationTest.java @@ -50,20 +50,6 @@ class BlobGranuleIntegrationTest { } } - void waitForVerify(Database db, Range blobRange) throws InterruptedException { - System.out.println("Verifying"); - while (true) { - Long verifyVersion = db.verifyBlobRange(blobRange.begin, blobRange.end).join(); - if (verifyVersion != null && verifyVersion > 0) { - System.out.println("Verify succeeded @ " + verifyVersion); - return; - } else { - System.out.println("Verify failed, sleeping"); - Thread.sleep(100); - } - } - } - @Test void blobManagementFunctionsTest() throws Exception { /* @@ -79,9 +65,12 @@ class BlobGranuleIntegrationTest { Range blobRange = Range.startsWith(key); try (Database db = fdb.open()) { - db.blobbifyRange(blobRange.begin, blobRange.end).join(); + boolean blobbifySuccess = db.blobbifyRangeBlocking(blobRange.begin, blobRange.end).join(); + Assertions.assertTrue(blobbifySuccess); - waitForVerify(db, blobRange); + Long verifyVersion = db.verifyBlobRange(blobRange.begin, blobRange.end).join(); + + Assertions.assertTrue(verifyVersion >= 0); // list blob ranges KeyRangeArrayResult blobRanges = db.listBlobbifiedRanges(blobRange.begin, blobRange.end, 2).join(); @@ -89,23 +78,41 @@ class BlobGranuleIntegrationTest { Assertions.assertArrayEquals(blobRange.begin, blobRanges.getKeyRanges().get(0).begin); Assertions.assertArrayEquals(blobRange.end, blobRanges.getKeyRanges().get(0).end); + boolean flushSuccess = db.flushBlobRange(blobRange.begin, blobRange.end, false).join(); + Assertions.assertTrue(flushSuccess); + + // verify after flush + Long verifyVersionAfterFlush = db.verifyBlobRange(blobRange.begin, blobRange.end).join(); + Assertions.assertTrue(verifyVersionAfterFlush >= 0); + Assertions.assertTrue(verifyVersionAfterFlush >= verifyVersion); + + boolean compactSuccess = db.flushBlobRange(blobRange.begin, blobRange.end, true).join(); + Assertions.assertTrue(compactSuccess); + + Long verifyVersionAfterCompact = db.verifyBlobRange(blobRange.begin, blobRange.end).join(); + Assertions.assertTrue(verifyVersionAfterCompact >= 0); + Assertions.assertTrue(verifyVersionAfterCompact >= verifyVersionAfterFlush); + // purge/wait byte[] purgeKey = db.purgeBlobGranules(blobRange.begin, blobRange.end, -2, false).join(); db.waitPurgeGranulesComplete(purgeKey).join(); // verify again - waitForVerify(db, blobRange); + Long verifyVersionAfterPurge = db.verifyBlobRange(blobRange.begin, blobRange.end).join(); + Assertions.assertTrue(verifyVersionAfterPurge >= 0); + Assertions.assertTrue(verifyVersionAfterPurge >= verifyVersionAfterCompact); // force purge/wait byte[] forcePurgeKey = db.purgeBlobGranules(blobRange.begin, blobRange.end, -2, true).join(); db.waitPurgeGranulesComplete(forcePurgeKey).join(); // check verify fails - Long verifyVersion = db.verifyBlobRange(blobRange.begin, blobRange.end).join(); - Assertions.assertEquals(-1, verifyVersion); + Long verifyVersionLast = db.verifyBlobRange(blobRange.begin, blobRange.end).join(); + Assertions.assertEquals(-1, verifyVersionLast); // unblobbify - db.unblobbifyRange(blobRange.begin, blobRange.end).join(); + boolean unblobbifySuccess = db.unblobbifyRange(blobRange.begin, blobRange.end).join(); + Assertions.assertTrue(unblobbifySuccess); System.out.println("Blob granule management tests complete!"); } diff --git a/bindings/java/src/main/com/apple/foundationdb/Database.java b/bindings/java/src/main/com/apple/foundationdb/Database.java index cf5527b206..68a0d7c802 100644 --- a/bindings/java/src/main/com/apple/foundationdb/Database.java +++ b/bindings/java/src/main/com/apple/foundationdb/Database.java @@ -246,6 +246,31 @@ public interface Database extends AutoCloseable, TransactionContext { */ CompletableFuture blobbifyRange(byte[] beginKey, byte[] endKey, Executor e); + /** + * Runs {@link #blobbifyRange(byte[] beginKey, byte[] endKey, boolean wait)} on the default executor. + * + * @param beginKey start of the key range + * @param endKey end of the key range + * @param wait wait for blobbification to complete + + * @return if the recording of the range was successful + */ + default CompletableFuture blobbifyRangeBlocking(byte[] beginKey, byte[] endKey) { + return blobbifyRangeBlocking(beginKey, endKey, getExecutor()); + } + + /** + * Sets a range to be blobbified in the database. Must be a completely unblobbified range. + * + * @param beginKey start of the key range + * @param endKey end of the key range + * @param wait wait for blobbification to complete + * @param e the {@link Executor} to use for asynchronous callbacks + + * @return if the recording of the range was successful + */ + CompletableFuture blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e); + /** * Runs {@link #unblobbifyRange(byte[] beginKey, byte[] endKey)} on the default executor. * @@ -331,6 +356,46 @@ public interface Database extends AutoCloseable, TransactionContext { */ CompletableFuture verifyBlobRange(byte[] beginKey, byte[] endKey, long version, Executor e); + /** + * Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey)} on the default executor. + * + * @param beginKey start of the key range + * @param endKey end of the key range + * @param compact force compact or just flush + * + * @return a future with a boolean for success or failure + */ + default CompletableFuture flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact) { + return flushBlobRange(beginKey, endKey, compact, -2, getExecutor()); + } + + /** + * Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey, long version)} on the default executor. + * + * @param beginKey start of the key range + * @param endKey end of the key range + * @param compact force compact or just flush + * @param version version to flush at + * + * @return a future with a boolean for success or failure + */ + default CompletableFuture flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version) { + return flushBlobRange(beginKey, endKey, compact, version, getExecutor()); + } + + /** + * Checks if a blob range is blobbified. + * + * @param beginKey start of the key range + * @param endKey end of the key range + * @param compact force compact or just flush + * @param version version to flush at + * @param e the {@link Executor} to use for asynchronous callbacks + * + * @return a future with a boolean for success or failure + */ + CompletableFuture flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e); + /** * Runs a read-only transactional function against this {@code Database} with retry logic. * {@link Function#apply(Object) apply(ReadTransaction)} will be called on the diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java index 3ba848f330..35e48a9036 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java @@ -229,6 +229,16 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume } } + @Override + public CompletableFuture blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e) { + pointerReadLock.lock(); + try { + return new FutureBool(Database_blobbifyRangeBlocking(getPtr(), beginKey, endKey), e); + } finally { + pointerReadLock.unlock(); + } + } + @Override public CompletableFuture unblobbifyRange(byte[] beginKey, byte[] endKey, Executor e) { pointerReadLock.lock(); @@ -259,6 +269,16 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume } } + @Override + public CompletableFuture flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e) { + pointerReadLock.lock(); + try { + return new FutureBool(Database_flushBlobRange(getPtr(), beginKey, endKey, compact, version), e); + } finally { + pointerReadLock.unlock(); + } + } + @Override public Executor getExecutor() { return executor; @@ -287,8 +307,10 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume private native long Database_purgeBlobGranules(long cPtr, byte[] beginKey, byte[] endKey, long purgeVersion, boolean force); private native long Database_waitPurgeGranulesComplete(long cPtr, byte[] purgeKey); private native long Database_blobbifyRange(long cPtr, byte[] beginKey, byte[] endKey); + private native long Database_blobbifyRangeBlocking(long cPtr, byte[] beginKey, byte[] endKey); private native long Database_unblobbifyRange(long cPtr, byte[] beginKey, byte[] endKey); private native long Database_listBlobbifiedRanges(long cPtr, byte[] beginKey, byte[] endKey, int rangeLimit); private native long Database_verifyBlobRange(long cPtr, byte[] beginKey, byte[] endKey, long version); + private native long Database_flushBlobRange(long cPtr, byte[] beginKey, byte[] endKey, boolean compact, long version); private native long Database_getClientStatus(long cPtr); } \ No newline at end of file diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java b/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java index d7474430c7..3f79323c76 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBTenant.java @@ -170,6 +170,16 @@ class FDBTenant extends NativeObjectWrapper implements Tenant { } } + @Override + public CompletableFuture blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e) { + pointerReadLock.lock(); + try { + return new FutureBool(Tenant_blobbifyRangeBlocking(getPtr(), beginKey, endKey), e); + } finally { + pointerReadLock.unlock(); + } + } + @Override public CompletableFuture unblobbifyRange(byte[] beginKey, byte[] endKey, Executor e) { pointerReadLock.lock(); @@ -200,6 +210,16 @@ class FDBTenant extends NativeObjectWrapper implements Tenant { } } + @Override + public CompletableFuture flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e) { + pointerReadLock.lock(); + try { + return new FutureBool(Tenant_flushBlobRange(getPtr(), beginKey, endKey, compact, version), e); + } finally { + pointerReadLock.unlock(); + } + } + @Override public CompletableFuture getId(Executor e) { pointerReadLock.lock(); @@ -233,8 +253,10 @@ class FDBTenant extends NativeObjectWrapper implements Tenant { private native long Tenant_purgeBlobGranules(long cPtr, byte[] beginKey, byte[] endKey, long purgeVersion, boolean force); private native long Tenant_waitPurgeGranulesComplete(long cPtr, byte[] purgeKey); private native long Tenant_blobbifyRange(long cPtr, byte[] beginKey, byte[] endKey); + private native long Tenant_blobbifyRangeBlocking(long cPtr, byte[] beginKey, byte[] endKey); private native long Tenant_unblobbifyRange(long cPtr, byte[] beginKey, byte[] endKey); private native long Tenant_listBlobbifiedRanges(long cPtr, byte[] beginKey, byte[] endKey, int rangeLimit); private native long Tenant_verifyBlobRange(long cPtr, byte[] beginKey, byte[] endKey, long version); + private native long Tenant_flushBlobRange(long cPtr, byte[] beginKey, byte[] endKey, boolean compact, long version); private native long Tenant_getId(long cPtr); } \ No newline at end of file diff --git a/bindings/java/src/main/com/apple/foundationdb/Tenant.java b/bindings/java/src/main/com/apple/foundationdb/Tenant.java index 732ea8ffb3..2504d5be39 100644 --- a/bindings/java/src/main/com/apple/foundationdb/Tenant.java +++ b/bindings/java/src/main/com/apple/foundationdb/Tenant.java @@ -333,6 +333,31 @@ public interface Tenant extends AutoCloseable, TransactionContext { */ CompletableFuture blobbifyRange(byte[] beginKey, byte[] endKey, Executor e); + /** + * Runs {@link #blobbifyRange(byte[] beginKey, byte[] endKey, boolean wait)} on the default executor. + * + * @param beginKey start of the key range + * @param endKey end of the key range + * @param wait wait for blobbification to complete + + * @return if the recording of the range was successful + */ + default CompletableFuture blobbifyRangeBlocking(byte[] beginKey, byte[] endKey) { + return blobbifyRangeBlocking(beginKey, endKey, getExecutor()); + } + + /** + * Sets a range to be blobbified in this tenant. Must be a completely unblobbified range. + * + * @param beginKey start of the key range + * @param endKey end of the key range + * @param wait wait for blobbification to complete + * @param e the {@link Executor} to use for asynchronous callbacks + + * @return if the recording of the range was successful + */ + CompletableFuture blobbifyRangeBlocking(byte[] beginKey, byte[] endKey, Executor e); + /** * Runs {@link #unblobbifyRange(byte[] beginKey, byte[] endKey)} on the default executor. * @@ -418,6 +443,46 @@ public interface Tenant extends AutoCloseable, TransactionContext { */ CompletableFuture verifyBlobRange(byte[] beginKey, byte[] endKey, long version, Executor e); + /** + * Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey)} on the default executor. + * + * @param beginKey start of the key range + * @param endKey end of the key range + * @param compact force compact or just flush + * + * @return a future with a boolean for success or failure + */ + default CompletableFuture flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact) { + return flushBlobRange(beginKey, endKey, compact, -2, getExecutor()); + } + + /** + * Runs {@link #flushBlobRange(byte[] beginKey, byte[] endKey, long version)} on the default executor. + * + * @param beginKey start of the key range + * @param endKey end of the key range + * @param compact force compact or just flush + * @param version version to flush at + * + * @return a future with a boolean for success or failure + */ + default CompletableFuture flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version) { + return flushBlobRange(beginKey, endKey, compact, version, getExecutor()); + } + + /** + * Checks if a blob range is blobbified. + * + * @param beginKey start of the key range + * @param endKey end of the key range + * @param compact force compact or just flush + * @param version version to flush at + * @param e the {@link Executor} to use for asynchronous callbacks + * + * @return a future with a boolean for success or failure + */ + CompletableFuture flushBlobRange(byte[] beginKey, byte[] endKey, boolean compact, long version, Executor e); + /** * Runs {@link #getId()} on the default executor. * diff --git a/fdbcli/BlobRangeCommand.actor.cpp b/fdbcli/BlobRangeCommand.actor.cpp index 366013a6de..cf03695478 100644 --- a/fdbcli/BlobRangeCommand.actor.cpp +++ b/fdbcli/BlobRangeCommand.actor.cpp @@ -24,7 +24,6 @@ #include "fdbclient/IClientApi.h" #include "fdbclient/ManagementAPI.actor.h" #include "fdbclient/NativeAPI.actor.h" -#include "fdbclient/BlobGranuleRequest.actor.h" #include "flow/Arena.h" #include "flow/FastRef.h" @@ -90,17 +89,16 @@ ACTOR Future doBlobCheck(Database db, Key startKey, Key endKey, Optional doBlobFlush(Database db, Key startKey, Key endKey, Optional version, bool compact) { - // TODO make DB function? - state Version flushVersion; - if (version.present()) { - flushVersion = version.get(); - } else { - wait(store(flushVersion, getLatestReadVersion(db))); - } + state double elapsed = -timer_monotonic(); + state KeyRange keyRange(KeyRangeRef(startKey, endKey)); + bool result = wait(db->flushBlobRange(keyRange, compact, version)); + elapsed += timer_monotonic(); - KeyRange range(KeyRangeRef(startKey, endKey)); - FlushGranuleRequest req(-1, range, flushVersion, compact); - wait(success(doBlobGranuleRequests(db, range, req, &BlobWorkerInterface::flushGranuleRequest))); + fmt::print("Blob Flush [{0} - {1}) {2} in {3:.6f} seconds\n", + startKey.printable(), + endKey.printable(), + result ? "succeeded" : "failed", + elapsed); return Void(); } diff --git a/fdbcli/IdempotencyIdsCommand.actor.cpp b/fdbcli/IdempotencyIdsCommand.actor.cpp new file mode 100644 index 0000000000..22f653c01c --- /dev/null +++ b/fdbcli/IdempotencyIdsCommand.actor.cpp @@ -0,0 +1,83 @@ +/* + * IdempotencyIdsCommand.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.actor.h" +#include "fdbclient/IdempotencyId.actor.h" +#include "fdbclient/JsonBuilder.h" +#include "fdbclient/json_spirit/json_spirit_reader_template.h" +#include "flow/actorcompiler.h" // This must be the last include + +namespace { + +Optional parseAgeValue(StringRef token) { + try { + return std::stod(token.toString()); + } catch (...) { + return {}; + } +} + +} // namespace + +namespace fdb_cli { + +ACTOR Future idempotencyIdsCommandActor(Database db, std::vector tokens) { + if (tokens.size() < 2 || tokens.size() > 3) { + printUsage(tokens[0]); + return false; + } else { + auto const action = tokens[1]; + if (action == "status"_sr) { + if (tokens.size() != 2) { + printUsage(tokens[0]); + return false; + } + JsonBuilderObject status = wait(getIdmpKeyStatus(db)); + fmt::print("{}\n", status.getJson()); + return true; + } else if (action == "clear"_sr) { + if (tokens.size() != 3) { + printUsage(tokens[0]); + return false; + } + auto const age = parseAgeValue(tokens[2]); + if (!age.present()) { + printUsage(tokens[0]); + return false; + } + wait(cleanIdempotencyIds(db, age.get())); + fmt::print("Successfully cleared idempotency IDs.\n"); + return true; + } else { + printUsage(tokens[0]); + return false; + } + } +} + +CommandFactory idempotencyIdsCommandFactory( + "idempotencyids", + CommandHelp( + "idempotencyids [status | clear ]", + "View status of idempotency ids, or reclaim space used by idempotency ids older than the given age", + "View status of idempotency ids currently in the cluster, or reclaim space by clearing all the idempotency ids " + "older than min_age_seconds (which will expire all transaction versions older than this age)\n")); + +} // namespace fdb_cli diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 6c29fa9e50..e02cc20c28 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -2137,6 +2137,14 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise, Reference tssqCommandActor(Reference db, std::vector versionEpochCommandActor(Reference db, Database cx, std::vector tokens); // targetversion command ACTOR Future targetVersionCommandActor(Reference db, std::vector tokens); +// idempotencyids command +ACTOR Future idempotencyIdsCommandActor(Database cx, std::vector tokens); } // namespace fdb_cli diff --git a/fdbcli/tests/fdbcli_tests.py b/fdbcli/tests/fdbcli_tests.py index 4caa3e0625..30e9d74b22 100755 --- a/fdbcli/tests/fdbcli_tests.py +++ b/fdbcli/tests/fdbcli_tests.py @@ -19,6 +19,7 @@ def enable_logging(level=logging.DEBUG): Args: level (logging., optional): logging level for the decorated function. Defaults to logging.ERROR. """ + def func_decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): @@ -27,14 +28,18 @@ def enable_logging(level=logging.DEBUG): logger.setLevel(level) # set logging format handler = logging.StreamHandler() - handler_format = logging.Formatter('[%(asctime)s] - %(filename)s:%(lineno)d - %(levelname)s - %(name)s - %(message)s') + handler_format = logging.Formatter( + "[%(asctime)s] - %(filename)s:%(lineno)d - %(levelname)s - %(name)s - %(message)s" + ) handler.setFormatter(handler_format) handler.setLevel(level) logger.addHandler(handler) # pass the logger to the decorated function result = func(logger, *args, **kwargs) return result + return wrapper + return func_decorator @@ -44,13 +49,15 @@ def run_fdbcli_command(*args): Returns: string: Console output from fdbcli """ - commands = command_template + ["{}".format(' '.join(args))] + commands = command_template + ["{}".format(" ".join(args))] try: # if the fdbcli command is stuck for more than 20 seconds, the database is definitely unavailable - process = subprocess.run(commands, stdout=subprocess.PIPE, env=fdbcli_env, timeout=20) - return process.stdout.decode('utf-8').strip() + process = subprocess.run( + commands, stdout=subprocess.PIPE, env=fdbcli_env, timeout=20 + ) + return process.stdout.decode("utf-8").strip() except subprocess.TimeoutExpired: - raise Exception('The fdbcli command is stuck, database is unavailable') + raise Exception("The fdbcli command is stuck, database is unavailable") def run_fdbcli_command_and_get_error(*args): @@ -59,29 +66,35 @@ def run_fdbcli_command_and_get_error(*args): Returns: string: Stderr output from fdbcli """ - commands = command_template + ["{}".format(' '.join(args))] - return subprocess.run(commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env).stderr.decode('utf-8').strip() + commands = command_template + ["{}".format(" ".join(args))] + return ( + subprocess.run( + commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env + ) + .stderr.decode("utf-8") + .strip() + ) @enable_logging() def advanceversion(logger): # get current read version - version1 = int(run_fdbcli_command('getversion')) + version1 = int(run_fdbcli_command("getversion")) logger.debug("Read version: {}".format(version1)) # advance version to a much larger value compared to the current version version2 = version1 * 10000 logger.debug("Advanced to version: " + str(version2)) - run_fdbcli_command('advanceversion', str(version2)) + run_fdbcli_command("advanceversion", str(version2)) # after running the advanceversion command, # check the read version is advanced to the specified value - version3 = int(run_fdbcli_command('getversion')) + version3 = int(run_fdbcli_command("getversion")) logger.debug("Read version: {}".format(version3)) assert version3 >= version2 # advance version to a smaller value compared to the current version # this should be a no-op - run_fdbcli_command('advanceversion', str(version1)) + run_fdbcli_command("advanceversion", str(version1)) # get the current version to make sure the version did not decrease - version4 = int(run_fdbcli_command('getversion')) + version4 = int(run_fdbcli_command("getversion")) logger.debug("Read version: {}".format(version4)) assert version4 >= version3 @@ -89,202 +102,231 @@ def advanceversion(logger): @enable_logging() def maintenance(logger): # expected fdbcli output when running 'maintenance' while there's no ongoing maintenance - no_maintenance_output = 'No ongoing maintenance.' - output1 = run_fdbcli_command('maintenance') + no_maintenance_output = "No ongoing maintenance." + output1 = run_fdbcli_command("maintenance") assert output1 == no_maintenance_output # set maintenance on a fake zone id for 10 seconds - run_fdbcli_command('maintenance', 'on', 'fake_zone_id', '10') + run_fdbcli_command("maintenance", "on", "fake_zone_id", "10") # show current maintenance status - output2 = run_fdbcli_command('maintenance') + output2 = run_fdbcli_command("maintenance") logger.debug("Maintenance status: " + output2) - items = output2.split(' ') + items = output2.split(" ") # make sure this specific zone id is under maintenance - assert 'fake_zone_id' in items + assert "fake_zone_id" in items logger.debug("Remaining time(seconds): " + items[-2]) assert 0 < int(items[-2]) < 10 # turn off maintenance - run_fdbcli_command('maintenance', 'off') + run_fdbcli_command("maintenance", "off") # check maintenance status - output3 = run_fdbcli_command('maintenance') + output3 = run_fdbcli_command("maintenance") assert output3 == no_maintenance_output + @enable_logging() def quota(logger): # Should be a noop - command = 'quota clear green' + command = "quota clear green" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == 'Successfully cleared quota.' + logger.debug(command + " : " + output) + assert output == "Successfully cleared quota." - command = 'quota get green total_throughput' + command = "quota get green total_throughput" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == '' + logger.debug(command + " : " + output) + assert output == "" # Ignored update - command = 'quota set red total_throughput 49152' + command = "quota set red total_throughput 49152" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == 'Successfully updated quota.' + logger.debug(command + " : " + output) + assert output == "Successfully updated quota." - command = 'quota set green total_throughput 32768' + command = "quota set green total_throughput 32768" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == 'Successfully updated quota.' + logger.debug(command + " : " + output) + assert output == "Successfully updated quota." - command = 'quota set green reserved_throughput 16384' + command = "quota set green reserved_throughput 16384" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == 'Successfully updated quota.' + logger.debug(command + " : " + output) + assert output == "Successfully updated quota." - command = 'quota set green storage 98765' + command = "quota set green storage 98765" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == 'Successfully updated quota.' + logger.debug(command + " : " + output) + assert output == "Successfully updated quota." - command = 'quota get green total_throughput' + command = "quota get green total_throughput" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == '32768' + logger.debug(command + " : " + output) + assert output == "32768" - command = 'quota get green reserved_throughput' + command = "quota get green reserved_throughput" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == '16384' + logger.debug(command + " : " + output) + assert output == "16384" - command = 'quota get green storage' + command = "quota get green storage" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == '98765' + logger.debug(command + " : " + output) + assert output == "98765" - command = 'quota clear green' + command = "quota clear green" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == 'Successfully cleared quota.' + logger.debug(command + " : " + output) + assert output == "Successfully cleared quota." - command = 'quota get green total_throughput' + command = "quota get green total_throughput" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == '' + logger.debug(command + " : " + output) + assert output == "" - command = 'quota get green storage' + command = "quota get green storage" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) - assert output == '' + logger.debug(command + " : " + output) + assert output == "" # Too few arguments, should log help message - command = 'quota get green' + command = "quota get green" output = run_fdbcli_command(command) - logger.debug(command + ' : ' + output) + logger.debug(command + " : " + output) + @enable_logging() def setclass(logger): # get all processes' network addresses - output1 = run_fdbcli_command('setclass') + output1 = run_fdbcli_command("setclass") logger.debug(output1) # except the first line, each line is one process - process_types = output1.split('\n')[1:] + process_types = output1.split("\n")[1:] assert len(process_types) == args.process_number addresses = [] for line in process_types: - assert '127.0.0.1' in line + assert "127.0.0.1" in line # check class type - assert 'unset' in line + assert "unset" in line # check class source - assert 'command_line' in line + assert "command_line" in line # check process' network address - network_address = ':'.join(line.split(':')[:2]) + network_address = ":".join(line.split(":")[:2]) logger.debug("Network address: {}".format(network_address)) addresses.append(network_address) random_address = random.choice(addresses) logger.debug("Randomly selected address: {}".format(random_address)) # set class to a random valid type - class_types = ['storage', 'transaction', 'resolution', - 'commit_proxy', 'grv_proxy', 'master', 'stateless', 'log', - 'router', 'cluster_controller', 'fast_restore', 'data_distributor', - 'coordinator', 'ratekeeper', 'storage_cache', 'backup' - ] + class_types = [ + "storage", + "transaction", + "resolution", + "commit_proxy", + "grv_proxy", + "master", + "stateless", + "log", + "router", + "cluster_controller", + "fast_restore", + "data_distributor", + "coordinator", + "ratekeeper", + "storage_cache", + "backup", + ] random_class_type = random.choice(class_types) logger.debug("Change to type: {}".format(random_class_type)) - run_fdbcli_command('setclass', random_address, random_class_type) + run_fdbcli_command("setclass", random_address, random_class_type) # check the set successful - output2 = run_fdbcli_command('setclass') + output2 = run_fdbcli_command("setclass") logger.debug(output2) assert random_address in output2 - process_types = output2.split('\n')[1:] + process_types = output2.split("\n")[1:] # check process' network address for line in process_types: if random_address in line: # check class type changed to the specified value assert random_class_type in line # check class source - assert 'set_class' in line + assert "set_class" in line # set back to unset - run_fdbcli_command('setclass', random_address, 'unset') + run_fdbcli_command("setclass", random_address, "unset") # Attempt to set an invalid address and check error message - output3 = run_fdbcli_command('setclass', '0.0.0.0:4000', 'storage') + output3 = run_fdbcli_command("setclass", "0.0.0.0:4000", "storage") logger.debug(output3) - assert 'No matching addresses found' in output3 + assert "No matching addresses found" in output3 # Verify setclass did not execute - output4 = run_fdbcli_command('setclass') + output4 = run_fdbcli_command("setclass") logger.debug(output4) # except the first line, each line is one process - process_types = output4.split('\n')[1:] + process_types = output4.split("\n")[1:] assert len(process_types) == args.process_number addresses = [] for line in process_types: - assert '127.0.0.1' in line + assert "127.0.0.1" in line # check class type - assert 'unset' in line + assert "unset" in line # check class source - assert 'command_line' in line or 'set_class' in line + assert "command_line" in line or "set_class" in line @enable_logging() def lockAndUnlock(logger): # lock an unlocked database, should be successful - output1 = run_fdbcli_command('lock') + output1 = run_fdbcli_command("lock") # UID is 32 bytes - lines = output1.split('\n') + lines = output1.split("\n") lock_uid = lines[0][-32:] - assert lines[1] == 'Database locked.' + assert lines[1] == "Database locked." logger.debug("UID: {}".format(lock_uid)) - assert get_value_from_status_json(True, 'cluster', 'database_lock_state', 'locked') + assert get_value_from_status_json(True, "cluster", "database_lock_state", "locked") # lock a locked database, should get the error code 1038 output2 = run_fdbcli_command_and_get_error("lock") - assert output2 == 'ERROR: Database is locked (1038)' + assert output2 == "ERROR: Database is locked (1038)" # unlock the database - process = subprocess.Popen(command_template + ['unlock ' + lock_uid], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env) - line1 = process.stdout.readline() + process = subprocess.Popen( + command_template + ["unlock " + lock_uid], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + env=fdbcli_env, + ) + process.stdout.readline() # The randome passphrease we need to confirm to proceed the unlocking line2 = process.stdout.readline() logger.debug("Random passphrase: {}".format(line2)) output3, err = process.communicate(input=line2) # No error and unlock was successful assert err is None - assert output3.decode('utf-8').strip() == 'Database unlocked.' - assert not get_value_from_status_json(True, 'cluster', 'database_lock_state', 'locked') + assert output3.decode("utf-8").strip() == "Database unlocked." + assert not get_value_from_status_json( + True, "cluster", "database_lock_state", "locked" + ) @enable_logging() def kill(logger): - output1 = run_fdbcli_command('kill') - lines = output1.split('\n') + output1 = run_fdbcli_command("kill") + lines = output1.split("\n") assert len(lines) == 2 - assert lines[1].startswith('127.0.0.1:') + assert lines[1].startswith("127.0.0.1:") address = lines[1] logger.debug("Address: {}".format(address)) - old_generation = get_value_from_status_json(False, 'cluster', 'generation') + old_generation = get_value_from_status_json(False, "cluster", "generation") # This is currently an issue with fdbcli, # where you need to first run 'kill' to initialize processes' list # and then specify the certain process to kill - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env) + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + env=fdbcli_env, + ) # - output2, err = process.communicate(input='kill; kill {}; sleep 1\n'.format(address).encode()) + output2, err = process.communicate( + input="kill; kill {}; sleep 1\n".format(address).encode() + ) logger.debug(output2) # wait for a second for the cluster recovery time.sleep(1) - new_generation = get_value_from_status_json(True, 'cluster', 'generation') + new_generation = get_value_from_status_json(True, "cluster", "generation") logger.debug("Old: {}, New: {}".format(old_generation, new_generation)) assert new_generation > old_generation @@ -292,17 +334,24 @@ def kill(logger): @enable_logging() def killall(logger): # test is designed to make sure 'kill all' sends all requests simultaneously - old_generation = get_value_from_status_json(False, 'cluster', 'generation') + old_generation = get_value_from_status_json(False, "cluster", "generation") # This is currently an issue with fdbcli, # where you need to first run 'kill' to initialize processes' list # and then specify the certain process to kill - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env) - output, error = process.communicate(input='kill; kill all; sleep 1\n'.encode()) + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + env=fdbcli_env, + ) + output, error = process.communicate(input="kill; kill all; sleep 1\n".encode()) logger.debug(output) # wait for a second for the cluster recovery time.sleep(1) - new_generation = get_value_from_status_json(True, 'cluster', 'generation') - logger.debug("Old generation: {}, New generation: {}".format(old_generation, new_generation)) + new_generation = get_value_from_status_json(True, "cluster", "generation") + logger.debug( + "Old generation: {}, New generation: {}".format(old_generation, new_generation) + ) # Make sure the kill is not happening sequentially # Pre: each recovery will increase the generated number by 2 # Relax the condition to allow one additional recovery happening when we fetched the old generation @@ -314,77 +363,108 @@ def suspend(logger): if not shutil.which("pidof"): logger.debug("Skipping suspend test. Pidof not available") return - output1 = run_fdbcli_command('suspend') - lines = output1.split('\n') + output1 = run_fdbcli_command("suspend") + lines = output1.split("\n") assert len(lines) == 2 - assert lines[1].startswith('127.0.0.1:') + assert lines[1].startswith("127.0.0.1:") address = lines[1] logger.debug("Address: {}".format(address)) - db_available = get_value_from_status_json(False, 'client', 'database_status', 'available') + db_available = get_value_from_status_json( + False, "client", "database_status", "available" + ) assert db_available # use pgrep to get the pid of the fdb process - pinfos = subprocess.check_output(['pgrep', '-a', 'fdbserver']).decode().strip().split('\n') - port = address.split(':')[1] + pinfos = ( + subprocess.check_output(["pgrep", "-a", "fdbserver"]) + .decode() + .strip() + .split("\n") + ) + port = address.split(":")[1] logger.debug("Port: {}".format(port)) # use the port number to find the exact fdb process we are connecting to # child process like fdbserver -r flowprocess does not provide `datadir` in the command line - pinfo = list(filter(lambda x: port in x and 'datadir' in x, pinfos)) + pinfo = list(filter(lambda x: port in x and "datadir" in x, pinfos)) assert len(pinfo) == 1 - pid = pinfo[0].split(' ')[0] + pid = pinfo[0].split(" ")[0] logger.debug("Pid: {}".format(pid)) - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env) + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + env=fdbcli_env, + ) # suspend the process for enough long time - output2, err = process.communicate(input='suspend; suspend 3600 {}; sleep 1\n'.format(address).encode()) + output2, err = process.communicate( + input="suspend; suspend 3600 {}; sleep 1\n".format(address).encode() + ) # the cluster should be unavailable after the only process being suspended - assert not get_value_from_status_json(False, 'client', 'database_status', 'available') + assert not get_value_from_status_json( + False, "client", "database_status", "available" + ) # check the process pid still exists - pids = subprocess.check_output(['pidof', 'fdbserver']).decode().strip() + pids = subprocess.check_output(["pidof", "fdbserver"]).decode().strip() logger.debug("PIDs: {}".format(pids)) assert pid in pids # kill the process by pid - kill_output = subprocess.check_output(['kill', pid]).decode().strip() + kill_output = subprocess.check_output(["kill", pid]).decode().strip() logger.debug("Kill result: {}".format(kill_output)) # The process should come back after a few time duration = 0 # seconds we already wait - while not get_value_from_status_json(False, 'client', 'database_status', 'available') and duration < 60: + while ( + not get_value_from_status_json(False, "client", "database_status", "available") + and duration < 60 + ): logger.debug("Sleep for 1 second to wait cluster recovery") time.sleep(1) duration += 1 # at most after 60 seconds, the cluster should be available - assert get_value_from_status_json(False, 'client', 'database_status', 'available') + assert get_value_from_status_json(False, "client", "database_status", "available") @enable_logging() def versionepoch(logger): - version1 = run_fdbcli_command('versionepoch') + version1 = run_fdbcli_command("versionepoch") assert version1 == "Version epoch is unset" - version2 = run_fdbcli_command('versionepoch get') + version2 = run_fdbcli_command("versionepoch get") assert version2 == "Version epoch is unset" - version3 = run_fdbcli_command('versionepoch commit') - assert version3 == "Must set the version epoch before committing it (see `versionepoch enable`)" - version4 = run_fdbcli_command('versionepoch enable') - assert version4 == "Version epoch enabled. Run `versionepoch commit` to irreversibly jump to the target version" - version5 = run_fdbcli_command('versionepoch get') + version3 = run_fdbcli_command("versionepoch commit") + assert ( + version3 + == "Must set the version epoch before committing it (see `versionepoch enable`)" + ) + version4 = run_fdbcli_command("versionepoch enable") + assert ( + version4 + == "Version epoch enabled. Run `versionepoch commit` to irreversibly jump to the target version" + ) + version5 = run_fdbcli_command("versionepoch get") assert version5 == "Current version epoch is 0" - version6 = run_fdbcli_command('versionepoch set 10') - assert version6 == "Version epoch enabled. Run `versionepoch commit` to irreversibly jump to the target version" - version7 = run_fdbcli_command('versionepoch get') + version6 = run_fdbcli_command("versionepoch set 10") + assert ( + version6 + == "Version epoch enabled. Run `versionepoch commit` to irreversibly jump to the target version" + ) + version7 = run_fdbcli_command("versionepoch get") assert version7 == "Current version epoch is 10" - run_fdbcli_command('versionepoch disable') - version8 = run_fdbcli_command('versionepoch get') + run_fdbcli_command("versionepoch disable") + version8 = run_fdbcli_command("versionepoch get") assert version8 == "Version epoch is unset" - version9 = run_fdbcli_command('versionepoch enable') - assert version9 == "Version epoch enabled. Run `versionepoch commit` to irreversibly jump to the target version" - version10 = run_fdbcli_command('versionepoch get') + version9 = run_fdbcli_command("versionepoch enable") + assert ( + version9 + == "Version epoch enabled. Run `versionepoch commit` to irreversibly jump to the target version" + ) + version10 = run_fdbcli_command("versionepoch get") assert version10 == "Current version epoch is 0" - version11 = run_fdbcli_command('versionepoch commit') + version11 = run_fdbcli_command("versionepoch commit") assert version11.startswith("Current read version is ") def get_value_from_status_json(retry, *args): while True: - result = json.loads(run_fdbcli_command('status', 'json')) - if result['client']['database_status']['available'] or not retry: + result = json.loads(run_fdbcli_command("status", "json")) + if result["client"]["database_status"]["available"] or not retry: break for arg in args: assert arg in result @@ -395,15 +475,15 @@ def get_value_from_status_json(retry, *args): @enable_logging() def consistencycheck(logger): - consistency_check_on_output = 'ConsistencyCheck is on' - consistency_check_off_output = 'ConsistencyCheck is off' - output1 = run_fdbcli_command('consistencycheck') + consistency_check_on_output = "ConsistencyCheck is on" + consistency_check_off_output = "ConsistencyCheck is off" + output1 = run_fdbcli_command("consistencycheck") assert output1 == consistency_check_on_output - run_fdbcli_command('consistencycheck', 'off') - output2 = run_fdbcli_command('consistencycheck') + run_fdbcli_command("consistencycheck", "off") + output2 = run_fdbcli_command("consistencycheck") assert output2 == consistency_check_off_output - run_fdbcli_command('consistencycheck', 'on') - output3 = run_fdbcli_command('consistencycheck') + run_fdbcli_command("consistencycheck", "on") + output3 = run_fdbcli_command("consistencycheck") assert output3 == consistency_check_on_output @@ -413,147 +493,192 @@ def knobmanagement(logger): # must use begin/commit to avoid prompt for description # Incorrect arguments - output = run_fdbcli_command('setknob') + output = run_fdbcli_command("setknob") assert output == "Usage: setknob [CONFIG_CLASS]" - output = run_fdbcli_command('setknob', 'min_trace_severity') + output = run_fdbcli_command("setknob", "min_trace_severity") assert output == "Usage: setknob [CONFIG_CLASS]" - output = run_fdbcli_command('getknob') + output = run_fdbcli_command("getknob") assert output == "Usage: getknob [CONFIG_CLASS]" logger.debug("incorrect args passed") # Invalid knob name - err = run_fdbcli_command_and_get_error('begin; setknob dummy_knob 20; commit \"fdbcli change\";') + err = run_fdbcli_command_and_get_error( + 'begin; setknob dummy_knob 20; commit "fdbcli change";' + ) logger.debug("err is: {}".format(err)) assert len(err) > 0 logger.debug("invalid knob name passed") # Invalid type for knob - err = run_fdbcli_command_and_get_error('begin; setknob min_trace_severity dummy-text; commit \"fdbcli change\";') + err = run_fdbcli_command_and_get_error( + 'begin; setknob min_trace_severity dummy-text; commit "fdbcli change";' + ) logger.debug("err is: {}".format(err)) assert len(err) > 0 logger.debug("invalid knob type passed") - # Verifying we can't do a normal set, clear, get, getrange, clearrange + # Verifying we can't do a normal set, clear, get, getrange, clearrange # with a setknob - err = run_fdbcli_command_and_get_error('writemode on; begin; set foo bar; setknob max_metric_size 1000; commit;') + err = run_fdbcli_command_and_get_error( + "writemode on; begin; set foo bar; setknob max_metric_size 1000; commit;" + ) logger.debug("err is: {}".format(err)) assert len(err) > 0 - err = run_fdbcli_command_and_get_error('writemode on; begin; clear foo; setknob max_metric_size 1000; commit') + err = run_fdbcli_command_and_get_error( + "writemode on; begin; clear foo; setknob max_metric_size 1000; commit" + ) logger.debug("err is: {}".format(err)) - assert len(err) > 0 + assert len(err) > 0 # Various setknobs and verified by getknob - output = run_fdbcli_command('begin; setknob min_trace_severity 30; setknob max_metric_size 1000; \ + output = run_fdbcli_command( + 'begin; setknob min_trace_severity 30; setknob max_metric_size 1000; \ setknob tracing_udp_listener_addr 192.168.0.1; \ setknob tracing_sample_rate 0.3; \ - commit \"This is an fdbcli test for knobs\";') + commit "This is an fdbcli test for knobs";' + ) assert "Committed" in output - output = run_fdbcli_command('getknob', 'min_trace_severity') + output = run_fdbcli_command("getknob", "min_trace_severity") assert r"`min_trace_severity' is `30'" == output - output = run_fdbcli_command('getknob', 'max_metric_size') + output = run_fdbcli_command("getknob", "max_metric_size") assert r"`max_metric_size' is `1000'" == output - output = run_fdbcli_command('getknob', 'tracing_udp_listener_addr') + output = run_fdbcli_command("getknob", "tracing_udp_listener_addr") assert r"`tracing_udp_listener_addr' is `'192.168.0.1''" == output - output = run_fdbcli_command('getknob', 'tracing_sample_rate') + output = run_fdbcli_command("getknob", "tracing_sample_rate") assert r"`tracing_sample_rate' is `0.300000'" == output + @enable_logging() def cache_range(logger): # this command is currently experimental # just test we can set and clear the cached range - run_fdbcli_command('cache_range', 'set', 'a', 'b') - run_fdbcli_command('cache_range', 'clear', 'a', 'b') + run_fdbcli_command("cache_range", "set", "a", "b") + run_fdbcli_command("cache_range", "clear", "a", "b") @enable_logging() def datadistribution(logger): - output1 = run_fdbcli_command('datadistribution', 'off') - assert output1 == 'Data distribution is turned off.' - output2 = run_fdbcli_command('datadistribution', 'on') - assert output2 == 'Data distribution is turned on.' - output3 = run_fdbcli_command('datadistribution', 'disable', 'ssfailure') - assert output3 == 'Data distribution is disabled for storage server failures.' + output1 = run_fdbcli_command("datadistribution", "off") + assert output1 == "Data distribution is turned off." + output2 = run_fdbcli_command("datadistribution", "on") + assert output2 == "Data distribution is turned on." + output3 = run_fdbcli_command("datadistribution", "disable", "ssfailure") + assert output3 == "Data distribution is disabled for storage server failures." # While we disable ssfailure, maintenance should fail - error_msg = run_fdbcli_command_and_get_error('maintenance', 'on', 'fake_zone_id', '1') - assert error_msg == "ERROR: Maintenance mode cannot be used while data distribution is disabled for storage server failures. Use 'datadistribution on' to reenable data distribution." - output4 = run_fdbcli_command('datadistribution', 'enable', 'ssfailure') - assert output4 == 'Data distribution is enabled for storage server failures.' - output5 = run_fdbcli_command('datadistribution', 'disable', 'rebalance') - assert output5 == 'Data distribution is disabled for rebalance.' - output6 = run_fdbcli_command('datadistribution', 'enable', 'rebalance') - assert output6 == 'Data distribution is enabled for rebalance.' + error_msg = run_fdbcli_command_and_get_error( + "maintenance", "on", "fake_zone_id", "1" + ) + assert ( + error_msg + == "ERROR: Maintenance mode cannot be used while data distribution is disabled for storage server failures. Use 'datadistribution on' to reenable data distribution." + ) + output4 = run_fdbcli_command("datadistribution", "enable", "ssfailure") + assert output4 == "Data distribution is enabled for storage server failures." + output5 = run_fdbcli_command("datadistribution", "disable", "rebalance") + assert output5 == "Data distribution is disabled for rebalance." + output6 = run_fdbcli_command("datadistribution", "enable", "rebalance") + assert output6 == "Data distribution is enabled for rebalance." time.sleep(1) @enable_logging() def transaction(logger): """This test will cover the transaction related fdbcli commands. - In particular, - 'begin', 'rollback', 'option' - 'getversion', 'get', 'getrange', 'clear', 'clearrange', 'set', 'commit' + In particular, + 'begin', 'rollback', 'option' + 'getversion', 'get', 'getrange', 'clear', 'clearrange', 'set', 'commit' """ - err1 = run_fdbcli_command_and_get_error('set', 'key', 'value') - assert err1 == 'ERROR: writemode must be enabled to set or clear keys in the database.' - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env) - transaction_flow = ['writemode on', 'begin', 'getversion', 'set key value', 'get key', 'commit'] - output1, _ = process.communicate(input='\n'.join(transaction_flow).encode()) + err1 = run_fdbcli_command_and_get_error("set", "key", "value") + assert ( + err1 == "ERROR: writemode must be enabled to set or clear keys in the database." + ) + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + env=fdbcli_env, + ) + transaction_flow = [ + "writemode on", + "begin", + "getversion", + "set key value", + "get key", + "commit", + ] + output1, _ = process.communicate(input="\n".join(transaction_flow).encode()) # split the output into lines - lines = list(filter(len, output1.decode().split('\n')))[-4:] - assert lines[0] == 'Transaction started' + lines = list(filter(len, output1.decode().split("\n")))[-4:] + assert lines[0] == "Transaction started" read_version = int(lines[1]) logger.debug("Read version {}".format(read_version)) # line[1] is the printed read version assert lines[2] == "`key' is `value'" - assert lines[3].startswith('Committed (') and lines[3].endswith(')') + assert lines[3].startswith("Committed (") and lines[3].endswith(")") # validate commit version is larger than the read version - commit_verion = int(lines[3][len('Committed ('):-1]) + commit_verion = int(lines[3][len("Committed (") : -1]) logger.debug("Commit version: {}".format(commit_verion)) assert commit_verion >= read_version # check the transaction is committed - output2 = run_fdbcli_command('get', 'key') + output2 = run_fdbcli_command("get", "key") assert output2 == "`key' is `value'" # test rollback and read-your-write behavior - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env) + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + env=fdbcli_env, + ) transaction_flow = [ - 'writemode on', 'begin', 'getrange a z', - 'clear key', 'get key', + "writemode on", + "begin", + "getrange a z", + "clear key", + "get key", # 'option on READ_YOUR_WRITES_DISABLE', 'get key', - 'rollback' + "rollback", ] - output3, _ = process.communicate(input='\n'.join(transaction_flow).encode()) - lines = list(filter(len, output3.decode().split('\n')))[-5:] + output3, _ = process.communicate(input="\n".join(transaction_flow).encode()) + lines = list(filter(len, output3.decode().split("\n")))[-5:] # lines[0] == "Transaction started" and lines[1] == 'Range limited to 25 keys' assert lines[2] == "`key' is `value'" assert lines[3] == "`key': not found" assert lines[4] == "Transaction rolled back" # make sure the rollback works - output4 = run_fdbcli_command('get', 'key') + output4 = run_fdbcli_command("get", "key") assert output4 == "`key' is `value'" # test read_your_write_disable option and clear the inserted key - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env) + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + env=fdbcli_env, + ) transaction_flow = [ - 'writemode on', 'begin', - 'option on READ_YOUR_WRITES_DISABLE', - 'clear key', 'get key', - 'commit' + "writemode on", + "begin", + "option on READ_YOUR_WRITES_DISABLE", + "clear key", + "get key", + "commit", ] - output6, _ = process.communicate(input='\n'.join(transaction_flow).encode()) - lines = list(filter(len, output6.decode().split('\n')))[-4:] - assert lines[1] == 'Option enabled for current transaction' + output6, _ = process.communicate(input="\n".join(transaction_flow).encode()) + lines = list(filter(len, output6.decode().split("\n")))[-4:] + assert lines[1] == "Option enabled for current transaction" # the get key should still return the value even we clear it in the transaction assert lines[2] == "`key' is `value'" # Check the transaction is committed - output7 = run_fdbcli_command('get', 'key') + output7 = run_fdbcli_command("get", "key") assert output7 == "`key': not found" def get_fdb_process_addresses(logger): # get all processes' network addresses - output = run_fdbcli_command('kill') + output = run_fdbcli_command("kill") logger.debug(output) # except the first line, each line is one process - addresses = output.split('\n')[1:] + addresses = output.split("\n")[1:] assert len(addresses) == args.process_number return addresses @@ -561,28 +686,39 @@ def get_fdb_process_addresses(logger): @enable_logging(logging.DEBUG) def coordinators(logger): # we should only have one coordinator for now - output1 = run_fdbcli_command('coordinators') - assert len(output1.split('\n')) > 2 - cluster_description = output1.split('\n')[0].split(': ')[-1] + output1 = run_fdbcli_command("coordinators") + assert len(output1.split("\n")) > 2 + cluster_description = output1.split("\n")[0].split(": ")[-1] logger.debug("Cluster description: {}".format(cluster_description)) - coordinators = output1.split('\n')[1].split(': ')[-1] + coordinators = output1.split("\n")[1].split(": ")[-1] # verify the coordinator - coordinator_list = get_value_from_status_json(True, 'client', 'coordinators', 'coordinators') + coordinator_list = get_value_from_status_json( + True, "client", "coordinators", "coordinators" + ) assert len(coordinator_list) == 1 - assert coordinator_list[0]['address'] == coordinators + assert coordinator_list[0]["address"] == coordinators # verify the cluster description - assert get_value_from_status_json(True, 'cluster', 'connection_string').startswith('{}:'.format(cluster_description)) + assert get_value_from_status_json(True, "cluster", "connection_string").startswith( + "{}:".format(cluster_description) + ) addresses = get_fdb_process_addresses(logger) # set all 5 processes as coordinators and update the cluster description - new_cluster_description = 'a_simple_description' - run_fdbcli_command('coordinators', *addresses, 'description={}'.format(new_cluster_description)) + new_cluster_description = "a_simple_description" + run_fdbcli_command( + "coordinators", *addresses, "description={}".format(new_cluster_description) + ) # verify now we have 5 coordinators and the description is updated - output2 = run_fdbcli_command('coordinators') - assert output2.split('\n')[0].split(': ')[-1] == new_cluster_description - assert output2.split('\n')[1] == 'Cluster coordinators ({}): {}'.format(args.process_number, ','.join(addresses)) + output2 = run_fdbcli_command("coordinators") + assert output2.split("\n")[0].split(": ")[-1] == new_cluster_description + assert output2.split("\n")[1] == "Cluster coordinators ({}): {}".format( + args.process_number, ",".join(addresses) + ) # auto change should go back to 1 coordinator - run_fdbcli_command('coordinators', 'auto') - assert len(get_value_from_status_json(True, 'client', 'coordinators', 'coordinators')) == 1 + run_fdbcli_command("coordinators", "auto") + assert ( + len(get_value_from_status_json(True, "client", "coordinators", "coordinators")) + == 1 + ) wait_for_database_available(logger) @@ -590,10 +726,12 @@ def coordinators(logger): def exclude(logger): # get all processes' network addresses addresses = get_fdb_process_addresses(logger) - logger.debug("Cluster processes: {}".format(' '.join(addresses))) + logger.debug("Cluster processes: {}".format(" ".join(addresses))) # There should be no excluded process for now - no_excluded_process_output = 'There are currently no servers or localities excluded from the database.' - output1 = run_fdbcli_command('exclude') + no_excluded_process_output = ( + "There are currently no servers or localities excluded from the database." + ) + output1 = run_fdbcli_command("exclude") assert no_excluded_process_output in output1 # randomly pick one and exclude the process excluded_address = random.choice(addresses) @@ -604,16 +742,25 @@ def exclude(logger): while True: logger.debug("Excluding process: {}".format(excluded_address)) if force: - error_message = run_fdbcli_command_and_get_error('exclude', 'FORCE', excluded_address) + error_message = run_fdbcli_command_and_get_error( + "exclude", "FORCE", excluded_address + ) else: - error_message = run_fdbcli_command_and_get_error('exclude', excluded_address) - if error_message == 'WARNING: {} is a coordinator!'.format(excluded_address): + error_message = run_fdbcli_command_and_get_error( + "exclude", excluded_address + ) + if error_message == "WARNING: {} is a coordinator!".format(excluded_address): # exclude coordinator will print the warning, verify the randomly selected process is the coordinator - coordinator_list = get_value_from_status_json(True, 'client', 'coordinators', 'coordinators') + coordinator_list = get_value_from_status_json( + True, "client", "coordinators", "coordinators" + ) assert len(coordinator_list) == 1 - assert coordinator_list[0]['address'] == excluded_address + assert coordinator_list[0]["address"] == excluded_address break - elif 'ERROR: This exclude may cause the total free space in the cluster to drop below 10%.' in error_message: + elif ( + "ERROR: This exclude may cause the total free space in the cluster to drop below 10%." + in error_message + ): # exclude the process may cause the free space not enough # use FORCE option to ignore it and proceed assert not force @@ -625,42 +772,46 @@ def exclude(logger): logger.debug("Error message: {}\n".format(error_message)) logger.debug("Retry exclude after 1 second") time.sleep(1) - output2 = run_fdbcli_command('exclude') - assert 'There are currently 1 servers or localities being excluded from the database' in output2 + output2 = run_fdbcli_command("exclude") + assert ( + "There are currently 1 servers or localities being excluded from the database" + in output2 + ) assert excluded_address in output2 - run_fdbcli_command('include', excluded_address) + run_fdbcli_command("include", excluded_address) # check the include is successful - output4 = run_fdbcli_command('exclude') + output4 = run_fdbcli_command("exclude") assert no_excluded_process_output in output4 wait_for_database_available(logger) + # read the system key 'k', need to enable the option first def read_system_key(k): - output = run_fdbcli_command('option', 'on', 'READ_SYSTEM_KEYS;', 'get', k) - if 'is' not in output: + output = run_fdbcli_command("option", "on", "READ_SYSTEM_KEYS;", "get", k) + if "is" not in output: # key not present return None - _, value = output.split(' is ') + _, value = output.split(" is ") return value @enable_logging() def throttle(logger): # no throttled tags at the beginning - no_throttle_tags_output = 'There are no throttled tags' - output = run_fdbcli_command('throttle', 'list') + no_throttle_tags_output = "There are no throttled tags" + output = run_fdbcli_command("throttle", "list") logger.debug(output) assert output == no_throttle_tags_output # test 'throttle enable auto' - run_fdbcli_command('throttle', 'enable', 'auto') + run_fdbcli_command("throttle", "enable", "auto") # verify the change is applied by reading the system key # not an elegant way, may change later - enable_flag = read_system_key('\\xff\\x02/throttledTags/autoThrottlingEnabled') + enable_flag = read_system_key("\\xff\\x02/throttledTags/autoThrottlingEnabled") assert enable_flag == "`1'" - run_fdbcli_command('throttle', 'disable', 'auto') - enable_flag = read_system_key('\\xff\\x02/throttledTags/autoThrottlingEnabled') + run_fdbcli_command("throttle", "disable", "auto") + enable_flag = read_system_key("\\xff\\x02/throttledTags/autoThrottlingEnabled") # verify disabled assert enable_flag == "`0'" # TODO : test manual throttling, not easy to do now @@ -669,7 +820,9 @@ def throttle(logger): def wait_for_database_available(logger): # sometimes the change takes some time to have effect and the database can be unavailable at that time # this is to wait until the database is available again - while not get_value_from_status_json(True, 'client', 'database_status', 'available'): + while not get_value_from_status_json( + True, "client", "database_status", "available" + ): logger.debug("Database unavailable for now, wait for one second") time.sleep(1) @@ -678,225 +831,288 @@ def wait_for_database_available(logger): def profile(logger): # profile list should return the same list as kill addresses = get_fdb_process_addresses(logger) - output1 = run_fdbcli_command('profile', 'list') - assert output1.split('\n') == addresses + output1 = run_fdbcli_command("profile", "list") + assert output1.split("\n") == addresses # check default output - default_profile_client_get_output = 'Client profiling rate is set to default and size limit is set to default.' - output2 = run_fdbcli_command('profile', 'client', 'get') + default_profile_client_get_output = ( + "Client profiling rate is set to default and size limit is set to default." + ) + output2 = run_fdbcli_command("profile", "client", "get") assert output2 == default_profile_client_get_output # set rate and size limit - run_fdbcli_command('profile', 'client', 'set', '0.5', '1GB') - time.sleep(1) # global config can take some time to sync - output3 = run_fdbcli_command('profile', 'client', 'get') + run_fdbcli_command("profile", "client", "set", "0.5", "1GB") + time.sleep(1) # global config can take some time to sync + output3 = run_fdbcli_command("profile", "client", "get") logger.debug(output3) - output3_list = output3.split(' ') + output3_list = output3.split(" ") assert float(output3_list[6]) == 0.5 # size limit should be 1GB - assert output3_list[-1] == '1000000000.' + assert output3_list[-1] == "1000000000." # change back to default value and check - run_fdbcli_command('profile', 'client', 'set', 'default', 'default') - time.sleep(1) # global config can take some time to sync - assert run_fdbcli_command('profile', 'client', 'get') == default_profile_client_get_output + run_fdbcli_command("profile", "client", "set", "default", "default") + time.sleep(1) # global config can take some time to sync + assert ( + run_fdbcli_command("profile", "client", "get") + == default_profile_client_get_output + ) @enable_logging() def test_available(logger): duration = 0 # seconds we already wait - while not get_value_from_status_json(False, 'client', 'database_status', 'available') and duration < 10: + while ( + not get_value_from_status_json(False, "client", "database_status", "available") + and duration < 10 + ): logger.debug("Sleep for 1 second to wait cluster recovery") time.sleep(1) duration += 1 if duration >= 10: - logger.debug(run_fdbcli_command('status', 'json')) + logger.debug(run_fdbcli_command("status", "json")) assert False @enable_logging() def triggerddteaminfolog(logger): # this command is straightforward and only has one code path - output = run_fdbcli_command('triggerddteaminfolog') - assert output == 'Triggered team info logging in data distribution.' + output = run_fdbcli_command("triggerddteaminfolog") + assert output == "Triggered team info logging in data distribution." + def setup_tenants(tenants): - command = '; '.join(['tenant create %s' % t for t in tenants]) + command = "; ".join(["tenant create %s" % t for t in tenants]) run_fdbcli_command(command) + def clear_database_and_tenants(): - run_fdbcli_command('writemode on; option on SPECIAL_KEY_SPACE_ENABLE_WRITES; clearrange "" \\xff; clearrange \\xff\\xff/management/tenant/map/ \\xff\\xff/management/tenant/map0') + run_fdbcli_command( + 'writemode on; option on SPECIAL_KEY_SPACE_ENABLE_WRITES; clearrange "" \\xff; clearrange \\xff\\xff/management/tenant/map/ \\xff\\xff/management/tenant/map0' + ) + def run_tenant_test(test_func): test_func() clear_database_and_tenants() + @enable_logging() def tenant_create(logger): - output1 = run_fdbcli_command('tenant create tenant') - assert output1 == 'The tenant `tenant\' has been created' + output1 = run_fdbcli_command("tenant create tenant") + assert output1 == "The tenant `tenant' has been created" - output = run_fdbcli_command('tenant create tenant2 tenant_group=tenant_group2') - assert output == 'The tenant `tenant2\' has been created' + output = run_fdbcli_command("tenant create tenant2 tenant_group=tenant_group2") + assert output == "The tenant `tenant2' has been created" + + output = run_fdbcli_command_and_get_error("tenant create tenant") + assert output == "ERROR: A tenant with the given name already exists (2132)" - output = run_fdbcli_command_and_get_error('tenant create tenant') - assert output == 'ERROR: A tenant with the given name already exists (2132)' @enable_logging() def tenant_delete(logger): - setup_tenants(['tenant', 'tenant2']) - run_fdbcli_command('writemode on; usetenant tenant2; set tenant_test value') + setup_tenants(["tenant", "tenant2"]) + run_fdbcli_command("writemode on; usetenant tenant2; set tenant_test value") # delete a tenant while the fdbcli is using that tenant - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env) - cmd_sequence = ['writemode on', 'usetenant tenant', 'tenant delete tenant', 'get tenant_test', 'defaulttenant', 'usetenant tenant'] - output, error_output = process.communicate(input='\n'.join(cmd_sequence).encode()) + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=fdbcli_env, + ) + cmd_sequence = [ + "writemode on", + "usetenant tenant", + "tenant delete tenant", + "get tenant_test", + "defaulttenant", + "usetenant tenant", + ] + output, error_output = process.communicate(input="\n".join(cmd_sequence).encode()) - lines = output.decode().strip().split('\n')[-6:] - error_lines = error_output.decode().strip().split('\n')[-2:] - assert lines[0] == 'Using tenant `tenant\'' - assert lines[1] == 'The tenant `tenant\' has been deleted' - assert lines[2] == 'WARNING: the active tenant was deleted. Use the `usetenant\' or `defaulttenant\'' - assert lines[3] == 'command to choose a new tenant.' - assert error_lines[0] == 'ERROR: Tenant does not exist (2131)' - assert lines[5] == 'Using the default tenant' - assert error_lines[1] == 'ERROR: Tenant `tenant\' does not exist' + lines = output.decode().strip().split("\n")[-6:] + error_lines = error_output.decode().strip().split("\n")[-2:] + assert lines[0] == "Using tenant `tenant'" + assert lines[1] == "The tenant `tenant' has been deleted" + assert ( + lines[2] + == "WARNING: the active tenant was deleted. Use the `usetenant' or `defaulttenant'" + ) + assert lines[3] == "command to choose a new tenant." + assert error_lines[0] == "ERROR: Tenant does not exist (2131)" + assert lines[5] == "Using the default tenant" + assert error_lines[1] == "ERROR: Tenant `tenant' does not exist" # delete a non-empty tenant - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env) - cmd_sequence = ['writemode on', 'tenant delete tenant2', 'usetenant tenant2', 'clear tenant_test', 'defaulttenant', 'tenant delete tenant2'] - output, error_output = process.communicate(input='\n'.join(cmd_sequence).encode()) + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=fdbcli_env, + ) + cmd_sequence = [ + "writemode on", + "tenant delete tenant2", + "usetenant tenant2", + "clear tenant_test", + "defaulttenant", + "tenant delete tenant2", + ] + output, error_output = process.communicate(input="\n".join(cmd_sequence).encode()) - lines = output.decode().strip().split('\n')[-4:] - error_lines = error_output.decode().strip().split('\n')[-1:] - assert error_lines[0] == 'ERROR: Cannot delete a non-empty tenant (2133)' - assert lines[0] == 'Using tenant `tenant2\'' - assert lines[1].startswith('Committed') - assert lines[2] == 'Using the default tenant' - assert lines[3] == 'The tenant `tenant2\' has been deleted' + lines = output.decode().strip().split("\n")[-4:] + error_lines = error_output.decode().strip().split("\n")[-1:] + assert error_lines[0] == "ERROR: Cannot delete a non-empty tenant (2133)" + assert lines[0] == "Using tenant `tenant2'" + assert lines[1].startswith("Committed") + assert lines[2] == "Using the default tenant" + assert lines[3] == "The tenant `tenant2' has been deleted" # delete a non-existing tenant - output = run_fdbcli_command_and_get_error('tenant delete tenant') - assert output == 'ERROR: Tenant does not exist (2131)' + output = run_fdbcli_command_and_get_error("tenant delete tenant") + assert output == "ERROR: Tenant does not exist (2131)" + @enable_logging() def tenant_list(logger): - output = run_fdbcli_command('tenant list') - assert output == 'The cluster has no tenants' + output = run_fdbcli_command("tenant list") + assert output == "The cluster has no tenants" - setup_tenants(['tenant', 'tenant2']) + setup_tenants(["tenant", "tenant2"]) - output = run_fdbcli_command('tenant list') - assert output == '1. tenant\n 2. tenant2' + output = run_fdbcli_command("tenant list") + assert output == "1. tenant\n 2. tenant2" - output = run_fdbcli_command('tenant list a z limit=1') - assert output == '1. tenant' + output = run_fdbcli_command("tenant list a z limit=1") + assert output == "1. tenant" - output = run_fdbcli_command('tenant list a tenant2') - assert output == '1. tenant' + output = run_fdbcli_command("tenant list a tenant2") + assert output == "1. tenant" - output = run_fdbcli_command('tenant list tenant2 z') - assert output == '1. tenant2' + output = run_fdbcli_command("tenant list tenant2 z") + assert output == "1. tenant2" - output = run_fdbcli_command('tenant list a b') - assert output == 'The cluster has no tenants in the specified range' + output = run_fdbcli_command("tenant list a b") + assert output == "The cluster has no tenants in the specified range" - output = run_fdbcli_command_and_get_error('tenant list b a') - assert output == 'ERROR: end must be larger than begin' + output = run_fdbcli_command_and_get_error("tenant list b a") + assert output == "ERROR: end must be larger than begin" - output = run_fdbcli_command_and_get_error('tenant list a b limit=12x') - assert output == 'ERROR: invalid limit `12x\'' + output = run_fdbcli_command_and_get_error("tenant list a b limit=12x") + assert output == "ERROR: invalid limit `12x'" - output = run_fdbcli_command_and_get_error('tenant list a b offset=13y') - assert output == 'ERROR: invalid offset `13y\'' + output = run_fdbcli_command_and_get_error("tenant list a b offset=13y") + assert output == "ERROR: invalid offset `13y'" + + output = run_fdbcli_command_and_get_error("tenant list a b state=14z") + assert output == "ERROR: unrecognized tenant state(s) `14z'." - output = run_fdbcli_command_and_get_error('tenant list a b state=14z') - assert output == 'ERROR: unrecognized tenant state(s) `14z\'.' @enable_logging() def tenant_get(logger): - setup_tenants(['tenant', 'tenant2 tenant_group=tenant_group2']) + setup_tenants(["tenant", "tenant2 tenant_group=tenant_group2"]) - output = run_fdbcli_command('tenant get tenant') - lines = output.split('\n') + output = run_fdbcli_command("tenant get tenant") + lines = output.split("\n") assert len(lines) == 3 - assert lines[0].strip().startswith('id: ') - assert lines[1].strip().startswith('prefix: ') - assert lines[2].strip() == 'tenant state: ready' + assert lines[0].strip().startswith("id: ") + assert lines[1].strip().startswith("prefix: ") + assert lines[2].strip() == "tenant state: ready" - output = run_fdbcli_command('tenant get tenant JSON') + output = run_fdbcli_command("tenant get tenant JSON") json_output = json.loads(output, strict=False) - assert(len(json_output) == 2) - assert('tenant' in json_output) - assert(json_output['type'] == 'success') - assert(len(json_output['tenant']) == 4) - assert('id' in json_output['tenant']) - assert('name' in json_output['tenant']) - assert(len(json_output['tenant']['name']) == 2) - assert('base64' in json_output['tenant']['name']) - assert('printable' in json_output['tenant']['name']) - assert('prefix' in json_output['tenant']) - assert(len(json_output['tenant']['prefix']) == 2) - assert('base64' in json_output['tenant']['prefix']) - assert('printable' in json_output['tenant']['prefix']) - assert(json_output['tenant']['tenant_state'] == 'ready') + assert len(json_output) == 2 + assert "tenant" in json_output + assert json_output["type"] == "success" + assert len(json_output["tenant"]) == 4 + assert "id" in json_output["tenant"] + assert "name" in json_output["tenant"] + assert len(json_output["tenant"]["name"]) == 2 + assert "base64" in json_output["tenant"]["name"] + assert "printable" in json_output["tenant"]["name"] + assert "prefix" in json_output["tenant"] + assert len(json_output["tenant"]["prefix"]) == 2 + assert "base64" in json_output["tenant"]["prefix"] + assert "printable" in json_output["tenant"]["prefix"] + assert json_output["tenant"]["tenant_state"] == "ready" - output = run_fdbcli_command('tenant get tenant2') - lines = output.split('\n') + output = run_fdbcli_command("tenant get tenant2") + lines = output.split("\n") assert len(lines) == 4 - assert lines[0].strip().startswith('id: ') - assert lines[1].strip().startswith('prefix: ') - assert lines[2].strip() == 'tenant state: ready' - assert lines[3].strip() == 'tenant group: tenant_group2' + assert lines[0].strip().startswith("id: ") + assert lines[1].strip().startswith("prefix: ") + assert lines[2].strip() == "tenant state: ready" + assert lines[3].strip() == "tenant group: tenant_group2" - output = run_fdbcli_command('tenant get tenant2 JSON') + output = run_fdbcli_command("tenant get tenant2 JSON") json_output = json.loads(output, strict=False) - assert(len(json_output) == 2) - assert('tenant' in json_output) - assert(json_output['type'] == 'success') - assert(len(json_output['tenant']) == 5) - assert('id' in json_output['tenant']) - assert('name' in json_output['tenant']) - assert(len(json_output['tenant']['name']) == 2) - assert('base64' in json_output['tenant']['name']) - assert('printable' in json_output['tenant']['name']) - assert('prefix' in json_output['tenant']) - assert(json_output['tenant']['tenant_state'] == 'ready') - assert('tenant_group' in json_output['tenant']) - assert(len(json_output['tenant']['tenant_group']) == 2) - assert('base64' in json_output['tenant']['tenant_group']) - assert(json_output['tenant']['tenant_group']['printable'] == 'tenant_group2') + assert len(json_output) == 2 + assert "tenant" in json_output + assert json_output["type"] == "success" + assert len(json_output["tenant"]) == 5 + assert "id" in json_output["tenant"] + assert "name" in json_output["tenant"] + assert len(json_output["tenant"]["name"]) == 2 + assert "base64" in json_output["tenant"]["name"] + assert "printable" in json_output["tenant"]["name"] + assert "prefix" in json_output["tenant"] + assert json_output["tenant"]["tenant_state"] == "ready" + assert "tenant_group" in json_output["tenant"] + assert len(json_output["tenant"]["tenant_group"]) == 2 + assert "base64" in json_output["tenant"]["tenant_group"] + assert json_output["tenant"]["tenant_group"]["printable"] == "tenant_group2" + @enable_logging() def tenant_configure(logger): - setup_tenants(['tenant']) + setup_tenants(["tenant"]) - output = run_fdbcli_command('tenant configure tenant tenant_group=tenant_group1') - assert output == 'The configuration for tenant `tenant\' has been updated' + output = run_fdbcli_command("tenant configure tenant tenant_group=tenant_group1") + assert output == "The configuration for tenant `tenant' has been updated" - output = run_fdbcli_command('tenant get tenant') - lines = output.split('\n') + output = run_fdbcli_command("tenant get tenant") + lines = output.split("\n") assert len(lines) == 4 - assert lines[3].strip() == 'tenant group: tenant_group1' + assert lines[3].strip() == "tenant group: tenant_group1" - output = run_fdbcli_command('tenant configure tenant unset tenant_group') - assert output == 'The configuration for tenant `tenant\' has been updated' + output = run_fdbcli_command("tenant configure tenant unset tenant_group") + assert output == "The configuration for tenant `tenant' has been updated" - output = run_fdbcli_command('tenant get tenant') - lines = output.split('\n') + output = run_fdbcli_command("tenant get tenant") + lines = output.split("\n") assert len(lines) == 3 - output = run_fdbcli_command_and_get_error('tenant configure tenant tenant_group=tenant_group1 tenant_group=tenant_group2') - assert output == 'ERROR: configuration parameter `tenant_group\' specified more than once.' + output = run_fdbcli_command_and_get_error( + "tenant configure tenant tenant_group=tenant_group1 tenant_group=tenant_group2" + ) + assert ( + output + == "ERROR: configuration parameter `tenant_group' specified more than once." + ) - output = run_fdbcli_command_and_get_error('tenant configure tenant unset') - assert output == 'ERROR: `unset\' specified without a configuration parameter.' + output = run_fdbcli_command_and_get_error("tenant configure tenant unset") + assert output == "ERROR: `unset' specified without a configuration parameter." - output = run_fdbcli_command_and_get_error('tenant configure tenant unset tenant_group=tenant_group1') - assert output == 'ERROR: unrecognized configuration parameter `tenant_group=tenant_group1\'.' + output = run_fdbcli_command_and_get_error( + "tenant configure tenant unset tenant_group=tenant_group1" + ) + assert ( + output + == "ERROR: unrecognized configuration parameter `tenant_group=tenant_group1'." + ) - output = run_fdbcli_command_and_get_error('tenant configure tenant tenant_group') - assert output == 'ERROR: invalid configuration string `tenant_group\'. String must specify a value using `=\'.' + output = run_fdbcli_command_and_get_error("tenant configure tenant tenant_group") + assert ( + output + == "ERROR: invalid configuration string `tenant_group'. String must specify a value using `='." + ) + + output = run_fdbcli_command_and_get_error( + "tenant configure tenant3 tenant_group=tenant_group1" + ) + assert output == "ERROR: Tenant does not exist (2131)" - output = run_fdbcli_command_and_get_error('tenant configure tenant3 tenant_group=tenant_group1') - assert output == 'ERROR: Tenant does not exist (2131)' expected_output = """ ERROR: assigned_cluster is only valid in metacluster configuration. @@ -907,79 +1123,118 @@ ERROR: Tenant configuration is invalid (2140) @enable_logging() def tenant_rename(logger): - setup_tenants(['tenant', 'tenant2']) + setup_tenants(["tenant", "tenant2"]) - output = run_fdbcli_command('tenant rename tenant tenant3') - assert output == 'The tenant `tenant\' has been renamed to `tenant3\'' + output = run_fdbcli_command("tenant rename tenant tenant3") + assert output == "The tenant `tenant' has been renamed to `tenant3'" - output = run_fdbcli_command_and_get_error('tenant rename tenant tenant4') - assert output == 'ERROR: Tenant does not exist (2131)' + output = run_fdbcli_command_and_get_error("tenant rename tenant tenant4") + assert output == "ERROR: Tenant does not exist (2131)" + + output = run_fdbcli_command_and_get_error("tenant rename tenant2 tenant3") + assert output == "ERROR: A tenant with the given name already exists (2132)" - output = run_fdbcli_command_and_get_error('tenant rename tenant2 tenant3') - assert output == 'ERROR: A tenant with the given name already exists (2132)' @enable_logging() def tenant_usetenant(logger): - setup_tenants(['tenant', 'tenant2']) + setup_tenants(["tenant", "tenant2"]) - output = run_fdbcli_command('usetenant') - assert output == 'Using the default tenant' + output = run_fdbcli_command("usetenant") + assert output == "Using the default tenant" - output = run_fdbcli_command_and_get_error('usetenant tenant3') - assert output == 'ERROR: Tenant `tenant3\' does not exist' + output = run_fdbcli_command_and_get_error("usetenant tenant3") + assert output == "ERROR: Tenant `tenant3' does not exist" # Test writing keys to different tenants and make sure they all work correctly - run_fdbcli_command('writemode on; set tenant_test default_tenant') - output = run_fdbcli_command('get tenant_test') - assert output == '`tenant_test\' is `default_tenant\'' + run_fdbcli_command("writemode on; set tenant_test default_tenant") + output = run_fdbcli_command("get tenant_test") + assert output == "`tenant_test' is `default_tenant'" - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env) - cmd_sequence = ['writemode on', 'usetenant tenant', 'get tenant_test', 'set tenant_test tenant'] - output, _ = process.communicate(input='\n'.join(cmd_sequence).encode()) + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + env=fdbcli_env, + ) + cmd_sequence = [ + "writemode on", + "usetenant tenant", + "get tenant_test", + "set tenant_test tenant", + ] + output, _ = process.communicate(input="\n".join(cmd_sequence).encode()) - lines = output.decode().strip().split('\n')[-3:] - assert lines[0] == 'Using tenant `tenant\'' - assert lines[1] == '`tenant_test\': not found' - assert lines[2].startswith('Committed') + lines = output.decode().strip().split("\n")[-3:] + assert lines[0] == "Using tenant `tenant'" + assert lines[1] == "`tenant_test': not found" + assert lines[2].startswith("Committed") - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env) - cmd_sequence = ['writemode on', 'usetenant tenant2', 'get tenant_test', 'set tenant_test tenant2', 'get tenant_test'] - output, _ = process.communicate(input='\n'.join(cmd_sequence).encode()) + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + env=fdbcli_env, + ) + cmd_sequence = [ + "writemode on", + "usetenant tenant2", + "get tenant_test", + "set tenant_test tenant2", + "get tenant_test", + ] + output, _ = process.communicate(input="\n".join(cmd_sequence).encode()) - lines = output.decode().strip().split('\n')[-4:] - assert lines[0] == 'Using tenant `tenant2\'' - assert lines[1] == '`tenant_test\': not found' - assert lines[2].startswith('Committed') - assert lines[3] == '`tenant_test\' is `tenant2\'' + lines = output.decode().strip().split("\n")[-4:] + assert lines[0] == "Using tenant `tenant2'" + assert lines[1] == "`tenant_test': not found" + assert lines[2].startswith("Committed") + assert lines[3] == "`tenant_test' is `tenant2'" - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=fdbcli_env) - cmd_sequence = ['usetenant tenant', 'get tenant_test', 'usetenant tenant2', 'get tenant_test', 'defaulttenant', 'get tenant_test'] - output, _ = process.communicate(input='\n'.join(cmd_sequence).encode()) + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + env=fdbcli_env, + ) + cmd_sequence = [ + "usetenant tenant", + "get tenant_test", + "usetenant tenant2", + "get tenant_test", + "defaulttenant", + "get tenant_test", + ] + output, _ = process.communicate(input="\n".join(cmd_sequence).encode()) + + lines = output.decode().strip().split("\n")[-6:] + assert lines[0] == "Using tenant `tenant'" + assert lines[1] == "`tenant_test' is `tenant'" + assert lines[2] == "Using tenant `tenant2'" + assert lines[3] == "`tenant_test' is `tenant2'" + assert lines[4] == "Using the default tenant" + assert lines[5] == "`tenant_test' is `default_tenant'" - lines = output.decode().strip().split('\n')[-6:] - assert lines[0] == 'Using tenant `tenant\'' - assert lines[1] == '`tenant_test\' is `tenant\'' - assert lines[2] == 'Using tenant `tenant2\'' - assert lines[3] == '`tenant_test\' is `tenant2\'' - assert lines[4] == 'Using the default tenant' - assert lines[5] == '`tenant_test\' is `default_tenant\'' @enable_logging() def tenant_old_commands(logger): - create_output = run_fdbcli_command('tenant create tenant') - list_output = run_fdbcli_command('tenant list') - get_output = run_fdbcli_command('tenant get tenant') + create_output = run_fdbcli_command("tenant create tenant") + list_output = run_fdbcli_command("tenant list") + get_output = run_fdbcli_command("tenant get tenant") # Run the gettenant command here because the ID will be different in the second block - get_output_old = run_fdbcli_command('gettenant tenant') - configure_output = run_fdbcli_command('tenant configure tenant tenant_group=tenant_group1') - rename_output = run_fdbcli_command('tenant rename tenant tenant2') - delete_output = run_fdbcli_command('tenant delete tenant2') + get_output_old = run_fdbcli_command("gettenant tenant") + configure_output = run_fdbcli_command( + "tenant configure tenant tenant_group=tenant_group1" + ) + rename_output = run_fdbcli_command("tenant rename tenant tenant2") + delete_output = run_fdbcli_command("tenant delete tenant2") - create_output_old = run_fdbcli_command('createtenant tenant') - list_output_old = run_fdbcli_command('listtenants') - configure_output_old = run_fdbcli_command('configuretenant tenant tenant_group=tenant_group1') - rename_output_old = run_fdbcli_command('renametenant tenant tenant2') - delete_output_old = run_fdbcli_command('deletetenant tenant2') + create_output_old = run_fdbcli_command("createtenant tenant") + list_output_old = run_fdbcli_command("listtenants") + configure_output_old = run_fdbcli_command( + "configuretenant tenant tenant_group=tenant_group1" + ) + rename_output_old = run_fdbcli_command("renametenant tenant tenant2") + delete_output_old = run_fdbcli_command("deletetenant tenant2") assert create_output == create_output_old assert list_output == list_output_old @@ -988,56 +1243,65 @@ def tenant_old_commands(logger): assert rename_output == rename_output_old assert delete_output == delete_output_old + @enable_logging() def tenant_group_list(logger): - output = run_fdbcli_command('tenantgroup list') - assert output == 'The cluster has no tenant groups' + output = run_fdbcli_command("tenantgroup list") + assert output == "The cluster has no tenant groups" - setup_tenants(['tenant', 'tenant2 tenant_group=tenant_group2', 'tenant3 tenant_group=tenant_group3']) + setup_tenants( + [ + "tenant", + "tenant2 tenant_group=tenant_group2", + "tenant3 tenant_group=tenant_group3", + ] + ) - output = run_fdbcli_command('tenantgroup list') - assert output == '1. tenant_group2\n 2. tenant_group3' + output = run_fdbcli_command("tenantgroup list") + assert output == "1. tenant_group2\n 2. tenant_group3" - output = run_fdbcli_command('tenantgroup list a z 1') - assert output == '1. tenant_group2' + output = run_fdbcli_command("tenantgroup list a z 1") + assert output == "1. tenant_group2" - output = run_fdbcli_command('tenantgroup list a tenant_group3') - assert output == '1. tenant_group2' + output = run_fdbcli_command("tenantgroup list a tenant_group3") + assert output == "1. tenant_group2" - output = run_fdbcli_command('tenantgroup list tenant_group3 z') - assert output == '1. tenant_group3' + output = run_fdbcli_command("tenantgroup list tenant_group3 z") + assert output == "1. tenant_group3" - output = run_fdbcli_command('tenantgroup list a b') - assert output == 'The cluster has no tenant groups in the specified range' + output = run_fdbcli_command("tenantgroup list a b") + assert output == "The cluster has no tenant groups in the specified range" - output = run_fdbcli_command_and_get_error('tenantgroup list b a') - assert output == 'ERROR: end must be larger than begin' + output = run_fdbcli_command_and_get_error("tenantgroup list b a") + assert output == "ERROR: end must be larger than begin" + + output = run_fdbcli_command_and_get_error("tenantgroup list a b 12x") + assert output == "ERROR: invalid limit `12x'" - output = run_fdbcli_command_and_get_error('tenantgroup list a b 12x') - assert output == 'ERROR: invalid limit `12x\'' @enable_logging() def tenant_group_get(logger): - setup_tenants(['tenant tenant_group=tenant_group']) + setup_tenants(["tenant tenant_group=tenant_group"]) - output = run_fdbcli_command('tenantgroup get tenant_group') - assert output == 'The tenant group is present in the cluster' + output = run_fdbcli_command("tenantgroup get tenant_group") + assert output == "The tenant group is present in the cluster" - output = run_fdbcli_command('tenantgroup get tenant_group JSON') + output = run_fdbcli_command("tenantgroup get tenant_group JSON") json_output = json.loads(output, strict=False) - assert(len(json_output) == 2) - assert('tenant_group' in json_output) - assert(json_output['type'] == 'success') - assert(len(json_output['tenant_group']) == 0) + assert len(json_output) == 2 + assert "tenant_group" in json_output + assert json_output["type"] == "success" + assert len(json_output["tenant_group"]) == 0 - output = run_fdbcli_command_and_get_error('tenantgroup get tenant_group2') - assert output == 'ERROR: tenant group not found' + output = run_fdbcli_command_and_get_error("tenantgroup get tenant_group2") + assert output == "ERROR: tenant group not found" - output = run_fdbcli_command('tenantgroup get tenant_group2 JSON') + output = run_fdbcli_command("tenantgroup get tenant_group2 JSON") json_output = json.loads(output, strict=False) - assert(len(json_output) == 2) - assert(json_output['type'] == 'error') - assert(json_output['error'] == 'tenant group not found') + assert len(json_output) == 2 + assert json_output["type"] == "error" + assert json_output["error"] == "tenant group not found" + def tenants(): run_tenant_test(tenant_create) @@ -1051,19 +1315,52 @@ def tenants(): run_tenant_test(tenant_group_list) run_tenant_test(tenant_group_get) -def integer_options(): - process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env) - cmd_sequence = ['option on TIMEOUT 1000', 'writemode on', 'clear foo'] - output, error_output = process.communicate(input='\n'.join(cmd_sequence).encode()) - lines = output.decode().strip().split('\n')[-2:] - assert lines[0] == 'Option enabled for all transactions' - assert lines[1].startswith('Committed') - assert error_output == b'' +@enable_logging() +def idempotency_ids(logger): + command = "idempotencyids status" + output = run_fdbcli_command(command) + logger.debug(command + " : " + output) + json.loads(output) + + command = "idempotencyids clear 5" + output = run_fdbcli_command(command) + logger.debug(command + " : " + output) + assert output == "Successfully cleared idempotency IDs.", output + + # Incorrect number of tokens + command = "idempotencyids clear" + output = run_fdbcli_command(command) + logger.debug(command + " : " + output) + assert output == "Usage: idempotencyids [status | clear ]", output + + # Incorrect number of tokens + command = "idempotencyids" + output = run_fdbcli_command(command) + logger.debug(command + " : " + output) + assert output == "Usage: idempotencyids [status | clear ]", output + + +def integer_options(): + process = subprocess.Popen( + command_template[:-1], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=fdbcli_env, + ) + cmd_sequence = ["option on TIMEOUT 1000", "writemode on", "clear foo"] + output, error_output = process.communicate(input="\n".join(cmd_sequence).encode()) + + lines = output.decode().strip().split("\n")[-2:] + assert lines[0] == "Option enabled for all transactions" + assert lines[1].startswith("Committed") + assert error_output == b"" + def tls_address_suffix(): # fdbcli shall prevent a non-TLS fdbcli run from connecting to an all-TLS cluster - preamble = 'eNW1yf1M:eNW1yf1M@' + preamble = "eNW1yf1M:eNW1yf1M@" num_server_addrs = [1, 2, 5] err_output_server_tls = "ERROR: fdbcli is not configured with TLS, but all of the coordinators have TLS addresses." @@ -1071,28 +1368,54 @@ def tls_address_suffix(): cluster_fn = tmpdir + "/fdb.cluster" for num_server_addr in num_server_addrs: with open(cluster_fn, "w") as fp: - fp.write(preamble + ",".join( - ["127.0.0.1:{}:tls".format(4000 + addr_idx) for addr_idx in range(num_server_addr)])) + fp.write( + preamble + + ",".join( + [ + "127.0.0.1:{}:tls".format(4000 + addr_idx) + for addr_idx in range(num_server_addr) + ] + ) + ) fp.close() - fdbcli_process = subprocess.run(command_template[:2] + [cluster_fn], capture_output=True) + fdbcli_process = subprocess.run( + command_template[:2] + [cluster_fn], capture_output=True + ) assert fdbcli_process.returncode != 0 err_out = fdbcli_process.stderr.decode("utf8").strip() assert err_out == err_output_server_tls, f"unexpected output: {err_out}" -if __name__ == '__main__': - parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, - description=""" + +if __name__ == "__main__": + parser = ArgumentParser( + formatter_class=RawDescriptionHelpFormatter, + description=""" The test calls fdbcli commands through fdbcli --exec "" interactively using subprocess. The outputs from fdbcli are returned and compared to predefined results. Consequently, changing fdbcli outputs or breaking any commands will cause the test to fail. Commands that are easy to test will run against a single process cluster. For complex commands like exclude, they will run against a cluster with multiple(current set to 5) processes. If external_client_library is given, we will disable the local client and use the external client to run fdbcli. - """) - parser.add_argument('build_dir', metavar='BUILD_DIRECTORY', help='FDB build directory') - parser.add_argument('cluster_file', metavar='CLUSTER_FILE', help='FDB cluster file') - parser.add_argument('process_number', nargs='?', metavar='PROCESS_NUMBER', help="Number of fdb processes", type=int, default=1) - parser.add_argument('--external-client-library', '-e', metavar='EXTERNAL_CLIENT_LIBRARY_PATH', help="External client library path") + """, + ) + parser.add_argument( + "build_dir", metavar="BUILD_DIRECTORY", help="FDB build directory" + ) + parser.add_argument("cluster_file", metavar="CLUSTER_FILE", help="FDB cluster file") + parser.add_argument( + "process_number", + nargs="?", + metavar="PROCESS_NUMBER", + help="Number of fdb processes", + type=int, + default=1, + ) + parser.add_argument( + "--external-client-library", + "-e", + metavar="EXTERNAL_CLIENT_LIBRARY_PATH", + help="External client library path", + ) args = parser.parse_args() # keep current environment variables @@ -1100,11 +1423,18 @@ if __name__ == '__main__': # set external client library if provided if args.external_client_library: # disable local client and use the external client library - fdbcli_env['FDB_NETWORK_OPTION_DISABLE_LOCAL_CLIENT'] = '' - fdbcli_env['FDB_NETWORK_OPTION_EXTERNAL_CLIENT_LIBRARY'] = args.external_client_library + fdbcli_env["FDB_NETWORK_OPTION_DISABLE_LOCAL_CLIENT"] = "" + fdbcli_env[ + "FDB_NETWORK_OPTION_EXTERNAL_CLIENT_LIBRARY" + ] = args.external_client_library # shell command template - command_template = [args.build_dir + '/bin/fdbcli', '-C', args.cluster_file, '--exec'] + command_template = [ + args.build_dir + "/bin/fdbcli", + "-C", + args.cluster_file, + "--exec", + ] # tests for fdbcli commands # assertions will fail if fdbcli does not work as expected test_available() @@ -1122,7 +1452,7 @@ if __name__ == '__main__': # suspend() transaction() # this is replaced by the "quota" command - #throttle() + # throttle() triggerddteaminfolog() tenants() versionepoch() @@ -1130,7 +1460,8 @@ if __name__ == '__main__': tls_address_suffix() knobmanagement() # TODO: fix the issue when running through the external client - #quota() + # quota() + idempotency_ids() else: assert args.process_number > 1, "Process number should be positive" coordinators() diff --git a/fdbcli/tests/metacluster_fdbcli_tests.py b/fdbcli/tests/metacluster_fdbcli_tests.py index e01157126d..6e0ce8ca1c 100755 --- a/fdbcli/tests/metacluster_fdbcli_tests.py +++ b/fdbcli/tests/metacluster_fdbcli_tests.py @@ -38,29 +38,7 @@ def enable_logging(level=logging.DEBUG): return func_decorator -def run_command(*args): - commands = ["{}".format(args)] - print(commands) - try: - process = subprocess.run(commands, stdout=subprocess.PIPE, env=fdbcli_env, timeout=20) - return process.stdout.decode('utf-8').strip() - except subprocess.TimeoutExpired: - raise Exception('the command is stuck') - - def run_fdbcli_command(cluster_file, *args): - command_template = [fdbcli_bin, '-C', "{}".format(cluster_file), '--exec'] - commands = command_template + ["{}".format(' '.join(args))] - print(commands) - try: - # if the fdbcli command is stuck for more than 20 seconds, the database is definitely unavailable - process = subprocess.run(commands, stdout=subprocess.PIPE, env=fdbcli_env, timeout=20) - return process.stdout.decode('utf-8').strip() - except subprocess.TimeoutExpired: - raise Exception('The fdbcli command is stuck, database is unavailable') - - -def run_fdbcli_command_and_get_error(cluster_file, *args): """run the fdbcli statement: fdbcli --exec ' ... '. Returns: @@ -70,7 +48,10 @@ def run_fdbcli_command_and_get_error(cluster_file, *args): commands = command_template + ["{}".format(' '.join(args))] try: process = subprocess.run(commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env, timeout=20) - return process.stdout.decode('utf-8').strip(), process.stderr.decode('utf-8').strip() + rc = process.returncode + out = process.stdout.decode('utf-8').strip() + err = process.stderr.decode('utf-8').strip() + return rc, out, err except subprocess.TimeoutExpired: raise Exception('The fdbcli command is stuck, database is unavailable') @@ -81,35 +62,46 @@ def get_cluster_connection_str(cluster_file_path): return conn_str -def metacluster_create(cluster_file, name, tenant_id_prefix): - return run_fdbcli_command(cluster_file, "metacluster create_experimental", name, str(tenant_id_prefix)) +@enable_logging() +def metacluster_create(logger, cluster_file, name, tenant_id_prefix): + rc, out, err = run_fdbcli_command(cluster_file, "metacluster create_experimental", name, str(tenant_id_prefix)) + if rc != 0: + raise Exception(err) + logger.debug(out) + logger.debug('Metacluster {} created'.format(name)) -def metacluster_register(management_cluster_file, data_cluster_file, name): +def metacluster_register(management_cluster_file, data_cluster_file, name, max_tenant_groups): conn_str = get_cluster_connection_str(data_cluster_file) - return run_fdbcli_command(management_cluster_file, "metacluster register", name, "connection_string={}".format( - conn_str), 'max_tenant_groups=6') + rc, out, err = run_fdbcli_command(management_cluster_file, + "metacluster register", + name, + "connection_string={}".format(conn_str), + 'max_tenant_groups={}'.format(max_tenant_groups)) + if rc != 0: + raise Exception(err) @enable_logging() -def setup_metacluster(logger, management_cluster, data_clusters): +def setup_metacluster(logger, management_cluster, data_clusters, max_tenant_groups_per_cluster): management_cluster_file = management_cluster[0] management_cluster_name = management_cluster[1] tenant_id_prefix = random.randint(0, 32767) - output = metacluster_create(management_cluster_file, management_cluster_name, tenant_id_prefix) - logger.debug(output) + logger.debug('management cluster: {}'.format(management_cluster_name)) + logger.debug('data clusters: {}'.format([name for (cf, name) in data_clusters])) + metacluster_create(management_cluster_file, management_cluster_name, tenant_id_prefix) for (cf, name) in data_clusters: - output = metacluster_register(management_cluster_file, cf, name) - logger.debug(output) + metacluster_register(management_cluster_file, cf, name, max_tenant_groups=max_tenant_groups_per_cluster) def metacluster_status(cluster_file): - return run_fdbcli_command(cluster_file, "metacluster status") + _, out, _ = run_fdbcli_command(cluster_file, "metacluster status") + return out def setup_tenants(management_cluster_file, data_cluster_files, tenants): for tenant in tenants: - output = run_fdbcli_command(management_cluster_file, 'tenant create', tenant) + _, output, _ = run_fdbcli_command(management_cluster_file, 'tenant create', tenant) expected_output = 'The tenant `{}\' has been created'.format(tenant) assert output == expected_output @@ -121,61 +113,74 @@ def configure_tenant(management_cluster_file, data_cluster_files, tenant, tenant if assigned_cluster: command = command + ' assigned_cluster={}'.format(assigned_cluster) - output, err = run_fdbcli_command_and_get_error(management_cluster_file, command) + _, output, err = run_fdbcli_command(management_cluster_file, command) return output, err -@enable_logging() -def clear_database_and_tenants(logger, management_cluster_file, data_cluster_files): +def clear_database_and_tenants(management_cluster_file, data_cluster_files): subcmd1 = 'writemode on' subcmd2 = 'option on SPECIAL_KEY_SPACE_ENABLE_WRITES' subcmd3 = 'clearrange ' '"" \\xff' subcmd4 = 'clearrange \\xff\\xff/management/tenant/map/ \\xff\\xff/management/tenant/map0' - output = run_fdbcli_command(management_cluster_file, subcmd1, subcmd2, subcmd3, subcmd4) - logger.debug(output) - - -def run_tenant_test(management_cluster_file, data_cluster_files, test_func): - test_func(management_cluster_file, data_cluster_files) - clear_database_and_tenants(management_cluster_file, data_cluster_files) + run_fdbcli_command(management_cluster_file, subcmd1, subcmd2, subcmd3, subcmd4) @enable_logging() -def clusters_status_test(logger, cluster_files): +def clusters_status_test(logger, cluster_files, max_tenant_groups_per_cluster): + logger.debug('Verifying no cluster is part of a metacluster') for cf in cluster_files: output = metacluster_status(cf) assert output == "This cluster is not part of a metacluster" + logger.debug('Verified') num_clusters = len(cluster_files) names = ['meta_mgmt'] names.extend(['data{}'.format(i) for i in range(1, num_clusters)]) - setup_metacluster([cluster_files[0], names[0]], zip(cluster_files[1:], names[1:])) + logger.debug('Setting up a metacluster') + setup_metacluster([cluster_files[0], names[0]], list(zip(cluster_files[1:], names[1:])), + max_tenant_groups_per_cluster=max_tenant_groups_per_cluster) expected = """ number of data clusters: {} tenant group capacity: {} allocated tenant groups: 0 """ - expected = expected.format(num_clusters - 1, 12).strip() + expected = expected.format(num_clusters - 1, (num_clusters - 1) * max_tenant_groups_per_cluster).strip() output = metacluster_status(cluster_files[0]) assert expected == output + logger.debug('Metacluster setup correctly') + for (cf, name) in zip(cluster_files[1:], names[1:]): output = metacluster_status(cf) - expected = "This cluster \"{}\" is a data cluster within the metacluster named \"{" \ - "}\"".format(name, names[0]) + expected = "This cluster \"{}\" is a data cluster within the metacluster named \"{}\"".format(name, names[0]) assert expected == output @enable_logging() def configure_tenants_test_disableClusterAssignment(logger, cluster_files): tenants = ['tenant1', 'tenant2'] - logger.debug('Creating tenants {}'.format(tenants)) + logger.debug('Tenants to create: {}'.format(tenants)) setup_tenants(cluster_files[0], cluster_files[1:], tenants) + # Once we reach here, the tenants have been created successfully + logger.debug('Tenants created: {}'.format(tenants)) for tenant in tenants: out, err = configure_tenant(cluster_files[0], cluster_files[1:], tenant, assigned_cluster='cluster') assert err == 'ERROR: Tenant configuration is invalid (2140)' + logger.debug('Tenants configured') clear_database_and_tenants(cluster_files[0], cluster_files[1:]) + logger.debug('Tenants cleared') + + +@enable_logging() +def test_main(logger): + logger.debug('Tests start') + # This must be the first test to run, since it sets up the metacluster that + # will be used throughout the test + clusters_status_test(cluster_files, max_tenant_groups_per_cluster=5) + + configure_tenants_test_disableClusterAssignment(cluster_files) + logger.debug('Tests complete') if __name__ == "__main__": @@ -197,8 +202,4 @@ if __name__ == "__main__": fdbcli_bin = args.build_dir + '/bin/fdbcli' - # This must be the first test to run, since it sets up the metacluster that - # will be used throughout the test - clusters_status_test(cluster_files) - - configure_tenants_test_disableClusterAssignment(cluster_files) + test_main() diff --git a/fdbclient/BackupAgentBase.actor.cpp b/fdbclient/BackupAgentBase.actor.cpp index 7dd2f25ab7..f65f293930 100644 --- a/fdbclient/BackupAgentBase.actor.cpp +++ b/fdbclient/BackupAgentBase.actor.cpp @@ -262,7 +262,7 @@ bool validTenantAccess(std::map* tenantMap, } int64_t tenantId = TenantInfo::INVALID_TENANT; if (m.isEncrypted()) { - tenantId = m.encryptionHeader()->cipherTextDetails.encryptDomainId; + tenantId = m.encryptDomainId(); } else { tenantId = TenantAPI::extractTenantIdFromMutation(m); } @@ -383,12 +383,18 @@ ACTOR static Future decodeBackupLogValue(Arena* arena, // Decrypt mutation ref if encrypted if (logValue.isEncrypted()) { encryptedLogValue = logValue; - state EncryptCipherDomainId domainId = logValue.encryptionHeader()->cipherTextDetails.encryptDomainId; + state EncryptCipherDomainId domainId = logValue.encryptDomainId(); Reference const> dbInfo = cx->clientInfo; try { - TextAndHeaderCipherKeys cipherKeys = - wait(getEncryptCipherKeys(dbInfo, *logValue.encryptionHeader(), BlobCipherMetrics::RESTORE)); - logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::BACKUP); + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys( + dbInfo, logValue.configurableEncryptionHeader(), BlobCipherMetrics::RESTORE)); + logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::RESTORE); + } else { + TextAndHeaderCipherKeys cipherKeys = wait( + getEncryptCipherKeys(dbInfo, *logValue.encryptionHeader(), BlobCipherMetrics::RESTORE)); + logValue = logValue.decrypt(cipherKeys, tempArena, BlobCipherMetrics::RESTORE); + } } catch (Error& e) { // It's possible a tenant was deleted and the encrypt key fetch failed TraceEvent(SevWarnAlways, "MutationLogRestoreEncryptKeyFetchFailed") diff --git a/fdbclient/BlobCipher.cpp b/fdbclient/BlobCipher.cpp index 695a95f85a..e59a898af0 100644 --- a/fdbclient/BlobCipher.cpp +++ b/fdbclient/BlobCipher.cpp @@ -40,7 +40,6 @@ #include "flow/Trace.h" #include "flow/UnitTest.h" #include "flow/xxhash.h" -#include "include/fdbclient/BlobCipher.h" #include #include @@ -211,6 +210,10 @@ EncryptAuthTokenMode BlobCipherEncryptHeaderRef::getAuthTokenMode() const { flags); } +EncryptCipherDomainId BlobCipherEncryptHeaderRef::getDomainId() const { + return std::visit([](auto& h) { return h.v1.cipherTextDetails.encryptDomainId; }, algoHeader); +} + void BlobCipherEncryptHeaderRef::validateEncryptionHeaderDetails(const BlobCipherDetails& textCipherDetails, const BlobCipherDetails& headerCipherDetails, const StringRef& ivRef) const { @@ -754,8 +757,7 @@ std::vector> BlobCipherKeyCache::getAllCiphers(const En return keyIdCache->getAllCipherKeys(); } -namespace { -int getEncryptAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo) { +int getEncryptCurrentAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo) { if (mode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) { return CLIENT_KNOBS->ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION; } else { @@ -768,7 +770,20 @@ int getEncryptAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAu } } } -} // namespace + +void BlobCipherDetails::validateCipherDetailsWithCipherKey(Reference cipherKey) { + if (!(baseCipherId == cipherKey->getBaseCipherId() && encryptDomainId == cipherKey->getDomainId() && + salt == cipherKey->getSalt())) { + TraceEvent(SevWarn, "EncryptionHeaderCipherMismatch") + .detail("TextDomainId", cipherKey->getDomainId()) + .detail("ExpectedTextDomainId", encryptDomainId) + .detail("TextBaseCipherId", cipherKey->getBaseCipherId()) + .detail("ExpectedTextBaseCipherId", baseCipherId) + .detail("TextSalt", cipherKey->getSalt()) + .detail("ExpectedTextSalt", salt); + throw encrypt_header_metadata_mismatch(); + } +} // EncryptBlobCipherAes265Ctr class methods @@ -896,8 +911,8 @@ void EncryptBlobCipherAes265Ctr::setCipherAlgoHeaderV1(const uint8_t* ciphertext const BlobCipherEncryptHeaderFlagsV1& flags, BlobCipherEncryptHeaderRef* headerRef) { ASSERT_EQ(1, - getEncryptAlgoHeaderVersion((EncryptAuthTokenMode)flags.authTokenMode, - (EncryptAuthTokenAlgo)flags.authTokenAlgo)); + getEncryptCurrentAlgoHeaderVersion((EncryptAuthTokenMode)flags.authTokenMode, + (EncryptAuthTokenAlgo)flags.authTokenAlgo)); if (flags.authTokenMode == EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) { setCipherAlgoHeaderNoAuthV1(flags, headerRef); @@ -930,7 +945,7 @@ void EncryptBlobCipherAes265Ctr::updateEncryptHeader(const uint8_t* ciphertext, updateEncryptHeaderFlagsV1(headerRef, &flags); // update cipher algo header - int algoHeaderVersion = getEncryptAlgoHeaderVersion(authTokenMode, authTokenAlgo); + int algoHeaderVersion = getEncryptCurrentAlgoHeaderVersion(authTokenMode, authTokenAlgo); ASSERT_EQ(algoHeaderVersion, 1); setCipherAlgoHeaderV1(ciphertext, ciphertextLen, flags, headerRef); } @@ -1045,9 +1060,10 @@ Reference EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plainte if (authTokenMode != ENCRYPT_HEADER_AUTH_TOKEN_MODE_NONE) { header->cipherHeaderDetails = headerCipherKeyOpt.get()->details(); } else { - header->cipherHeaderDetails.encryptDomainId = INVALID_ENCRYPT_DOMAIN_ID; - header->cipherHeaderDetails.baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID; - header->cipherHeaderDetails.salt = INVALID_ENCRYPT_RANDOM_SALT; + header->cipherHeaderDetails = BlobCipherDetails(); + ASSERT_EQ(INVALID_ENCRYPT_DOMAIN_ID, header->cipherHeaderDetails.encryptDomainId); + ASSERT_EQ(INVALID_ENCRYPT_CIPHER_KEY_ID, header->cipherHeaderDetails.baseCipherId); + ASSERT_EQ(INVALID_ENCRYPT_RANDOM_SALT, header->cipherHeaderDetails.salt); } memcpy(&header->iv[0], &iv[0], AES_256_IV_LENGTH); diff --git a/fdbclient/BlobGranuleFiles.cpp b/fdbclient/BlobGranuleFiles.cpp index f1ecefcb6d..de79b975d5 100644 --- a/fdbclient/BlobGranuleFiles.cpp +++ b/fdbclient/BlobGranuleFiles.cpp @@ -1531,6 +1531,7 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk, GranuleMaterializeStats& stats) { // TODO REMOVE with early replying ASSERT(readVersion == chunk.includedVersion); + Version lastFileVersion = 0; // Arena to hold all allocations for applying deltas. Most of it, and the arenas produced by reading the files, // will likely be tossed if there are a significant number of mutations, so we copy at the end instead of doing a @@ -1569,6 +1570,8 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk, arena.dependsOn(streams.back().arena()); stats.snapshotRows += snapshotRows.size(); } + ASSERT_WE_THINK(lastFileVersion < chunk.snapshotFile.get().fileVersion); + lastFileVersion = chunk.snapshotFile.get().fileVersion; } else { ASSERT(!chunk.snapshotFile.present()); } @@ -1593,6 +1596,9 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk, arena.dependsOn(streams.back().arena()); } arena.dependsOn(deltaRows.arena()); + + ASSERT_WE_THINK(lastFileVersion < chunk.deltaFiles[deltaIdx].fileVersion); + lastFileVersion = chunk.deltaFiles[deltaIdx].fileVersion; } if (BG_READ_DEBUG) { fmt::print("Applying {} memory deltas\n", chunk.newDeltas.size()); @@ -1600,6 +1606,7 @@ RangeResult materializeBlobGranule(const BlobGranuleChunkRef& chunk, if (!chunk.newDeltas.empty()) { stats.inputBytes += chunk.newDeltas.expectedSize(); // TODO REMOVE validation + ASSERT_WE_THINK(lastFileVersion < chunk.newDeltas.front().version); ASSERT(beginVersion <= chunk.newDeltas.front().version); ASSERT(readVersion >= chunk.newDeltas.back().version); auto memoryRows = sortMemoryDeltas(chunk.newDeltas, chunk.keyRange, requestRange, beginVersion, readVersion); @@ -2188,7 +2195,8 @@ struct KeyValueGen { int targetMutationsPerDelta; KeyRange allRange; - Version version = 0; + // start at higher version to allow snapshot files to have version 1 + Version version = 10; // encryption/compression settings // TODO: possibly different cipher keys or meta context per file? @@ -2551,7 +2559,7 @@ void checkDeltaRead(const KeyValueGen& kvGen, deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), readVersion, ".delta"); Standalone chunk; chunk.deltaFiles.emplace_back_deep( - chunk.arena(), filename, 0, serialized[0].size(), serialized[0].size(), kvGen.cipherKeys); + chunk.arena(), filename, 0, serialized[0].size(), serialized[0].size(), 1, kvGen.cipherKeys); chunk.keyRange = kvGen.allRange; chunk.includedVersion = readVersion; chunk.snapshotVersion = invalidVersion; @@ -2672,8 +2680,13 @@ void checkGranuleRead(const KeyValueGen& kvGen, if (beginVersion == 0) { std::string snapshotFilename = randomBGFilename( deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), 0, ".snapshot"); - chunk.snapshotFile = BlobFilePointerRef( - chunk.arena(), snapshotFilename, 0, serializedSnapshot.size(), serializedSnapshot.size(), kvGen.cipherKeys); + chunk.snapshotFile = BlobFilePointerRef(chunk.arena(), + snapshotFilename, + 0, + serializedSnapshot.size(), + serializedSnapshot.size(), + 1, + kvGen.cipherKeys); } int deltaIdx = 0; while (deltaIdx < serializedDeltas.size() && serializedDeltas[deltaIdx].first < beginVersion) { @@ -2684,7 +2697,8 @@ void checkGranuleRead(const KeyValueGen& kvGen, std::string deltaFilename = randomBGFilename( deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUniqueID(), readVersion, ".delta"); size_t fsize = serializedDeltas[deltaIdx].second.size(); - chunk.deltaFiles.emplace_back_deep(chunk.arena(), deltaFilename, 0, fsize, fsize, kvGen.cipherKeys); + chunk.deltaFiles.emplace_back_deep( + chunk.arena(), deltaFilename, 0, fsize, fsize, serializedDeltas[deltaIdx].first, kvGen.cipherKeys); deltaPtrsVector.push_back(serializedDeltas[deltaIdx].second); if (serializedDeltas[deltaIdx].first >= readVersion) { @@ -3207,12 +3221,12 @@ void chunkFromFileSet(const FileSet& fileSet, int numDeltaFiles) { size_t snapshotSize = std::get<3>(fileSet.snapshotFile).size(); chunk.snapshotFile = - BlobFilePointerRef(chunk.arena(), std::get<0>(fileSet.snapshotFile), 0, snapshotSize, snapshotSize, keys); + BlobFilePointerRef(chunk.arena(), std::get<0>(fileSet.snapshotFile), 0, snapshotSize, snapshotSize, 1, keys); for (int i = 0; i < numDeltaFiles; i++) { size_t deltaSize = std::get<3>(fileSet.deltaFiles[i]).size(); chunk.deltaFiles.emplace_back_deep( - chunk.arena(), std::get<0>(fileSet.deltaFiles[i]), 0, deltaSize, deltaSize, keys); + chunk.arena(), std::get<0>(fileSet.deltaFiles[i]), 0, deltaSize, deltaSize, 2 + i, keys); deltaPtrs[i] = std::get<2>(fileSet.deltaFiles[i]); } diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp index cc1c09b599..59474e7141 100644 --- a/fdbclient/ClientKnobs.cpp +++ b/fdbclient/ClientKnobs.cpp @@ -24,6 +24,7 @@ #include "fdbclient/Tenant.h" #include "flow/IRandom.h" #include "flow/UnitTest.h" +#include "flow/flow.h" #define init(...) KNOB_FN(__VA_ARGS__, INIT_ATOMIC_KNOB, INIT_KNOB)(__VA_ARGS__) @@ -302,8 +303,9 @@ void ClientKnobs::initialize(Randomize randomize) { init( CLIENT_ENABLE_USING_CLUSTER_ID_KEY, false ); init( ENABLE_ENCRYPTION_CPU_TIME_LOGGING, false ); + init( SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS, true ); init( SIMULATION_EKP_TENANT_IDS_TO_DROP, "-1" ); - init( ENABLE_CONFIGURABLE_ENCRYPTION, false ); + init( ENABLE_CONFIGURABLE_ENCRYPTION, true ); init( ENCRYPT_HEADER_FLAGS_VERSION, 1 ); init( ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION, 1 ); init( ENCRYPT_HEADER_AES_CTR_AES_CMAC_AUTH_VERSION, 1 ); diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index 3f676f769e..915610caaa 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -61,6 +61,7 @@ #include #include #include +#include #include "flow/actorcompiler.h" // This must be the last #include. @@ -551,14 +552,13 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter { struct Options { constexpr static FileIdentifier file_identifier = 3152016; - // TODO: Compression is not currently supported so this should always be false - bool compressionEnabled = false; + bool configurableEncryptionEnabled = false; Options() {} template void serialize(Ar& ar) { - serializer(ar, compressionEnabled); + serializer(ar, configurableEncryptionEnabled); } }; @@ -575,54 +575,44 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter { wPtr = mutateString(buffer); } - static void validateEncryptionHeader(Optional> headerCipherKey, - Reference textCipherKey, - BlobCipherEncryptHeader& header) { - // Validate encryption header 'cipherHeader' details - if (header.cipherHeaderDetails.isValid() && - (!headerCipherKey.present() || header.cipherHeaderDetails != headerCipherKey.get()->details())) { - TraceEvent(SevWarn, "EncryptionHeader_CipherHeaderMismatch") - .detail("HeaderDomainId", headerCipherKey.get()->getDomainId()) - .detail("ExpectedHeaderDomainId", header.cipherHeaderDetails.encryptDomainId) - .detail("HeaderBaseCipherId", headerCipherKey.get()->getBaseCipherId()) - .detail("ExpectedHeaderBaseCipherId", header.cipherHeaderDetails.baseCipherId) - .detail("HeaderSalt", headerCipherKey.get()->getSalt()) - .detail("ExpectedHeaderSalt", header.cipherHeaderDetails.salt); - throw encrypt_header_metadata_mismatch(); - } - - // Validate encryption text 'cipherText' details sanity - if (!header.cipherTextDetails.isValid() || header.cipherTextDetails != textCipherKey->details()) { - TraceEvent(SevWarn, "EncryptionHeader_CipherTextMismatch") - .detail("TextDomainId", textCipherKey->getDomainId()) - .detail("ExpectedTextDomainId", header.cipherTextDetails.encryptDomainId) - .detail("TextBaseCipherId", textCipherKey->getBaseCipherId()) - .detail("ExpectedTextBaseCipherId", header.cipherTextDetails.baseCipherId) - .detail("TextSalt", textCipherKey->getSalt()) - .detail("ExpectedTextSalt", header.cipherTextDetails.salt); - throw encrypt_header_metadata_mismatch(); - } - } - - ACTOR static Future decryptImpl(Database cx, - BlobCipherEncryptHeader header, - const uint8_t* dataP, - int64_t dataLen, - Arena* arena) { + ACTOR static Future decryptImpl( + Database cx, + std::variant headerVariant, + const uint8_t* dataP, + int64_t dataLen, + Arena* arena) { Reference const> dbInfo = cx->clientInfo; - TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, header, BlobCipherMetrics::RESTORE)); - validateEncryptionHeader(cipherKeys.cipherHeaderKey, cipherKeys.cipherTextKey, header); - DecryptBlobCipherAes256Ctr decryptor( - cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.iv, BlobCipherMetrics::BACKUP); - return decryptor.decrypt(dataP, dataLen, header, *arena)->toStringRef(); + if (std::holds_alternative(headerVariant)) { // configurable encryption + state BlobCipherEncryptHeaderRef headerRef = std::get(headerVariant); + TextAndHeaderCipherKeys cipherKeys = + wait(getEncryptCipherKeys(dbInfo, headerRef, BlobCipherMetrics::RESTORE)); + EncryptHeaderCipherDetails cipherDetails = headerRef.getCipherDetails(); + cipherDetails.textCipherDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherTextKey); + if (cipherDetails.headerCipherDetails.present()) { + cipherDetails.headerCipherDetails.get().validateCipherDetailsWithCipherKey(cipherKeys.cipherHeaderKey); + } + DecryptBlobCipherAes256Ctr decryptor( + cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, headerRef.getIV(), BlobCipherMetrics::RESTORE); + return decryptor.decrypt(dataP, dataLen, headerRef, *arena); + } else { + state BlobCipherEncryptHeader header = std::get(headerVariant); + TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, header, BlobCipherMetrics::RESTORE)); + header.cipherTextDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherTextKey); + if (header.cipherHeaderDetails.isValid()) { + header.cipherHeaderDetails.validateCipherDetailsWithCipherKey(cipherKeys.cipherHeaderKey); + } + DecryptBlobCipherAes256Ctr decryptor( + cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.iv, BlobCipherMetrics::RESTORE); + return decryptor.decrypt(dataP, dataLen, header, *arena)->toStringRef(); + } } static Future decrypt(Database cx, - BlobCipherEncryptHeader headerS, + std::variant header, const uint8_t* dataP, int64_t dataLen, Arena* arena) { - return decryptImpl(cx, headerS, dataP, dataLen, arena); + return decryptImpl(cx, header, dataP, dataLen, arena); } ACTOR static Future> refreshKey(EncryptedRangeFileWriter* self, @@ -655,12 +645,26 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter { AES_256_IV_LENGTH, getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE), BlobCipherMetrics::BACKUP); - Arena arena; int64_t payloadSize = self->wPtr - self->dataPayloadStart; - auto encryptedData = encryptor.encrypt(self->dataPayloadStart, payloadSize, self->encryptHeader, arena); + StringRef encryptedData; + if (self->options.configurableEncryptionEnabled) { + BlobCipherEncryptHeaderRef headerRef; + encryptedData = encryptor.encrypt(self->dataPayloadStart, payloadSize, &headerRef, *self->arena); + Standalone serialized = BlobCipherEncryptHeaderRef::toStringRef(headerRef); + self->arena->dependsOn(serialized.arena()); + ASSERT(serialized.size() == self->encryptHeader.size()); + std::memcpy(mutateString(self->encryptHeader), serialized.begin(), self->encryptHeader.size()); + } else { + BlobCipherEncryptHeader header; + encryptedData = + encryptor.encrypt(self->dataPayloadStart, payloadSize, &header, *self->arena)->toStringRef(); + StringRef encryptHeaderStringRef = BlobCipherEncryptHeader::toStringRef(header, *self->arena); + ASSERT(encryptHeaderStringRef.size() == self->encryptHeader.size()); + std::memcpy(mutateString(self->encryptHeader), encryptHeaderStringRef.begin(), self->encryptHeader.size()); + } // re-write encrypted data to buffer - std::memcpy(self->dataPayloadStart, encryptedData->begin(), payloadSize); + std::memcpy(self->dataPayloadStart, encryptedData.begin(), payloadSize); return Void(); } @@ -767,13 +771,32 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter { copyToBuffer(self, (uint8_t*)&self->fileVersion, sizeof(self->fileVersion)); // write options struct + self->options.configurableEncryptionEnabled = CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION; Value serialized = ObjectWriter::toValue(self->options, IncludeVersion(ProtocolVersion::withEncryptedSnapshotBackupFile())); appendStringRefWithLenToBuffer(self, &serialized); + // calculate encryption header size + uint32_t headerSize = 0; + if (self->options.configurableEncryptionEnabled) { + EncryptAuthTokenMode authTokenMode = + getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE); + EncryptAuthTokenAlgo authTokenAlgo = getAuthTokenAlgoFromMode(authTokenMode); + headerSize = BlobCipherEncryptHeaderRef::getHeaderSize( + CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION, + getEncryptCurrentAlgoHeaderVersion(authTokenMode, authTokenAlgo), + ENCRYPT_CIPHER_MODE_AES_256_CTR, + authTokenMode, + authTokenAlgo); + } else { + headerSize = BlobCipherEncryptHeader::headerSize; + } + ASSERT(headerSize > 0); + // write header size to buffer + copyToBuffer(self, (uint8_t*)&headerSize, sizeof(headerSize)); // leave space for encryption header - self->encryptHeader = (BlobCipherEncryptHeader*)self->wPtr; - self->wPtr += BlobCipherEncryptHeader::headerSize; + self->encryptHeader = StringRef(self->wPtr, headerSize); + self->wPtr += headerSize; self->dataPayloadStart = self->wPtr; // If this is NOT the first block then write duplicate stuff needed from last block @@ -913,7 +936,7 @@ struct EncryptedRangeFileWriter : public IRangeFileWriter { private: Standalone buffer; uint8_t* wPtr; - BlobCipherEncryptHeader* encryptHeader; + StringRef encryptHeader; uint8_t* dataPayloadStart; int64_t blockEnd; uint32_t fileVersion; @@ -1050,7 +1073,7 @@ ACTOR static Future decodeKVPairs(StringRefReader* reader, Standalone>* results, bool encryptedBlock, EncryptionAtRestMode encryptMode, - Optional encryptHeader, + Optional blockDomainId, Optional>> tenantCache) { // Read begin key, if this fails then block was invalid. state uint32_t kLen = reader->consumeNetworkUInt32(); @@ -1068,9 +1091,9 @@ ACTOR static Future decodeKVPairs(StringRefReader* reader, // make sure that all keys in a block belong to exactly one tenant, // unless its the last key in which case it can be a truncated (different) tenant prefix - if (encryptedBlock && g_network && g_network->isSimulated() && - !isReservedEncryptDomain(encryptHeader.get().cipherTextDetails.encryptDomainId)) { - ASSERT(encryptHeader.present()); + ASSERT(!encryptedBlock || blockDomainId.present()); + if (CLIENT_KNOBS->SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS && encryptedBlock && g_network && + g_network->isSimulated() && !isReservedEncryptDomain(blockDomainId.get())) { state KeyRef curKey = KeyRef(k, kLen); if (!prevDomainId.present()) { EncryptCipherDomainId domainId = @@ -1088,7 +1111,7 @@ ACTOR static Future decodeKVPairs(StringRefReader* reader, } // make sure that all keys (except possibly the last key) in a block are encrypted using the correct key if (!prevKey.empty()) { - ASSERT(prevDomainId.get() == encryptHeader.get().cipherTextDetails.encryptDomainId); + ASSERT(prevDomainId.get() == blockDomainId.get()); } prevKey = curKey; prevDomainId = curDomainId; @@ -1152,15 +1175,14 @@ ACTOR Future>> decodeRangeFileBlock(Reference< wait(tenantCache.get()->init()); } state EncryptionAtRestMode encryptMode = config.encryptionAtRestMode; - state int64_t blockTenantId = TenantInfo::INVALID_TENANT; + state int64_t blockDomainId = TenantInfo::INVALID_TENANT; try { // Read header, currently only decoding BACKUP_AGENT_SNAPSHOT_FILE_VERSION or // BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION int32_t file_version = reader.consume(); if (file_version == BACKUP_AGENT_SNAPSHOT_FILE_VERSION) { - wait( - decodeKVPairs(&reader, &results, false, encryptMode, Optional(), tenantCache)); + wait(decodeKVPairs(&reader, &results, false, encryptMode, Optional(), tenantCache)); } else if (file_version == BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION) { CODE_PROBE(true, "decoding encrypted block"); // decode options struct @@ -1169,39 +1191,50 @@ ACTOR Future>> decodeRangeFileBlock(Reference< StringRef optionsStringRef = StringRef(o, optionsLen); EncryptedRangeFileWriter::Options options = ObjectReader::fromStringRef(optionsStringRef, IncludeVersion()); - ASSERT(!options.compressionEnabled); + // read header size + state uint32_t headerLen = reader.consume(); + // read the encryption header + state const uint8_t* headerStart = reader.consume(headerLen); + StringRef headerS = StringRef(headerStart, headerLen); + state std::variant encryptHeader; + if (options.configurableEncryptionEnabled) { + encryptHeader = BlobCipherEncryptHeaderRef::fromStringRef(headerS); + blockDomainId = std::get(encryptHeader) + .getCipherDetails() + .textCipherDetails.encryptDomainId; + } else { + encryptHeader = BlobCipherEncryptHeader::fromStringRef(headerS); + blockDomainId = std::get(encryptHeader).cipherTextDetails.encryptDomainId; + } - // read encryption header - state const uint8_t* headerStart = reader.consume(BlobCipherEncryptHeader::headerSize); - StringRef headerS = StringRef(headerStart, BlobCipherEncryptHeader::headerSize); - state BlobCipherEncryptHeader header = BlobCipherEncryptHeader::fromStringRef(headerS); - blockTenantId = header.cipherTextDetails.encryptDomainId; - if (config.tenantMode == TenantMode::REQUIRED && !isReservedEncryptDomain(blockTenantId)) { + if (config.tenantMode == TenantMode::REQUIRED && !isReservedEncryptDomain(blockDomainId)) { ASSERT(tenantCache.present()); - Optional> payload = wait(tenantCache.get()->getById(blockTenantId)); + Optional> payload = wait(tenantCache.get()->getById(blockDomainId)); if (!payload.present()) { throw tenant_not_found(); } } - const uint8_t* dataPayloadStart = headerStart + BlobCipherEncryptHeader::headerSize; + const uint8_t* dataPayloadStart = headerStart + headerLen; // calculate the total bytes read up to (and including) the header - int64_t bytesRead = sizeof(int32_t) + sizeof(uint32_t) + optionsLen + BlobCipherEncryptHeader::headerSize; + int64_t bytesRead = sizeof(int32_t) + sizeof(uint32_t) + sizeof(uint32_t) + optionsLen + headerLen; // get the size of the encrypted payload and decrypt it int64_t dataLen = len - bytesRead; StringRef decryptedData = - wait(EncryptedRangeFileWriter::decrypt(cx, header, dataPayloadStart, dataLen, &results.arena())); + wait(EncryptedRangeFileWriter::decrypt(cx, encryptHeader, dataPayloadStart, dataLen, &results.arena())); reader = StringRefReader(decryptedData, restore_corrupted_data()); - wait(decodeKVPairs(&reader, &results, true, encryptMode, header, tenantCache)); + wait(decodeKVPairs(&reader, &results, true, encryptMode, blockDomainId, tenantCache)); } else { throw restore_unsupported_file_version(); } return results; } catch (Error& e) { if (e.code() == error_code_encrypt_keys_fetch_failed) { - TraceEvent(SevWarnAlways, "SnapshotRestoreEncryptKeyFetchFailed").detail("TenantId", blockTenantId); + ASSERT(!isReservedEncryptDomain(blockDomainId)); + TraceEvent(SevWarnAlways, "SnapshotRestoreEncryptKeyFetchFailed").detail("TenantId", blockDomainId); CODE_PROBE(true, "Snapshot restore encrypt keys not found"); } else if (e.code() == error_code_tenant_not_found) { - TraceEvent(SevWarnAlways, "EncryptedSnapshotRestoreTenantNotFound").detail("TenantId", blockTenantId); + ASSERT(!isReservedEncryptDomain(blockDomainId)); + TraceEvent(SevWarnAlways, "EncryptedSnapshotRestoreTenantNotFound").detail("TenantId", blockDomainId); CODE_PROBE(true, "Encrypted Snapshot restore tenant not found"); } TraceEvent(SevWarn, "FileRestoreDecodeRangeFileBlockFailed") diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 9c9a517baf..f65fce885e 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -549,6 +549,21 @@ ThreadFuture DLTenant::blobbifyRange(const KeyRangeRef& keyRange) { }); } +ThreadFuture DLTenant::blobbifyRangeBlocking(const KeyRangeRef& keyRange) { + if (!api->tenantBlobbifyRangeBlocking) { + return unsupported_operation(); + } + + FdbCApi::FDBFuture* f = api->tenantBlobbifyRangeBlocking( + tenant, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size()); + + return toThreadFuture(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { + FdbCApi::fdb_bool_t ret = false; + ASSERT(!api->futureGetBool(f, &ret)); + return ret; + }); +} + ThreadFuture DLTenant::unblobbifyRange(const KeyRangeRef& keyRange) { if (!api->tenantUnblobbifyRange) { return unsupported_operation(); @@ -601,6 +616,28 @@ ThreadFuture DLTenant::verifyBlobRange(const KeyRangeRef& keyRange, Opt }); } +ThreadFuture DLTenant::flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional version) { + if (!api->tenantFlushBlobRange) { + return unsupported_operation(); + } + + Version readVersion = version.present() ? version.get() : latestVersion; + + FdbCApi::FDBFuture* f = api->tenantFlushBlobRange(tenant, + keyRange.begin.begin(), + keyRange.begin.size(), + keyRange.end.begin(), + keyRange.end.size(), + compact, + readVersion); + + return toThreadFuture(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { + FdbCApi::fdb_bool_t ret = false; + ASSERT(!api->futureGetBool(f, &ret)); + return ret; + }); +} + // DLDatabase DLDatabase::DLDatabase(Reference api, ThreadFuture dbFuture) : api(api), db(nullptr) { addref(); @@ -768,6 +805,21 @@ ThreadFuture DLDatabase::blobbifyRange(const KeyRangeRef& keyRange) { }); } +ThreadFuture DLDatabase::blobbifyRangeBlocking(const KeyRangeRef& keyRange) { + if (!api->databaseBlobbifyRangeBlocking) { + return unsupported_operation(); + } + + FdbCApi::FDBFuture* f = api->databaseBlobbifyRangeBlocking( + db, keyRange.begin.begin(), keyRange.begin.size(), keyRange.end.begin(), keyRange.end.size()); + + return toThreadFuture(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { + FdbCApi::fdb_bool_t ret = false; + ASSERT(!api->futureGetBool(f, &ret)); + return ret; + }); +} + ThreadFuture DLDatabase::unblobbifyRange(const KeyRangeRef& keyRange) { if (!api->databaseUnblobbifyRange) { return unsupported_operation(); @@ -820,6 +872,28 @@ ThreadFuture DLDatabase::verifyBlobRange(const KeyRangeRef& keyRange, O }); } +ThreadFuture DLDatabase::flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional version) { + if (!api->databaseFlushBlobRange) { + return unsupported_operation(); + } + + Version readVersion = version.present() ? version.get() : latestVersion; + + FdbCApi::FDBFuture* f = api->databaseFlushBlobRange(db, + keyRange.begin.begin(), + keyRange.begin.size(), + keyRange.end.begin(), + keyRange.end.size(), + compact, + readVersion); + + return toThreadFuture(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { + FdbCApi::fdb_bool_t ret = false; + ASSERT(!api->futureGetBool(f, &ret)); + return ret; + }); +} + ThreadFuture> DLDatabase::getClientStatus() { if (!api->databaseGetClientStatus) { return unsupported_operation(); @@ -931,6 +1005,11 @@ void DLApi::init() { fdbCPath, "fdb_database_blobbify_range", headerVersion >= ApiVersion::withBlobRangeApi().version()); + loadClientFunction(&api->databaseBlobbifyRangeBlocking, + lib, + fdbCPath, + "fdb_database_blobbify_range_blocking", + headerVersion >= ApiVersion::withTenantBlobRangeApi().version()); loadClientFunction(&api->databaseUnblobbifyRange, lib, fdbCPath, @@ -946,6 +1025,11 @@ void DLApi::init() { fdbCPath, "fdb_database_verify_blob_range", headerVersion >= ApiVersion::withBlobRangeApi().version()); + loadClientFunction(&api->databaseFlushBlobRange, + lib, + fdbCPath, + "fdb_database_flush_blob_range", + headerVersion >= ApiVersion::withTenantBlobRangeApi().version()); loadClientFunction(&api->databaseGetClientStatus, lib, fdbCPath, @@ -968,6 +1052,11 @@ void DLApi::init() { fdbCPath, "fdb_tenant_blobbify_range", headerVersion >= ApiVersion::withTenantBlobRangeApi().version()); + loadClientFunction(&api->tenantBlobbifyRangeBlocking, + lib, + fdbCPath, + "fdb_tenant_blobbify_range_blocking", + headerVersion >= ApiVersion::withTenantBlobRangeApi().version()); loadClientFunction(&api->tenantUnblobbifyRange, lib, fdbCPath, @@ -983,6 +1072,11 @@ void DLApi::init() { fdbCPath, "fdb_tenant_verify_blob_range", headerVersion >= ApiVersion::withTenantBlobRangeApi().version()); + loadClientFunction(&api->tenantFlushBlobRange, + lib, + fdbCPath, + "fdb_tenant_flush_blob_range", + headerVersion >= ApiVersion::withTenantBlobRangeApi().version()); loadClientFunction(&api->tenantGetId, lib, fdbCPath, "fdb_tenant_get_id", headerVersion >= 730); loadClientFunction(&api->tenantDestroy, lib, fdbCPath, "fdb_tenant_destroy", headerVersion >= 710); @@ -1837,6 +1931,10 @@ ThreadFuture MultiVersionTenant::blobbifyRange(const KeyRangeRef& keyRange return executeOperation(&ITenant::blobbifyRange, keyRange); } +ThreadFuture MultiVersionTenant::blobbifyRangeBlocking(const KeyRangeRef& keyRange) { + return executeOperation(&ITenant::blobbifyRangeBlocking, keyRange); +} + ThreadFuture MultiVersionTenant::unblobbifyRange(const KeyRangeRef& keyRange) { return executeOperation(&ITenant::unblobbifyRange, keyRange); } @@ -1850,6 +1948,13 @@ ThreadFuture MultiVersionTenant::verifyBlobRange(const KeyRangeRef& key return executeOperation(&ITenant::verifyBlobRange, keyRange, std::forward>(version)); } +ThreadFuture MultiVersionTenant::flushBlobRange(const KeyRangeRef& keyRange, + bool compact, + Optional version) { + return executeOperation( + &ITenant::flushBlobRange, keyRange, std::forward(compact), std::forward>(version)); +} + MultiVersionTenant::TenantState::TenantState(Reference db, TenantNameRef tenantName) : tenantVar(new ThreadSafeAsyncVar>(Reference(nullptr))), tenantName(tenantName), db(db), closed(false) { @@ -2071,6 +2176,10 @@ ThreadFuture MultiVersionDatabase::blobbifyRange(const KeyRangeRef& keyRan return executeOperation(&IDatabase::blobbifyRange, keyRange); } +ThreadFuture MultiVersionDatabase::blobbifyRangeBlocking(const KeyRangeRef& keyRange) { + return executeOperation(&IDatabase::blobbifyRangeBlocking, keyRange); +} + ThreadFuture MultiVersionDatabase::unblobbifyRange(const KeyRangeRef& keyRange) { return executeOperation(&IDatabase::unblobbifyRange, keyRange); } @@ -2084,6 +2193,13 @@ ThreadFuture MultiVersionDatabase::verifyBlobRange(const KeyRangeRef& k return executeOperation(&IDatabase::verifyBlobRange, keyRange, std::forward>(version)); } +ThreadFuture MultiVersionDatabase::flushBlobRange(const KeyRangeRef& keyRange, + bool compact, + Optional version) { + return executeOperation( + &IDatabase::flushBlobRange, keyRange, std::forward(compact), std::forward>(version)); +} + // Returns the protocol version reported by the coordinator this client is connected to // If an expected version is given, the future won't return until the protocol version is different than expected // Note: this will never return if the server is running a protocol from FDB 5.0 or older diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 3ee9f8197b..8daaa0d76f 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -47,6 +47,7 @@ #include "fdbclient/AnnotateActor.h" #include "fdbclient/Atomic.h" #include "fdbclient/BlobGranuleCommon.h" +#include "fdbclient/BlobGranuleRequest.actor.h" #include "fdbclient/ClusterInterface.h" #include "fdbclient/ClusterConnectionFile.h" #include "fdbclient/ClusterConnectionMemoryRecord.h" @@ -8526,6 +8527,41 @@ Future DatabaseContext::verifyBlobRange(const KeyRange& range, return verifyBlobRangeActor(Reference::addRef(this), range, version, tenant); } +ACTOR Future flushBlobRangeActor(Reference cx, + KeyRange range, + bool compact, + Optional version, + Optional> tenant) { + if (tenant.present()) { + wait(tenant.get()->ready()); + range = range.withPrefix(tenant.get()->prefix()); + } + state Database db(cx); + if (!version.present()) { + state Transaction tr(db); + Version _v = wait(tr.getReadVersion()); + version = _v; + } + FlushGranuleRequest req(-1, range, version.get(), compact); + try { + wait(success(doBlobGranuleRequests(db, range, req, &BlobWorkerInterface::flushGranuleRequest))); + return true; + } catch (Error& e) { + if (e.code() == error_code_blob_granule_transaction_too_old) { + // can't flush data at this version, because no granules + return false; + } + throw e; + } +} + +Future DatabaseContext::flushBlobRange(const KeyRange& range, + bool compact, + Optional version, + Optional> tenant) { + return flushBlobRangeActor(Reference::addRef(this), range, compact, version, tenant); +} + ACTOR Future>> readStorageWiggleValues(Database cx, bool primary, bool use_system_priority) { @@ -10928,8 +10964,39 @@ ACTOR Future setBlobRangeActor(Reference cx, } } +ACTOR Future blobbifyRangeActor(Reference cx, + KeyRange range, + bool doWait, + Optional> tenant) { + if (BG_REQUEST_DEBUG) { + fmt::print("BlobbifyRange [{0} - {1}) ({2})\n", range.begin.printable(), range.end.printable(), doWait); + } + state bool result = wait(setBlobRangeActor(cx, range, true, tenant)); + if (!doWait || !result) { + return result; + } + // FIXME: add blob worker verifyRange rpc call that just waits for granule to become readable at any version + loop { + Version verifyVersion = wait(cx->verifyBlobRange(range, latestVersion, tenant)); + if (verifyVersion != invalidVersion) { + if (BG_REQUEST_DEBUG) { + fmt::print("BlobbifyRange [{0} - {1}) got complete @ {2}\n", + range.begin.printable(), + range.end.printable(), + verifyVersion); + } + return result; + } + wait(delay(0.1)); + } +} + Future DatabaseContext::blobbifyRange(KeyRange range, Optional> tenant) { - return setBlobRangeActor(Reference::addRef(this), range, true, tenant); + return blobbifyRangeActor(Reference::addRef(this), range, false, tenant); +} + +Future DatabaseContext::blobbifyRangeBlocking(KeyRange range, Optional> tenant) { + return blobbifyRangeActor(Reference::addRef(this), range, true, tenant); } Future DatabaseContext::unblobbifyRange(KeyRange range, Optional> tenant) { diff --git a/fdbclient/RESTClient.actor.cpp b/fdbclient/RESTClient.actor.cpp index 3758bdd79d..b44c3cbaa6 100644 --- a/fdbclient/RESTClient.actor.cpp +++ b/fdbclient/RESTClient.actor.cpp @@ -40,6 +40,16 @@ #include "flow/actorcompiler.h" // always the last include +#define ENABLE_VERBOSE_DEBUG true + +#define TRACE_REST_OP(opName, url, secure) \ + do { \ + if (ENABLE_VERBOSE_DEBUG) { \ + const std::string urlStr = url.toString(); \ + TraceEvent("RESTClientOp").detail("Op", #opName).detail("Url", urlStr).detail("IsSecure", secure); \ + } \ + } while (0); + json_spirit::mObject RESTClient::Stats::getJSON() { json_spirit::mObject o; @@ -61,10 +71,13 @@ RESTClient::Stats RESTClient::Stats::operator-(const Stats& rhs) { return r; } -RESTClient::RESTClient() {} +RESTClient::RESTClient() { + conectionPool = makeReference(knobs.connection_pool_size); +} RESTClient::RESTClient(std::unordered_map& knobSettings) { knobs.set(knobSettings); + conectionPool = makeReference(knobs.connection_pool_size); } void RESTClient::setKnobs(const std::unordered_map& knobSettings) { @@ -83,6 +96,10 @@ ACTOR Future> doRequest_impl(Reference cli state UnsentPacketQueue content; state int contentLen = url->body.size(); + if (ENABLE_VERBOSE_DEBUG) { + TraceEvent(SevDebug, "DoRequestImpl").detail("Url", url->toString()); + } + if (url->body.size() > 0) { PacketWriter pw(content.getWriteBuffer(url->body.size()), nullptr, Unversioned()); pw.serializeBytes(url->body); @@ -166,7 +183,7 @@ ACTOR Future> doRequest_impl(Reference cli // But only if our previous attempt was not the last allowable try. retryable = retryable && (thisTry < maxTries); - TraceEvent event(SevWarn, retryable ? "RESTClient_FailedRetryable" : "RESTClient_RequestFailed"); + TraceEvent event(SevWarn, retryable ? "RESTClientFailedRetryable" : "RESTClientRequestFailed"); // Attach err to trace event if present, otherwise extract some stuff from the response if (err.present()) { @@ -269,6 +286,7 @@ Future> RESTClient::doPost(const std::string& fullUrl, const std::string& requestBody, Optional optHeaders) { RESTUrl url(fullUrl, requestBody, knobs.secure_connection); + TRACE_REST_OP("DoPost", url, knobs.secure_connection); return doPutOrPost(HTTP::HTTP_VERB_POST, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK }); } @@ -276,6 +294,7 @@ Future> RESTClient::doPut(const std::string& fullUrl, const std::string& requestBody, Optional optHeaders) { RESTUrl url(fullUrl, requestBody, knobs.secure_connection); + TRACE_REST_OP("DoPut", url, knobs.secure_connection); return doPutOrPost( HTTP::HTTP_VERB_PUT, optHeaders, @@ -299,16 +318,19 @@ Future> RESTClient::doGetHeadDeleteOrTrace(const std:: Future> RESTClient::doGet(const std::string& fullUrl, Optional optHeaders) { RESTUrl url(fullUrl, knobs.secure_connection); + TRACE_REST_OP("DoGet", url, knobs.secure_connection); return doGetHeadDeleteOrTrace(HTTP::HTTP_VERB_GET, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK }); } Future> RESTClient::doHead(const std::string& fullUrl, Optional optHeaders) { RESTUrl url(fullUrl, knobs.secure_connection); + TRACE_REST_OP("DoHead", url, knobs.secure_connection); return doGetHeadDeleteOrTrace(HTTP::HTTP_VERB_HEAD, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK }); } Future> RESTClient::doDelete(const std::string& fullUrl, Optional optHeaders) { RESTUrl url(fullUrl, knobs.secure_connection); + TRACE_REST_OP("DoDelete", url, knobs.secure_connection); return doGetHeadDeleteOrTrace( HTTP::HTTP_VERB_DELETE, optHeaders, @@ -321,6 +343,7 @@ Future> RESTClient::doDelete(const std::string& fullUr Future> RESTClient::doTrace(const std::string& fullUrl, Optional optHeaders) { RESTUrl url(fullUrl, knobs.secure_connection); + TRACE_REST_OP("DoTrace", url, knobs.secure_connection); return doGetHeadDeleteOrTrace( HTTP::HTTP_VERB_TRACE, optHeaders, std::addressof(url), { HTTP::HTTP_STATUS_CODE_OK }); } diff --git a/fdbclient/RESTUtils.actor.cpp b/fdbclient/RESTUtils.actor.cpp index 684fea3f76..93c907b1d1 100644 --- a/fdbclient/RESTUtils.actor.cpp +++ b/fdbclient/RESTUtils.actor.cpp @@ -25,6 +25,7 @@ #include "flow/IConnection.h" #include +#include #include "flow/actorcompiler.h" // always the last include @@ -66,12 +67,12 @@ RESTClientKnobs::RESTClientKnobs() { } void RESTClientKnobs::set(const std::unordered_map& knobSettings) { - TraceEvent trace = TraceEvent("RESTClient_SetKnobs"); + TraceEvent trace = TraceEvent("RESTClientSetKnobs"); for (const auto& itr : knobSettings) { const auto& kItr = RESTClientKnobs::knobMap.find(itr.first); if (kItr == RESTClientKnobs::knobMap.end()) { - trace.detail("RESTClient_InvalidKnobName", itr.first); + trace.detail("RESTClientInvalidKnobName", itr.first); throw rest_invalid_rest_client_knob(); } *(kItr->second) = itr.second; @@ -98,28 +99,37 @@ ACTOR Future connect_impl(ReferenceconnectionPoolMap.find(connectKey); - if (poolItr == connectionPool->connectionPoolMap.end()) { - throw rest_connectpool_key_not_found(); - } - - while (!poolItr->second.empty()) { + while (poolItr != connectionPool->connectionPoolMap.end() && !poolItr->second.empty()) { RESTConnectionPool::ReusableConnection rconn = poolItr->second.front(); poolItr->second.pop(); if (rconn.expirationTime > now()) { - TraceEvent("RESTClient_ReusableConnection") + TraceEvent("RESTClientReuseConn") .suppressFor(60) + .detail("Host", connectKey.first) + .detail("Service", connectKey.second) .detail("RemoteEndpoint", rconn.conn->getPeerAddress()) .detail("ExpireIn", rconn.expirationTime - now()); return rconn; } } + // No valid connection exists, create a new one state Reference conn = wait(INetworkConnections::net()->connect(connectKey.first, connectKey.second, isSecure)); wait(conn->connectHandshake()); - return RESTConnectionPool::ReusableConnection({ conn, now() + maxConnLife }); + RESTConnectionPool::ReusableConnection reusableConn = + RESTConnectionPool::ReusableConnection({ conn, now() + maxConnLife }); + connectionPool->connectionPoolMap.insert( + { connectKey, std::queue({ reusableConn }) }); + + TraceEvent("RESTClientCreateNewConn") + .suppressFor(60) + .detail("Host", connectKey.first) + .detail("Service", connectKey.second) + .detail("RemoteEndpoint", conn->getPeerAddress()); + return reusableConn; } Future RESTConnectionPool::connect(RESTConnectionPoolKey connectKey, @@ -188,14 +198,14 @@ void RESTUrl::parseUrl(const std::string& fullUrl, const bool isSecure) { host = h.toString(); service = hRef.eat().toString(); - TraceEvent("RESTClient_ParseURI") + TraceEvent("RESTClientParseURI") .detail("URI", fullUrl) .detail("Host", host) .detail("Service", service) .detail("Resource", resource) .detail("ReqParameters", reqParameters); } catch (std::string& err) { - TraceEvent("RESTClient_ParseError").detail("URI", fullUrl).detail("Error", err); + TraceEvent("RESTClientParseError").detail("URI", fullUrl).detail("Error", err); throw rest_invalid_uri(); } } diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 6e3e7ff5c3..138d2f7e02 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -302,7 +302,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD, 20 ); init( DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC, isSimulated ? 2 : 21 * 60 * 60 * 24 ); if(randomize && BUGGIFY) DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC = isSimulated ? 0: 120; init( DD_TENANT_AWARENESS_ENABLED, false ); - init( STORAGE_QUOTA_ENABLED, false ); if(isSimulated) STORAGE_QUOTA_ENABLED = deterministicRandom()->coinflip(); + init( STORAGE_QUOTA_ENABLED, true ); if(isSimulated) STORAGE_QUOTA_ENABLED = deterministicRandom()->coinflip(); init( TENANT_CACHE_LIST_REFRESH_INTERVAL, 2 ); if( randomize && BUGGIFY ) TENANT_CACHE_LIST_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10); init( TENANT_CACHE_STORAGE_USAGE_REFRESH_INTERVAL, 2 ); if( randomize && BUGGIFY ) TENANT_CACHE_STORAGE_USAGE_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10); init( TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL, 10 ); if( randomize && BUGGIFY ) TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10); @@ -987,13 +987,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init ( CLUSTER_RECOVERY_EVENT_NAME_PREFIX, "Master" ); // Encryption - init( ENABLE_ENCRYPTION, false ); if ( randomize && BUGGIFY ) ENABLE_ENCRYPTION = !ENABLE_ENCRYPTION; - init( ENCRYPTION_MODE, "AES-256-CTR" ); init( SIM_KMS_MAX_KEYS, 4096 ); init( ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH, 100000 ); - init( ENABLE_TLOG_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY && ENABLE_ENCRYPTION ) ENABLE_TLOG_ENCRYPTION = false; - init( ENABLE_STORAGE_SERVER_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY && ENABLE_ENCRYPTION) ENABLE_STORAGE_SERVER_ENCRYPTION = false; - init( ENABLE_BLOB_GRANULE_ENCRYPTION, ENABLE_ENCRYPTION ); if ( randomize && BUGGIFY && ENABLE_ENCRYPTION) ENABLE_BLOB_GRANULE_ENCRYPTION = false; // encrypt key proxy init( ENABLE_BLOB_GRANULE_COMPRESSION, false ); if ( randomize && BUGGIFY ) { ENABLE_BLOB_GRANULE_COMPRESSION = deterministicRandom()->coinflip(); } @@ -1060,6 +1055,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( BLOB_RESTORE_MANIFEST_URL, isSimulated ? "file://simfdb/fdbblob/manifest" : "" ); init( BLOB_RESTORE_MANIFEST_FILE_MAX_SIZE, isSimulated ? 10000 : 10000000 ); init( BLOB_RESTORE_MANIFEST_RETENTION_MAX, 10 ); + init( BLOB_RESTORE_MLOGS_RETENTION_SECS, isSimulated ? 120 : 3600 * 24 * 14 ); init( BGCC_TIMEOUT, isSimulated ? 10.0 : 120.0 ); init( BGCC_MIN_INTERVAL, isSimulated ? 1.0 : 10.0 ); diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index f497f6677c..97cab0ca64 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -1742,6 +1742,7 @@ Standalone decodeBlobRestoreArg(ValueRef const& value) { } const Key blobManifestVersionKey = "\xff\x02/blobManifestVersion"_sr; +const Key blobGranulesLastFlushKey = "\xff\x02/blobGranulesLastFlushTs"_sr; const KeyRangeRef idempotencyIdKeys("\xff\x02/idmp/"_sr, "\xff\x02/idmp0"_sr); const KeyRef idempotencyIdsExpiredVersion("\xff\x02/idmpExpiredVersion"_sr); diff --git a/fdbclient/Tenant.cpp b/fdbclient/Tenant.cpp index 39994cca41..0a70f09685 100644 --- a/fdbclient/Tenant.cpp +++ b/fdbclient/Tenant.cpp @@ -55,6 +55,17 @@ int64_t prefixToId(KeyRef prefix, EnforceValidTenantId enforceValidTenantId) { return id; } +KeyRangeRef clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena) { + if (tenantInfo.hasTenant()) { + return KeyRangeRef(range.begin.startsWith(tenantInfo.prefix.get()) ? range.begin : tenantInfo.prefix.get(), + range.end.startsWith(tenantInfo.prefix.get()) + ? range.end + : allKeys.end.withPrefix(tenantInfo.prefix.get(), arena)); + } else { + return range; + } +} + bool withinSingleTenant(KeyRangeRef const& range) { if (range.begin >= "\x80"_sr || range.begin.size() < TenantAPI::PREFIX_SIZE) { return false; @@ -63,7 +74,7 @@ bool withinSingleTenant(KeyRangeRef const& range) { return tRange.contains(range); } -}; // namespace TenantAPI +} // namespace TenantAPI std::string TenantAPI::tenantStateToString(TenantState tenantState) { switch (tenantState) { diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp index 19dc177de5..f0f2e74902 100644 --- a/fdbclient/ThreadSafeTransaction.cpp +++ b/fdbclient/ThreadSafeTransaction.cpp @@ -167,6 +167,15 @@ ThreadFuture ThreadSafeDatabase::blobbifyRange(const KeyRangeRef& keyRange }); } +ThreadFuture ThreadSafeDatabase::blobbifyRangeBlocking(const KeyRangeRef& keyRange) { + DatabaseContext* db = this->db; + KeyRange range = keyRange; + return onMainThread([=]() -> Future { + db->checkDeferredError(); + return db->blobbifyRangeBlocking(range); + }); +} + ThreadFuture ThreadSafeDatabase::unblobbifyRange(const KeyRangeRef& keyRange) { DatabaseContext* db = this->db; KeyRange range = keyRange; @@ -195,6 +204,17 @@ ThreadFuture ThreadSafeDatabase::verifyBlobRange(const KeyRangeRef& key }); } +ThreadFuture ThreadSafeDatabase::flushBlobRange(const KeyRangeRef& keyRange, + bool compact, + Optional version) { + DatabaseContext* db = this->db; + KeyRange range = keyRange; + return onMainThread([=]() -> Future { + db->checkDeferredError(); + return db->flushBlobRange(range, compact, version); + }); +} + ThreadSafeDatabase::ThreadSafeDatabase(ConnectionRecordType connectionRecordType, std::string connectionRecordString, int apiVersion) { @@ -277,6 +297,16 @@ ThreadFuture ThreadSafeTenant::blobbifyRange(const KeyRangeRef& keyRange) }); } +ThreadFuture ThreadSafeTenant::blobbifyRangeBlocking(const KeyRangeRef& keyRange) { + DatabaseContext* db = this->db->db; + KeyRange range = keyRange; + return onMainThread([=]() -> Future { + db->checkDeferredError(); + db->addref(); + return db->blobbifyRangeBlocking(range, Reference::addRef(tenant)); + }); +} + ThreadFuture ThreadSafeTenant::unblobbifyRange(const KeyRangeRef& keyRange) { DatabaseContext* db = this->db->db; KeyRange range = keyRange; @@ -308,6 +338,18 @@ ThreadFuture ThreadSafeTenant::verifyBlobRange(const KeyRangeRef& keyRa }); } +ThreadFuture ThreadSafeTenant::flushBlobRange(const KeyRangeRef& keyRange, + bool compact, + Optional version) { + DatabaseContext* db = this->db->db; + KeyRange range = keyRange; + return onMainThread([=]() -> Future { + db->checkDeferredError(); + db->addref(); + return db->flushBlobRange(range, compact, version, Reference::addRef(tenant)); + }); +} + ThreadSafeTenant::~ThreadSafeTenant() { Tenant* t = this->tenant; if (t) diff --git a/fdbclient/include/fdbclient/BlobCipher.h b/fdbclient/include/fdbclient/BlobCipher.h index 139643b680..f9de92ef17 100644 --- a/fdbclient/include/fdbclient/BlobCipher.h +++ b/fdbclient/include/fdbclient/BlobCipher.h @@ -154,6 +154,8 @@ private: uint8_t* buffer; }; +class BlobCipherKey; + #pragma pack(push, 1) // exact fit - no padding struct BlobCipherDetails { constexpr static FileIdentifier file_identifier = 1945731; @@ -163,7 +165,7 @@ struct BlobCipherDetails { // BaseCipher encryption key identifier EncryptCipherBaseKeyId baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID; // Random salt - EncryptCipherRandomSalt salt{}; + EncryptCipherRandomSalt salt = INVALID_ENCRYPT_RANDOM_SALT; static uint32_t getSize() { return sizeof(EncryptCipherDomainId) + sizeof(EncryptCipherBaseKeyId) + sizeof(EncryptCipherRandomSalt); @@ -175,6 +177,8 @@ struct BlobCipherDetails { const EncryptCipherRandomSalt& random) : encryptDomainId(dId), baseCipherId(bId), salt(random) {} + void validateCipherDetailsWithCipherKey(Reference headerCipherKey); + bool operator==(const BlobCipherDetails& o) const { return encryptDomainId == o.encryptDomainId && baseCipherId == o.baseCipherId && salt == o.salt; } @@ -209,6 +213,7 @@ struct EncryptHeaderCipherDetails { BlobCipherDetails textCipherDetails; Optional headerCipherDetails; + EncryptHeaderCipherDetails() = default; EncryptHeaderCipherDetails(const BlobCipherDetails& tCipherDetails) : textCipherDetails(tCipherDetails) {} EncryptHeaderCipherDetails(const BlobCipherDetails& tCipherDetails, const BlobCipherDetails& hCipherDetails) : textCipherDetails(tCipherDetails), headerCipherDetails(hCipherDetails) {} @@ -489,6 +494,7 @@ struct BlobCipherEncryptHeaderRef { const uint8_t* getIV() const; const EncryptHeaderCipherDetails getCipherDetails() const; EncryptAuthTokenMode getAuthTokenMode() const; + EncryptCipherDomainId getDomainId() const; void validateEncryptionHeaderDetails(const BlobCipherDetails& textCipherDetails, const BlobCipherDetails& headerCipherDetails, @@ -1005,5 +1011,6 @@ void computeAuthToken(const std::vector>& payl unsigned int digestMaxBufSz); EncryptAuthTokenMode getEncryptAuthTokenMode(const EncryptAuthTokenMode mode); +int getEncryptCurrentAlgoHeaderVersion(const EncryptAuthTokenMode mode, const EncryptAuthTokenAlgo algo); -#endif // FDBCLIENT_BLOB_CIPHER_H +#endif // FDBCLIENT_BLOB_CIPHER_H \ No newline at end of file diff --git a/fdbclient/include/fdbclient/BlobGranuleCommon.h b/fdbclient/include/fdbclient/BlobGranuleCommon.h index 2db4d3afd7..07efd05b11 100644 --- a/fdbclient/include/fdbclient/BlobGranuleCommon.h +++ b/fdbclient/include/fdbclient/BlobGranuleCommon.h @@ -197,6 +197,7 @@ struct BlobFilePointerRef { int64_t offset; int64_t length; int64_t fullFileLength; + Version fileVersion; Optional cipherKeysCtx; // Non-serializable fields @@ -205,25 +206,34 @@ struct BlobFilePointerRef { BlobFilePointerRef() {} - BlobFilePointerRef(Arena& to, const std::string& filename, int64_t offset, int64_t length, int64_t fullFileLength) - : filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength) {} + BlobFilePointerRef(Arena& to, + const std::string& filename, + int64_t offset, + int64_t length, + int64_t fullFileLength, + Version fileVersion) + : filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength), + fileVersion(fileVersion) {} BlobFilePointerRef(Arena& to, const std::string& filename, int64_t offset, int64_t length, int64_t fullFileLength, + Version fileVersion, Optional ciphKeysCtx) : filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength), - cipherKeysCtx(ciphKeysCtx) {} + fileVersion(fileVersion), cipherKeysCtx(ciphKeysCtx) {} BlobFilePointerRef(Arena& to, const std::string& filename, int64_t offset, int64_t length, int64_t fullFileLength, + Version fileVersion, Optional ciphKeysMeta) - : filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength) { + : filename(to, filename), offset(offset), length(length), fullFileLength(fullFileLength), + fileVersion(fileVersion) { if (ciphKeysMeta.present()) { cipherKeysMetaRef = BlobGranuleCipherKeysMetaRef(to, ciphKeysMeta.get()); } @@ -231,12 +241,12 @@ struct BlobFilePointerRef { template void serialize(Ar& ar) { - serializer(ar, filename, offset, length, fullFileLength, cipherKeysCtx); + serializer(ar, filename, offset, length, fullFileLength, fileVersion, cipherKeysCtx); } std::string toString() const { std::stringstream ss; - ss << filename.toString() << ":" << offset << ":" << length << ":" << fullFileLength; + ss << filename.toString() << ":" << offset << ":" << length << ":" << fullFileLength << "@" << fileVersion; if (cipherKeysCtx.present()) { ss << ":CipherKeysCtx:TextCipher:" << cipherKeysCtx.get().textCipherKey.encryptDomainId << ":" << cipherKeysCtx.get().textCipherKey.baseCipherId << ":" << cipherKeysCtx.get().textCipherKey.salt @@ -257,6 +267,7 @@ struct BlobGranuleChunkRef { constexpr static FileIdentifier file_identifier = 865198; KeyRangeRef keyRange; Version includedVersion; + // FIXME: remove snapshotVersion, it is deprecated with fileVersion in BlobFilePointerRef Version snapshotVersion; Optional snapshotFile; // not set if it's an incremental read VectorRef deltaFiles; diff --git a/fdbclient/include/fdbclient/ClientKnobs.h b/fdbclient/include/fdbclient/ClientKnobs.h index 551e478a8a..d8f9165e3c 100644 --- a/fdbclient/include/fdbclient/ClientKnobs.h +++ b/fdbclient/include/fdbclient/ClientKnobs.h @@ -302,6 +302,7 @@ public: // key_not_found errors for. If TenantInfo::INVALID_TENANT is contained within the list then no tenants will be // dropped. This Knob should ONLY be used in simulation for testing purposes std::string SIMULATION_EKP_TENANT_IDS_TO_DROP; + bool SIMULATION_ENABLE_SNAPSHOT_ENCRYPTION_CHECKS; bool ENABLE_CONFIGURABLE_ENCRYPTION; int ENCRYPT_HEADER_FLAGS_VERSION; int ENCRYPT_HEADER_AES_CTR_NO_AUTH_VERSION; diff --git a/fdbclient/include/fdbclient/CommitTransaction.h b/fdbclient/include/fdbclient/CommitTransaction.h index c714b1f3ac..ccce782729 100644 --- a/fdbclient/include/fdbclient/CommitTransaction.h +++ b/fdbclient/include/fdbclient/CommitTransaction.h @@ -30,6 +30,8 @@ #include "flow/EncryptUtils.h" #include "flow/Knobs.h" +#include + // The versioned message has wire format : -1, version, messages static const int32_t VERSION_HEADER = -1; @@ -143,6 +145,38 @@ struct MutationRef { return reinterpret_cast(param1.begin()); } + const BlobCipherEncryptHeaderRef configurableEncryptionHeader() const { + ASSERT(isEncrypted()); + return BlobCipherEncryptHeaderRef::fromStringRef(param1); + } + + EncryptCipherDomainId encryptDomainId() const { + ASSERT(isEncrypted()); + return CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION ? configurableEncryptionHeader().getDomainId() + : encryptionHeader()->cipherTextDetails.encryptDomainId; + } + + void updateEncryptCipherDetails(std::unordered_set& cipherDetails) { + ASSERT(isEncrypted()); + + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + BlobCipherEncryptHeaderRef header = configurableEncryptionHeader(); + EncryptHeaderCipherDetails details = header.getCipherDetails(); + ASSERT(details.textCipherDetails.isValid()); + cipherDetails.insert(details.textCipherDetails); + if (details.headerCipherDetails.present()) { + ASSERT(details.headerCipherDetails.get().isValid()); + cipherDetails.insert(details.headerCipherDetails.get()); + } + } else { + const BlobCipherEncryptHeader* header = encryptionHeader(); + cipherDetails.insert(header->cipherTextDetails); + if (header->cipherHeaderDetails.isValid()) { + cipherDetails.insert(header->cipherHeaderDetails); + } + } + } + MutationRef encrypt(TextAndHeaderCipherKeys cipherKeys, Arena& arena, BlobCipherMetrics::UsageType usageType) const { @@ -150,6 +184,7 @@ struct MutationRef { deterministicRandom()->randomBytes(iv, AES_256_IV_LENGTH); BinaryWriter bw(AssumeVersion(ProtocolVersion::withEncryptionAtRest())); bw << *this; + EncryptBlobCipherAes265Ctr cipher( cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, @@ -157,11 +192,22 @@ struct MutationRef { AES_256_IV_LENGTH, getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE), usageType); - BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader; - StringRef headerRef(reinterpret_cast(header), sizeof(BlobCipherEncryptHeader)); - StringRef payload = - cipher.encrypt(static_cast(bw.getData()), bw.getLength(), header, arena)->toStringRef(); - return MutationRef(Encrypted, headerRef, payload); + + StringRef serializedHeader; + StringRef payload; + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + BlobCipherEncryptHeaderRef header; + payload = cipher.encrypt(static_cast(bw.getData()), bw.getLength(), &header, arena); + Standalone headerStr = BlobCipherEncryptHeaderRef::toStringRef(header); + arena.dependsOn(headerStr.arena()); + serializedHeader = headerStr; + } else { + BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader; + serializedHeader = StringRef(reinterpret_cast(header), sizeof(BlobCipherEncryptHeader)); + payload = + cipher.encrypt(static_cast(bw.getData()), bw.getLength(), header, arena)->toStringRef(); + } + return MutationRef(Encrypted, serializedHeader, payload); } MutationRef encrypt(const std::unordered_map>& cipherKeys, @@ -183,6 +229,7 @@ struct MutationRef { deterministicRandom()->randomBytes(iv, AES_256_IV_LENGTH); BinaryWriter bw(AssumeVersion(ProtocolVersion::withEncryptionAtRest())); bw << *this; + EncryptBlobCipherAes265Ctr cipher( textCipherKey, headerCipherKey, @@ -190,11 +237,22 @@ struct MutationRef { AES_256_IV_LENGTH, getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE), usageType); - BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader; - StringRef headerRef(reinterpret_cast(header), sizeof(BlobCipherEncryptHeader)); - StringRef payload = - cipher.encrypt(static_cast(bw.getData()), bw.getLength(), header, arena)->toStringRef(); - return MutationRef(Encrypted, headerRef, payload); + + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + BlobCipherEncryptHeaderRef header; + StringRef payload = + cipher.encrypt(static_cast(bw.getData()), bw.getLength(), &header, arena); + Standalone serializedHeader = BlobCipherEncryptHeaderRef::toStringRef(header); + arena.dependsOn(serializedHeader.arena()); + return MutationRef(Encrypted, serializedHeader, payload); + } else { + BlobCipherEncryptHeader* header = new (arena) BlobCipherEncryptHeader; + StringRef serializedHeader = + StringRef(reinterpret_cast(header), sizeof(BlobCipherEncryptHeader)); + StringRef payload = + cipher.encrypt(static_cast(bw.getData()), bw.getLength(), header, arena)->toStringRef(); + return MutationRef(Encrypted, serializedHeader, payload); + } } MutationRef encryptMetadata(const std::unordered_map>& cipherKeys, @@ -207,9 +265,18 @@ struct MutationRef { Arena& arena, BlobCipherMetrics::UsageType usageType, StringRef* buf = nullptr) const { - const BlobCipherEncryptHeader* header = encryptionHeader(); - DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header->iv, usageType); - StringRef plaintext = cipher.decrypt(param2.begin(), param2.size(), *header, arena)->toStringRef(); + StringRef plaintext; + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + const BlobCipherEncryptHeaderRef header = configurableEncryptionHeader(); + DecryptBlobCipherAes256Ctr cipher( + cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header.getIV(), usageType); + plaintext = cipher.decrypt(param2.begin(), param2.size(), header, arena); + } else { + const BlobCipherEncryptHeader* header = encryptionHeader(); + DecryptBlobCipherAes256Ctr cipher( + cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, header->iv, usageType); + plaintext = cipher.decrypt(param2.begin(), param2.size(), *header, arena)->toStringRef(); + } if (buf != nullptr) { *buf = plaintext; } @@ -229,7 +296,6 @@ struct MutationRef { TextAndHeaderCipherKeys getCipherKeys( const std::unordered_map>& cipherKeys) const { - const BlobCipherEncryptHeader* header = encryptionHeader(); auto getCipherKey = [&](const BlobCipherDetails& details) -> Reference { if (!details.isValid()) { return {}; @@ -239,8 +305,22 @@ struct MutationRef { return iter->second; }; TextAndHeaderCipherKeys textAndHeaderKeys; - textAndHeaderKeys.cipherHeaderKey = getCipherKey(header->cipherHeaderDetails); - textAndHeaderKeys.cipherTextKey = getCipherKey(header->cipherTextDetails); + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + const BlobCipherEncryptHeaderRef header = configurableEncryptionHeader(); + EncryptHeaderCipherDetails cipherDetails = header.getCipherDetails(); + ASSERT(cipherDetails.textCipherDetails.isValid()); + textAndHeaderKeys.cipherTextKey = getCipherKey(cipherDetails.textCipherDetails); + if (cipherDetails.headerCipherDetails.present()) { + ASSERT(cipherDetails.headerCipherDetails.get().isValid()); + textAndHeaderKeys.cipherHeaderKey = getCipherKey(cipherDetails.headerCipherDetails.get()); + } else { + ASSERT(!FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED); + } + } else { + const BlobCipherEncryptHeader* header = encryptionHeader(); + textAndHeaderKeys.cipherHeaderKey = getCipherKey(header->cipherHeaderDetails); + textAndHeaderKeys.cipherTextKey = getCipherKey(header->cipherTextDetails); + } return textAndHeaderKeys; } diff --git a/fdbclient/include/fdbclient/DatabaseContext.h b/fdbclient/include/fdbclient/DatabaseContext.h index ba96e961be..6bc4f6d489 100644 --- a/fdbclient/include/fdbclient/DatabaseContext.h +++ b/fdbclient/include/fdbclient/DatabaseContext.h @@ -406,6 +406,7 @@ public: Future waitPurgeGranulesComplete(Key purgeKey); Future blobbifyRange(KeyRange range, Optional> tenant = {}); + Future blobbifyRangeBlocking(KeyRange range, Optional> tenant = {}); Future unblobbifyRange(KeyRange range, Optional> tenant = {}); Future>> listBlobbifiedRanges(KeyRange range, int rangeLimit, @@ -413,6 +414,10 @@ public: Future verifyBlobRange(const KeyRange& range, Optional version, Optional> tenant = {}); + Future flushBlobRange(const KeyRange& range, + bool compact, + Optional version, + Optional> tenant = {}); Future blobRestore(const KeyRange range, Optional version); // private: diff --git a/fdbclient/include/fdbclient/FDBTypes.h b/fdbclient/include/fdbclient/FDBTypes.h index 86e0618f78..dc8ece38d6 100644 --- a/fdbclient/include/fdbclient/FDBTypes.h +++ b/fdbclient/include/fdbclient/FDBTypes.h @@ -1063,9 +1063,9 @@ struct StorageBytes { constexpr static FileIdentifier file_identifier = 3928581; // Free space on the filesystem int64_t free; - // Total space on the filesystem + // Total capacity on the filesystem usable by non-privileged users. int64_t total; - // Used by *this* store, not total - free + // Total size of all files owned by *this* storage instance, not total - free int64_t used; // Amount of space available for use by the store, which includes free space on the filesystem // and internal free space within the store data that is immediately reusable. diff --git a/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h b/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h index ebecec3b69..495b19a7c4 100644 --- a/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h +++ b/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h @@ -56,7 +56,7 @@ Future onEncryptKeyProxyChange(Reference const> db) { break; } } - TraceEvent("GetEncryptCipherKeys_EncryptKeyProxyChanged") + TraceEvent("GetEncryptCipherKeysEncryptKeyProxyChanged") .detail("PreviousProxyId", previousProxyId.orDefault(UID())) .detail("CurrentProxyId", currentProxyId.orDefault(UID())); return Void(); @@ -69,19 +69,19 @@ Future getUncachedLatestEncryptCipherKeys(Refer Optional proxy = db->get().encryptKeyProxy; if (!proxy.present()) { // Wait for onEncryptKeyProxyChange. - TraceEvent("GetLatestEncryptCipherKeys_EncryptKeyProxyNotPresent").detail("UsageType", toString(usageType)); + TraceEvent("GetLatestEncryptCipherKeysEncryptKeyProxyNotPresent").detail("UsageType", toString(usageType)); return Never(); } request.reply.reset(); try { EKPGetLatestBaseCipherKeysReply reply = wait(proxy.get().getLatestBaseCipherKeys.getReply(request)); if (reply.error.present()) { - TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_RequestFailed").error(reply.error.get()); + TraceEvent(SevWarn, "GetLatestEncryptCipherKeysRequestFailed").error(reply.error.get()); throw encrypt_keys_fetch_failed(); } return reply; } catch (Error& e) { - TraceEvent("GetLatestEncryptCipherKeys_CaughtError").error(e); + TraceEvent("GetLatestEncryptCipherKeysCaughtError").error(e); if (e.code() == error_code_broken_promise) { // Wait for onEncryptKeyProxyChange. return Never(); @@ -103,7 +103,7 @@ Future>> getL state EKPGetLatestBaseCipherKeysRequest request; if (!db.isValid()) { - TraceEvent(SevError, "GetLatestEncryptCipherKeys_ServerDBInfoNotAvailable"); + TraceEvent(SevError, "GetLatestEncryptCipherKeysServerDBInfoNotAvailable"); throw encrypt_ops_error(); } @@ -140,7 +140,7 @@ Future>> getL // Check for any missing cipher keys. for (auto domainId : request.encryptDomainIds) { if (cipherKeys.count(domainId) == 0) { - TraceEvent(SevWarn, "GetLatestEncryptCipherKeys_KeyMissing").detail("DomainId", domainId); + TraceEvent(SevWarn, "GetLatestEncryptCipherKeysKeyMissing").detail("DomainId", domainId); throw encrypt_key_not_found(); } } @@ -176,14 +176,14 @@ Future getUncachedEncryptCipherKeys(Reference proxy = db->get().encryptKeyProxy; if (!proxy.present()) { // Wait for onEncryptKeyProxyChange. - TraceEvent("GetEncryptCipherKeys_EncryptKeyProxyNotPresent").detail("UsageType", toString(usageType)); + TraceEvent("GetEncryptCipherKeysEncryptKeyProxyNotPresent").detail("UsageType", toString(usageType)); return Never(); } request.reply.reset(); try { EKPGetBaseCipherKeysByIdsReply reply = wait(proxy.get().getBaseCipherKeysByIds.getReply(request)); if (reply.error.present()) { - TraceEvent(SevWarn, "GetEncryptCipherKeys_RequestFailed").error(reply.error.get()); + TraceEvent(SevWarn, "GetEncryptCipherKeysRequestFailed").error(reply.error.get()); throw encrypt_keys_fetch_failed(); } if (g_network && g_network->isSimulated() && usageType == BlobCipherMetrics::RESTORE) { @@ -192,7 +192,7 @@ Future getUncachedEncryptCipherKeys(Reference getUncachedEncryptCipherKeys(Reference>> getEncry state EKPGetBaseCipherKeysByIdsRequest request; if (!db.isValid()) { - TraceEvent(SevError, "GetEncryptCipherKeys_ServerDBInfoNotAvailable"); + TraceEvent(SevError, "GetEncryptCipherKeysServerDBInfoNotAvailable"); throw encrypt_ops_error(); } @@ -262,7 +262,7 @@ Future>> getEncry BaseCipherIndex baseIdx = std::make_pair(details.encryptDomainId, details.baseCipherId); const auto& itr = baseCipherKeys.find(baseIdx); if (itr == baseCipherKeys.end()) { - TraceEvent(SevError, "GetEncryptCipherKeys_KeyMissing") + TraceEvent(SevError, "GetEncryptCipherKeysKeyMissing") .detail("DomainId", details.encryptDomainId) .detail("BaseCipherId", details.baseCipherId); throw encrypt_key_not_found(); diff --git a/fdbclient/include/fdbclient/IClientApi.h b/fdbclient/include/fdbclient/IClientApi.h index cab78707ad..dc56e2198d 100644 --- a/fdbclient/include/fdbclient/IClientApi.h +++ b/fdbclient/include/fdbclient/IClientApi.h @@ -153,11 +153,13 @@ public: virtual ThreadFuture waitPurgeGranulesComplete(const KeyRef& purgeKey) = 0; virtual ThreadFuture blobbifyRange(const KeyRangeRef& keyRange) = 0; + virtual ThreadFuture blobbifyRangeBlocking(const KeyRangeRef& keyRange) = 0; virtual ThreadFuture unblobbifyRange(const KeyRangeRef& keyRange) = 0; virtual ThreadFuture>> listBlobbifiedRanges(const KeyRangeRef& keyRange, int rangeLimit) = 0; virtual ThreadFuture verifyBlobRange(const KeyRangeRef& keyRange, Optional version) = 0; + virtual ThreadFuture flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional version) = 0; virtual void addref() = 0; virtual void delref() = 0; @@ -200,11 +202,13 @@ public: virtual ThreadFuture waitPurgeGranulesComplete(const KeyRef& purgeKey) = 0; virtual ThreadFuture blobbifyRange(const KeyRangeRef& keyRange) = 0; + virtual ThreadFuture blobbifyRangeBlocking(const KeyRangeRef& keyRange) = 0; virtual ThreadFuture unblobbifyRange(const KeyRangeRef& keyRange) = 0; virtual ThreadFuture>> listBlobbifiedRanges(const KeyRangeRef& keyRange, int rangeLimit) = 0; virtual ThreadFuture verifyBlobRange(const KeyRangeRef& keyRange, Optional version) = 0; + virtual ThreadFuture flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional version) = 0; // Interface to manage shared state across multiple connections to the same Database virtual ThreadFuture createSharedState() = 0; diff --git a/fdbclient/include/fdbclient/MultiVersionTransaction.h b/fdbclient/include/fdbclient/MultiVersionTransaction.h index e5d83898ad..577268accb 100644 --- a/fdbclient/include/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/include/fdbclient/MultiVersionTransaction.h @@ -187,6 +187,12 @@ struct FdbCApi : public ThreadSafeReferenceCounted { uint8_t const* end_key_name, int end_key_name_length); + FDBFuture* (*databaseBlobbifyRangeBlocking)(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length); + FDBFuture* (*databaseUnblobbifyRange)(FDBDatabase* db, uint8_t const* begin_key_name, int begin_key_name_length, @@ -207,6 +213,14 @@ struct FdbCApi : public ThreadSafeReferenceCounted { int end_key_name_length, int64_t version); + FDBFuture* (*databaseFlushBlobRange)(FDBDatabase* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + fdb_bool_t compact, + int64_t version); + FDBFuture* (*databaseGetClientStatus)(FDBDatabase* db); // Tenant @@ -230,6 +244,12 @@ struct FdbCApi : public ThreadSafeReferenceCounted { uint8_t const* end_key_name, int end_key_name_length); + FDBFuture* (*tenantBlobbifyRangeBlocking)(FDBTenant* tenant, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length); + FDBFuture* (*tenantUnblobbifyRange)(FDBTenant* tenant, uint8_t const* begin_key_name, int begin_key_name_length, @@ -250,6 +270,14 @@ struct FdbCApi : public ThreadSafeReferenceCounted { int end_key_name_length, int64_t version); + FDBFuture* (*tenantFlushBlobRange)(FDBTenant* db, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + fdb_bool_t compact, + int64_t version); + FDBFuture* (*tenantGetId)(FDBTenant* tenant); void (*tenantDestroy)(FDBTenant* tenant); @@ -547,11 +575,13 @@ public: ThreadFuture waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture blobbifyRange(const KeyRangeRef& keyRange) override; + ThreadFuture blobbifyRangeBlocking(const KeyRangeRef& keyRange) override; ThreadFuture unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture>> listBlobbifiedRanges(const KeyRangeRef& keyRange, int rangeLimit) override; ThreadFuture verifyBlobRange(const KeyRangeRef& keyRange, Optional version) override; + ThreadFuture flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional version) override; void addref() override { ThreadSafeReferenceCounted::addref(); } void delref() override { ThreadSafeReferenceCounted::delref(); } @@ -597,11 +627,13 @@ public: ThreadFuture waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture blobbifyRange(const KeyRangeRef& keyRange) override; + ThreadFuture blobbifyRangeBlocking(const KeyRangeRef& keyRange) override; ThreadFuture unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture>> listBlobbifiedRanges(const KeyRangeRef& keyRange, int rangeLimit) override; ThreadFuture verifyBlobRange(const KeyRangeRef& keyRange, Optional version) override; + ThreadFuture flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional version) override; ThreadFuture createSharedState() override; void setSharedState(DatabaseSharedState* p) override; @@ -866,10 +898,12 @@ public: ThreadFuture waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture blobbifyRange(const KeyRangeRef& keyRange) override; + ThreadFuture blobbifyRangeBlocking(const KeyRangeRef& keyRange) override; ThreadFuture unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture>> listBlobbifiedRanges(const KeyRangeRef& keyRange, int rangeLimit) override; ThreadFuture verifyBlobRange(const KeyRangeRef& keyRange, Optional version) override; + ThreadFuture flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional version) override; void addref() override { ThreadSafeReferenceCounted::addref(); } void delref() override { ThreadSafeReferenceCounted::delref(); } @@ -994,10 +1028,12 @@ public: ThreadFuture waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture blobbifyRange(const KeyRangeRef& keyRange) override; + ThreadFuture blobbifyRangeBlocking(const KeyRangeRef& keyRange) override; ThreadFuture unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture>> listBlobbifiedRanges(const KeyRangeRef& keyRange, int rangeLimit) override; ThreadFuture verifyBlobRange(const KeyRangeRef& keyRange, Optional version) override; + ThreadFuture flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional version) override; ThreadFuture createSharedState() override; void setSharedState(DatabaseSharedState* p) override; diff --git a/fdbclient/include/fdbclient/RESTUtils.h b/fdbclient/include/fdbclient/RESTUtils.h index 43a34cb805..1aec2bcfe2 100644 --- a/fdbclient/include/fdbclient/RESTUtils.h +++ b/fdbclient/include/fdbclient/RESTUtils.h @@ -27,6 +27,7 @@ #include "flow/FastRef.h" #include "flow/Net2Packet.h" +#include #include #include @@ -108,6 +109,11 @@ public: explicit RESTUrl(const std::string& fullUrl, const bool isSecure); explicit RESTUrl(const std::string& fullUrl, const std::string& body, const bool isSecure); + std::string toString() const { + return fmt::format( + "Host {} Service {} Resource {} ReqParams {} Body {}", host, service, resource, reqParameters, body); + } + private: void parseUrl(const std::string& fullUrl, bool isSecure); }; diff --git a/fdbclient/include/fdbclient/ServerKnobs.h b/fdbclient/include/fdbclient/ServerKnobs.h index 08c1a270c3..0f12988a75 100644 --- a/fdbclient/include/fdbclient/ServerKnobs.h +++ b/fdbclient/include/fdbclient/ServerKnobs.h @@ -962,13 +962,8 @@ public: std::string CLUSTER_RECOVERY_EVENT_NAME_PREFIX; // Encryption - bool ENABLE_ENCRYPTION; - std::string ENCRYPTION_MODE; int SIM_KMS_MAX_KEYS; int ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH; - bool ENABLE_TLOG_ENCRYPTION; - bool ENABLE_STORAGE_SERVER_ENCRYPTION; // Currently only Redwood engine supports encryption - bool ENABLE_BLOB_GRANULE_ENCRYPTION; // Compression bool ENABLE_BLOB_GRANULE_COMPRESSION; @@ -1039,6 +1034,7 @@ public: std::string BLOB_RESTORE_MANIFEST_URL; int BLOB_RESTORE_MANIFEST_FILE_MAX_SIZE; int BLOB_RESTORE_MANIFEST_RETENTION_MAX; + int BLOB_RESTORE_MLOGS_RETENTION_SECS; // Blob metadata int64_t BLOB_METADATA_CACHE_TTL; diff --git a/fdbclient/include/fdbclient/SystemData.h b/fdbclient/include/fdbclient/SystemData.h index 8a1ca2482c..8e31a175be 100644 --- a/fdbclient/include/fdbclient/SystemData.h +++ b/fdbclient/include/fdbclient/SystemData.h @@ -733,6 +733,7 @@ const KeyRange decodeBlobRestoreArgKeyFor(const KeyRef key); const Value blobRestoreArgValueFor(BlobRestoreArg args); Standalone decodeBlobRestoreArg(ValueRef const& value); extern const Key blobManifestVersionKey; +extern const Key blobGranulesLastFlushKey; extern const KeyRangeRef idempotencyIdKeys; extern const KeyRef idempotencyIdsExpiredVersion; diff --git a/fdbclient/include/fdbclient/Tenant.h b/fdbclient/include/fdbclient/Tenant.h index 5f4d811324..ba5588338a 100644 --- a/fdbclient/include/fdbclient/Tenant.h +++ b/fdbclient/include/fdbclient/Tenant.h @@ -36,6 +36,7 @@ namespace TenantAPI { KeyRef idToPrefix(Arena& p, int64_t id); Key idToPrefix(int64_t id); int64_t prefixToId(KeyRef prefix, EnforceValidTenantId = EnforceValidTenantId::True); +KeyRangeRef clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena); // return true if begin and end has the same non-negative prefix id bool withinSingleTenant(KeyRangeRef const&); diff --git a/fdbclient/include/fdbclient/ThreadSafeTransaction.h b/fdbclient/include/fdbclient/ThreadSafeTransaction.h index d1e5dad1f3..8fed0d14ee 100644 --- a/fdbclient/include/fdbclient/ThreadSafeTransaction.h +++ b/fdbclient/include/fdbclient/ThreadSafeTransaction.h @@ -64,11 +64,13 @@ public: ThreadFuture waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture blobbifyRange(const KeyRangeRef& keyRange) override; + ThreadFuture blobbifyRangeBlocking(const KeyRangeRef& keyRange) override; ThreadFuture unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture>> listBlobbifiedRanges(const KeyRangeRef& keyRange, int rangeLimit) override; ThreadFuture verifyBlobRange(const KeyRangeRef& keyRange, Optional version) override; + ThreadFuture flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional version) override; ThreadFuture createSharedState() override; void setSharedState(DatabaseSharedState* p) override; @@ -101,11 +103,13 @@ public: ThreadFuture waitPurgeGranulesComplete(const KeyRef& purgeKey) override; ThreadFuture blobbifyRange(const KeyRangeRef& keyRange) override; + ThreadFuture blobbifyRangeBlocking(const KeyRangeRef& keyRange) override; ThreadFuture unblobbifyRange(const KeyRangeRef& keyRange) override; ThreadFuture>> listBlobbifiedRanges(const KeyRangeRef& keyRange, int rangeLimit) override; ThreadFuture verifyBlobRange(const KeyRangeRef& keyRange, Optional version) override; + ThreadFuture flushBlobRange(const KeyRangeRef& keyRange, bool compact, Optional version) override; void addref() override { ThreadSafeReferenceCounted::addref(); } void delref() override { ThreadSafeReferenceCounted::delref(); } diff --git a/fdbrpc/include/fdbrpc/TenantInfo.h b/fdbrpc/include/fdbrpc/TenantInfo.h index 3558561509..e91db198ff 100644 --- a/fdbrpc/include/fdbrpc/TenantInfo.h +++ b/fdbrpc/include/fdbrpc/TenantInfo.h @@ -47,7 +47,7 @@ struct TenantInfo { // Helper function for most endpoints that read/write data. This returns true iff // the client is either a) a trusted peer or b) is accessing keyspace belonging to a tenant, // for which it has a valid authorization token. - // NOTE: In a cluster where TenantMode is OPTIONAL or DISABLED, tenant name may be unset. + // NOTE: In a cluster where TenantMode is OPTIONAL or DISABLED, tenant ID may be invalid. // In such case, the request containing such TenantInfo is valid iff the requesting peer is trusted. bool isAuthorized() const { return trusted || tenantAuthorized; } bool hasTenant() const { return tenantId != INVALID_TENANT; } diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp index e119174e5b..8f8cb70993 100644 --- a/fdbserver/ApplyMetadataMutation.cpp +++ b/fdbserver/ApplyMetadataMutation.cpp @@ -25,7 +25,6 @@ #include "fdbclient/Notified.h" #include "fdbclient/SystemData.h" #include "fdbserver/ApplyMetadataMutation.h" -#include "fdbserver/EncryptionOpsUtils.h" #include "fdbserver/IKeyValueStore.h" #include "fdbserver/Knobs.h" #include "fdbserver/LogProtocolMessage.h" diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp index 0716129dee..73a1c244b7 100644 --- a/fdbserver/BackupWorker.actor.cpp +++ b/fdbserver/BackupWorker.actor.cpp @@ -105,11 +105,7 @@ struct VersionedMessage { ArenaReader reader(arena, message, AssumeVersion(ProtocolVersion::withEncryptionAtRest())); MutationRef m; reader >> m; - const BlobCipherEncryptHeader* header = m.encryptionHeader(); - cipherDetails.insert(header->cipherTextDetails); - if (header->cipherHeaderDetails.isValid()) { - cipherDetails.insert(header->cipherHeaderDetails); - } + m.updateEncryptCipherDetails(cipherDetails); } } }; diff --git a/fdbserver/BlobGranuleServerCommon.actor.cpp b/fdbserver/BlobGranuleServerCommon.actor.cpp index 53fbb748fd..f34c974341 100644 --- a/fdbserver/BlobGranuleServerCommon.actor.cpp +++ b/fdbserver/BlobGranuleServerCommon.actor.cpp @@ -209,6 +209,7 @@ void GranuleFiles::getFiles(Version beginVersion, snapshotF->offset, snapshotF->length, snapshotF->fullFileLength, + snapshotF->version, summarize ? Optional() : snapshotF->cipherKeysMeta); lastIncluded = chunk.snapshotVersion; } else { @@ -221,6 +222,7 @@ void GranuleFiles::getFiles(Version beginVersion, deltaF->offset, deltaF->length, deltaF->fullFileLength, + deltaF->version, summarize ? Optional() : deltaF->cipherKeysMeta); deltaBytesCounter += deltaF->length; ASSERT(lastIncluded < deltaF->version); @@ -235,6 +237,7 @@ void GranuleFiles::getFiles(Version beginVersion, deltaF->offset, deltaF->length, deltaF->fullFileLength, + deltaF->version, deltaF->cipherKeysMeta); deltaBytesCounter += deltaF->length; lastIncluded = deltaF->version; diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp index dcbdfbbddd..70fb4e168c 100644 --- a/fdbserver/BlobManager.actor.cpp +++ b/fdbserver/BlobManager.actor.cpp @@ -19,14 +19,20 @@ */ #include +#include #include #include #include #include #include +#include "fdbclient/BackupContainer.h" +#include "fdbclient/ClientBooleanParams.h" +#include "fdbclient/KeyBackedTypes.h" #include "fdbclient/ServerKnobs.h" #include "fdbrpc/simulator.h" +#include "flow/CodeProbe.h" +#include "flow/serialize.h" #include "fmt/format.h" #include "fdbclient/BackupContainerFileSystem.h" #include "fdbclient/BlobGranuleCommon.h" @@ -38,6 +44,7 @@ #include "fdbclient/ReadYourWrites.h" #include "fdbclient/SystemData.h" #include "fdbclient/Tuple.h" +#include "fdbclient/BackupAgent.actor.h" #include "fdbserver/BlobManagerInterface.h" #include "fdbserver/Knobs.h" #include "fdbserver/BlobGranuleValidation.actor.h" @@ -5243,6 +5250,91 @@ ACTOR Future bgConsistencyCheck(Reference bmData) { } } +ACTOR Future updateLastFlushTs(Database db) { + state Transaction tr(db); + loop { + try { + tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + KeyBackedProperty lastFlushTs(blobGranulesLastFlushKey); + int64_t epochs = (int64_t)now(); + lastFlushTs.set(&tr, epochs); + wait(tr.commit()); + return Void(); + } catch (Error& e) { + fmt::print("updateLastFlushTs {} \n", e.what()); + wait(tr.onError(e)); + } + } +} + +ACTOR Future getLastFlushTs(Database db) { + state Transaction tr(db); + loop { + try { + tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + KeyBackedProperty lastFlushTs(blobGranulesLastFlushKey); + state int64_t ret; + wait(store(ret, lastFlushTs.getD(&tr, Snapshot::False, 0))); + return ret; + } catch (Error& e) { + wait(tr.onError(e)); + } + } +} + +ACTOR Future maybeFlushAndTruncateMutationLogs(Reference bmData) { + try { + int64_t lastFlushTs = wait(getLastFlushTs(bmData->db)); + bool shouldFlush = (now() - lastFlushTs) > SERVER_KNOBS->BLOB_RESTORE_MLOGS_RETENTION_SECS; + if (!shouldFlush) { + TraceEvent("SkipBlobGranulesFlush").detail("LastFlushTs", lastFlushTs); + return Void(); + } + + state std::string mlogsUrl = wait(getMutationLogUrl()); + state Reference bc = IBackupContainer::openContainer(mlogsUrl, {}, {}); + state BackupDescription desc = wait(bc->describeBackup(true)); + if (!desc.contiguousLogEnd.present()) { + TraceEvent("SkipBlobGranulesFlush").detail("LogUrl", mlogsUrl); + return Void(); // skip truncation if no valid backup for mutation logs + } + + state Version logEndVersion = desc.contiguousLogEnd.get(); + // Force flush in-memory data of all blob workers until end of log + FlushGranuleRequest req(bmData->epoch, normalKeys, logEndVersion, false); + wait(success(doBlobGranuleRequests(bmData->db, normalKeys, req, &BlobWorkerInterface::flushGranuleRequest))); + wait(updateLastFlushTs(bmData->db)); + + // Truncate mutation logs to max retention period + Reference tr(new ReadYourWritesTransaction(bmData->db)); + Optional logEndEpochs = wait(timeKeeperEpochsFromVersion(logEndVersion, tr)); + if (!logEndEpochs.present()) { + TraceEvent("SkipMutationLogTruncation").detail("LogEndVersion", logEndVersion); + return Void(); // skip truncation if no timestamp about log end + } + + // Find timestamp and version to truncate + int64_t epochs = logEndEpochs.get() - SERVER_KNOBS->BLOB_RESTORE_MLOGS_RETENTION_SECS; + if (epochs <= 0) { + TraceEvent("SkipMutationLogTruncation").detail("Epochs", epochs); + return Void(); + } + state std::string timestamp = BackupAgentBase::formatTime(epochs); + state Version truncVersion = wait(timeKeeperVersionFromDatetime(timestamp, bmData->db)); + + wait(bc->expireData(truncVersion, true)); + TraceEvent("TruncateMutationLogs").detail("Version", truncVersion).detail("Timestamp", timestamp); + CODE_PROBE(true, "Flush blob granules and truncate mutation logs"); + } catch (Error& e) { + TraceEvent("TruncateMutationLogsError").error(e); // skip and retry next time + } + return Void(); +} + ACTOR Future backupManifest(Reference bmData) { bmData->initBStore(); @@ -5256,8 +5348,9 @@ ACTOR Future backupManifest(Reference bmData) { break; } } - if (activeRanges) { + if (activeRanges && SERVER_KNOBS->BLOB_MANIFEST_BACKUP) { if (bmData->manifestStore.isValid()) { + wait(maybeFlushAndTruncateMutationLogs(bmData)); wait(dumpManifest(bmData->db, bmData->manifestStore, bmData->epoch, bmData->manifestDumperSeqNo)); bmData->manifestDumperSeqNo++; } else { diff --git a/fdbserver/BlobManifest.actor.cpp b/fdbserver/BlobManifest.actor.cpp index fc3a99ed29..2cc970bd46 100644 --- a/fdbserver/BlobManifest.actor.cpp +++ b/fdbserver/BlobManifest.actor.cpp @@ -260,6 +260,7 @@ public: self->segmentNo_ = 1; self->totalRows_ = 0; self->logicalSize_ = 0; + self->totalBytes_ = 0; wait(waitForAll(self->pendingFutures_)); self->pendingFutures_.clear(); self->deleteSegmentFiles(); @@ -999,3 +1000,16 @@ ACTOR Future getManifestVersion(Database db) { } } } + +ACTOR Future getMutationLogUrl() { + state std::string baseUrl = SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL; + if (baseUrl.starts_with("file://")) { + state std::vector containers = wait(IBackupContainer::listContainers(baseUrl, {})); + if (containers.size() == 0) { + throw blob_restore_missing_logs(); + } + return containers.back(); + } else { + return baseUrl; + } +} diff --git a/fdbserver/BlobMigrator.actor.cpp b/fdbserver/BlobMigrator.actor.cpp index 00434d11f9..4dc8898c2a 100644 --- a/fdbserver/BlobMigrator.actor.cpp +++ b/fdbserver/BlobMigrator.actor.cpp @@ -329,27 +329,15 @@ private: // Apply mutation logs to blob granules so that they reach to a consistent version for all blob granules ACTOR static Future applyMutationLogs(Reference self) { // check last version in mutation logs - - state std::string baseUrl = SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL; - state std::string mlogsUrl; - if (baseUrl.starts_with("file://")) { - state std::vector containers = wait(IBackupContainer::listContainers(baseUrl, {})); - if (containers.size() == 0) { - TraceEvent("MissingMutationLogs", self->interf_.id()).detail("Url", baseUrl); - throw restore_missing_data(); - } - mlogsUrl = containers.front(); - } else { - mlogsUrl = baseUrl; - } + state std::string mlogsUrl = wait(getMutationLogUrl()); state Reference bc = IBackupContainer::openContainer(mlogsUrl, {}, {}); BackupDescription desc = wait(bc->describeBackup()); if (!desc.contiguousLogEnd.present()) { - TraceEvent("InvalidMutationLogs").detail("Url", baseUrl); + TraceEvent("InvalidMutationLogs").detail("Url", SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL); throw blob_restore_missing_logs(); } if (!desc.minLogBegin.present()) { - TraceEvent("InvalidMutationLogs").detail("Url", baseUrl); + TraceEvent("InvalidMutationLogs").detail("Url", SERVER_KNOBS->BLOB_RESTORE_MLOGS_URL); throw blob_restore_missing_logs(); } state Version minLogVersion = desc.minLogBegin.get(); diff --git a/fdbserver/BlobWorker.actor.cpp b/fdbserver/BlobWorker.actor.cpp index 796120a41c..d4abbfb538 100644 --- a/fdbserver/BlobWorker.actor.cpp +++ b/fdbserver/BlobWorker.actor.cpp @@ -37,7 +37,6 @@ #include "fdbclient/Notified.h" #include "fdbserver/BlobGranuleServerCommon.actor.h" -#include "fdbserver/EncryptionOpsUtils.h" #include "fdbclient/GetEncryptCipherKeys.actor.h" #include "fdbserver/Knobs.h" #include "fdbserver/MutationTracking.h" @@ -1401,6 +1400,7 @@ ACTOR Future compactFromBlob(Reference bwData, snapshotF.offset, snapshotF.length, snapshotF.fullFileLength, + snapshotF.version, snapCipherKeysCtx); compactBytesRead += snapshotF.length; @@ -1432,6 +1432,7 @@ ACTOR Future compactFromBlob(Reference bwData, deltaF.offset, deltaF.length, deltaF.fullFileLength, + deltaF.version, deltaCipherKeysCtx); compactBytesRead += deltaF.length; lastDeltaVersion = files.deltaFiles[deltaIdx].version; diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp index 15c1e74f0c..5ea23a258a 100644 --- a/fdbserver/ClusterRecovery.actor.cpp +++ b/fdbserver/ClusterRecovery.actor.cpp @@ -24,7 +24,6 @@ #include "fdbserver/ApplyMetadataMutation.h" #include "fdbserver/BackupProgress.actor.h" #include "fdbserver/ClusterRecovery.actor.h" -#include "fdbserver/EncryptionOpsUtils.h" #include "fdbserver/Knobs.h" #include "fdbserver/MasterInterface.h" #include "fdbserver/WaitFailure.h" diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp index fe93531364..c0b1542cd2 100644 --- a/fdbserver/CommitProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -42,7 +42,6 @@ #include "fdbserver/ApplyMetadataMutation.h" #include "fdbserver/ConflictSet.h" #include "fdbserver/DataDistributorInterface.h" -#include "fdbserver/EncryptionOpsUtils.h" #include "fdbserver/FDBExecHelper.actor.h" #include "fdbclient/GetEncryptCipherKeys.actor.h" #include "fdbserver/IKeyValueStore.h" @@ -1691,6 +1690,8 @@ ACTOR Future writeMutationEncryptedMutation(CommitBatchCont Arena* arena) { state MutationRef encryptedMutation = encryptedMutationOpt->get(); state const BlobCipherEncryptHeader* header; + state BlobCipherEncryptHeaderRef headerRef; + state MutationRef decryptedMutation; static_assert(TenantInfo::INVALID_TENANT == INVALID_ENCRYPT_DOMAIN_ID); ASSERT(self->pProxyCommitData->encryptMode.isEncryptionEnabled()); @@ -1698,9 +1699,15 @@ ACTOR Future writeMutationEncryptedMutation(CommitBatchCont ASSERT(encryptedMutation.isEncrypted()); Reference const> dbInfo = self->pProxyCommitData->db; - header = encryptedMutation.encryptionHeader(); - TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, *header, BlobCipherMetrics::TLOG)); - MutationRef decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG); + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + headerRef = encryptedMutation.configurableEncryptionHeader(); + TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, headerRef, BlobCipherMetrics::TLOG)); + decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG); + } else { + header = encryptedMutation.encryptionHeader(); + TextAndHeaderCipherKeys cipherKeys = wait(getEncryptCipherKeys(dbInfo, *header, BlobCipherMetrics::TLOG)); + decryptedMutation = encryptedMutation.decrypt(cipherKeys, *arena, BlobCipherMetrics::TLOG); + } ASSERT(decryptedMutation.type == mutation->type); ASSERT(decryptedMutation.param1 == mutation->param1); @@ -2668,7 +2675,7 @@ ACTOR static Future doKeyServerLocationRequest(GetKeyServerLocationsReques ssis.push_back(it->interf); maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); } - rep.results.emplace_back(r.range(), ssis); + rep.results.emplace_back(TenantAPI::clampRangeToTenant(r.range(), req.tenant, req.arena), ssis); } else if (!req.reverse) { int count = 0; for (auto r = commitData->keyInfo.rangeContaining(req.begin); @@ -2680,7 +2687,7 @@ ACTOR static Future doKeyServerLocationRequest(GetKeyServerLocationsReques ssis.push_back(it->interf); maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); } - rep.results.emplace_back(r.range(), ssis); + rep.results.emplace_back(TenantAPI::clampRangeToTenant(r.range(), req.tenant, req.arena), ssis); count++; } } else { @@ -2693,7 +2700,7 @@ ACTOR static Future doKeyServerLocationRequest(GetKeyServerLocationsReques ssis.push_back(it->interf); maybeAddTssMapping(rep, commitData, tssMappingsIncluded, it->interf.id()); } - rep.results.emplace_back(r.range(), ssis); + rep.results.emplace_back(TenantAPI::clampRangeToTenant(r.range(), req.tenant, req.arena), ssis); if (r == commitData->keyInfo.ranges().begin()) { break; } diff --git a/fdbserver/KeyValueStoreMemory.actor.cpp b/fdbserver/KeyValueStoreMemory.actor.cpp index e57631bb66..fe53068a46 100644 --- a/fdbserver/KeyValueStoreMemory.actor.cpp +++ b/fdbserver/KeyValueStoreMemory.actor.cpp @@ -35,6 +35,8 @@ #include "flow/actorcompiler.h" // This must be the last #include. #define OP_DISK_OVERHEAD (sizeof(OpHeader) + 1) +#define ENCRYPTION_ENABLED_BIT 31 +static_assert(sizeof(uint32_t) == 4); template class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable { @@ -308,7 +310,7 @@ private: OpCommit, // only in log, not in queue OpRollback, // only in log, not in queue OpSnapshotItemDelta, - OpEncrypted + OpEncrypted_Deprecated // deprecated since we now store the encryption status in the first bit of the opType }; struct OpRef { @@ -319,7 +321,7 @@ private: size_t expectedSize() const { return p1.expectedSize() + p2.expectedSize(); } }; struct OpHeader { - int op; + uint32_t op; int len1, len2; }; @@ -462,39 +464,56 @@ private: return total; } - // Data format for normal operation: - // +-------------+-------------+-------------+--------+--------+ - // | opType | len1 | len2 | param2 | param2 | - // | sizeof(int) | sizeof(int) | sizeof(int) | len1 | len2 | - // +-------------+-------------+-------------+--------+--------+ + static bool isOpEncrypted(OpHeader* header) { return header->op >> ENCRYPTION_ENABLED_BIT == 1; } + + static void setEncryptFlag(OpHeader* header, bool set) { + if (set) { + header->op |= (1UL << ENCRYPTION_ENABLED_BIT); + } else { + header->op &= ~(1UL << ENCRYPTION_ENABLED_BIT); + } + } + + // NOTE: The first bit of opType indicates whether the entry is encrypted or not. This is fine for backwards + // compatability since the first bit was never used previously // - // However, if the operation is encrypted: - // +-------------+-------------+-------------+---------------------------------+-------------+--------+--------+ - // | OpEncrypted | len1 | len2 | BlobCipherEncryptHeader | opType | param1 | param2 | - // | sizeof(int) | sizeof(int) | sizeof(int) | sizeof(BlobCipherEncryptHeader) | sizeof(int) | len1 | len2 | - // +-------------+-------------+-------------+---------------------------------+-------------+--------+--------+ - // | plaintext | encrypted | - // +-----------------------------------------------------------------------------------------------------------+ + // Unencrypted data format: + // +-------------+-------------+-------------+--------+--------+-----------+ + // | opType | len1 | len2 | param2 | param2 | \x01 | + // | sizeof(int) | sizeof(int) | sizeof(int) | len1 | len2 | 1 byte | + // +-------------+-------------+-------------+--------+--------+-----------+ + // + // Encrypted data format: + // +-------------+-------+---------+------------+-----------------------------+--------+--------+------------+ + // | opType |len1 | len2 | headerSize | BlobCipherEncryptHeader | param1 | param2 | \x01 | + // | s(uint32) | s(int)| s(int) | s(uint16) | s(BlobCipherEncryptHeader) | len1 | len2 | 1 byte | + // +-------------+-------+---------+------------+-----------------------------+--------+--------+------------+ + // | plaintext | encrypted | | + // +--------------------------------------------------------------------------+-----------------+------------+ // IDiskQueue::location log_op(OpType op, StringRef v1, StringRef v2) { // Metadata op types to be excluded from encryption. static std::unordered_set metaOps = { OpSnapshotEnd, OpSnapshotAbort, OpCommit, OpRollback }; + uint32_t opType = (uint32_t)op; + // Make sure the first bit of the optype is empty + ASSERT(opType >> ENCRYPTION_ENABLED_BIT == 0); if (!enableEncryption || metaOps.count(op) > 0) { - OpHeader h = { (int)op, v1.size(), v2.size() }; + OpHeader h = { opType, v1.size(), v2.size() }; log->push(StringRef((const uint8_t*)&h, sizeof(h))); log->push(v1); log->push(v2); } else { - OpHeader h = { (int)OpEncrypted, v1.size(), v2.size() }; + OpHeader h = { opType, v1.size(), v2.size() }; + // Set the first bit of the header to 1 to indicate that the log entry is encrypted + setEncryptFlag(&h, true); log->push(StringRef((const uint8_t*)&h, sizeof(h))); - uint8_t* plaintext = new uint8_t[sizeof(int) + v1.size() + v2.size()]; - *(int*)plaintext = op; + uint8_t* plaintext = new uint8_t[v1.size() + v2.size()]; if (v1.size()) { - memcpy(plaintext + sizeof(int), v1.begin(), v1.size()); + memcpy(plaintext, v1.begin(), v1.size()); } if (v2.size()) { - memcpy(plaintext + sizeof(int) + v1.size(), v2.begin(), v2.size()); + memcpy(plaintext + v1.size(), v2.begin(), v2.size()); } ASSERT(cipherKeys.cipherTextKey.isValid()); @@ -504,12 +523,28 @@ private: cipherKeys.cipherHeaderKey, getEncryptAuthTokenMode(EncryptAuthTokenMode::ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE), BlobCipherMetrics::KV_MEMORY); - BlobCipherEncryptHeader cipherHeader; + uint16_t encryptHeaderSize; + // TODO: If possible we want to avoid memcpy to the disk log by using the same arena used by IDiskQueue Arena arena; - StringRef ciphertext = - cipher.encrypt(plaintext, sizeof(int) + v1.size() + v2.size(), &cipherHeader, arena)->toStringRef(); - log->push(StringRef((const uint8_t*)&cipherHeader, BlobCipherEncryptHeader::headerSize)); - log->push(ciphertext); + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + BlobCipherEncryptHeaderRef headerRef; + StringRef cipherText = cipher.encrypt(plaintext, v1.size() + v2.size(), &headerRef, arena); + Standalone headerRefStr = BlobCipherEncryptHeaderRef::toStringRef(headerRef); + encryptHeaderSize = headerRefStr.size(); + ASSERT(encryptHeaderSize > 0); + log->push(StringRef((const uint8_t*)&encryptHeaderSize, sizeof(encryptHeaderSize))); + log->push(headerRefStr); + log->push(cipherText); + } else { + BlobCipherEncryptHeader cipherHeader; + StringRef ciphertext = + cipher.encrypt(plaintext, v1.size() + v2.size(), &cipherHeader, arena)->toStringRef(); + encryptHeaderSize = BlobCipherEncryptHeader::headerSize; + ASSERT(encryptHeaderSize > 0); + log->push(StringRef((const uint8_t*)&encryptHeaderSize, sizeof(encryptHeaderSize))); + log->push(StringRef((const uint8_t*)&cipherHeader, encryptHeaderSize)); + log->push(ciphertext); + } } return log->push("\x01"_sr); // Changes here should be reflected in OP_DISK_OVERHEAD } @@ -517,19 +552,38 @@ private: // In case the op data is not encrypted, simply read the operands and the zero fill flag. // Otherwise, decrypt the op type and data. ACTOR static Future> readOpData(KeyValueStoreMemory* self, - OpHeader* h, + OpHeader h, bool* isZeroFilled, - int* zeroFillSize) { + int* zeroFillSize, + bool encryptedOp) { + ASSERT(!isOpEncrypted(&h)); // Metadata op types to be excluded from encryption. static std::unordered_set metaOps = { OpSnapshotEnd, OpSnapshotAbort, OpCommit, OpRollback }; - if (metaOps.count((OpType)h->op) == 0) { + if (metaOps.count((OpType)h.op) == 0) { // It is not supported to open an encrypted store as unencrypted, or vice-versa. - ASSERT_EQ(h->op == OpEncrypted, self->enableEncryption); + ASSERT_EQ(encryptedOp, self->enableEncryption); } - state int remainingBytes = h->len1 + h->len2 + 1; - if (h->op == OpEncrypted) { + // if encrypted op read the header size + state uint16_t encryptHeaderSize = 0; + if (encryptedOp) { + state Standalone headerSizeStr = wait(self->log->readNext(sizeof(encryptHeaderSize))); + ASSERT(headerSizeStr.size() <= sizeof(encryptHeaderSize)); + // Partial read on the header size + memset(&encryptHeaderSize, 0, sizeof(encryptHeaderSize)); + memcpy(&encryptHeaderSize, headerSizeStr.begin(), headerSizeStr.size()); + if (headerSizeStr.size() < sizeof(encryptHeaderSize)) { + CODE_PROBE(true, "zero fill partial encryption header size", probe::decoration::rare); + *zeroFillSize = + (sizeof(encryptHeaderSize) - headerSizeStr.size()) + encryptHeaderSize + h.len1 + h.len2 + 1; + } + if (*zeroFillSize > 0) { + return headerSizeStr; + } + } + state int remainingBytes = h.len1 + h.len2 + 1; + if (encryptedOp) { // encryption header, plus the real (encrypted) op type - remainingBytes += BlobCipherEncryptHeader::headerSize + sizeof(int); + remainingBytes += encryptHeaderSize; } state Standalone data = wait(self->log->readNext(remainingBytes)); ASSERT(data.size() <= remainingBytes); @@ -537,23 +591,31 @@ private: if (*zeroFillSize == 0) { *isZeroFilled = (data[data.size() - 1] == 0); } - if (h->op != OpEncrypted || *zeroFillSize > 0 || *isZeroFilled) { + if (!encryptedOp || *zeroFillSize > 0 || *isZeroFilled) { return data; } - state BlobCipherEncryptHeader cipherHeader = *(BlobCipherEncryptHeader*)data.begin(); - TextAndHeaderCipherKeys cipherKeys = - wait(getEncryptCipherKeys(self->db, cipherHeader, BlobCipherMetrics::KV_MEMORY)); - DecryptBlobCipherAes256Ctr cipher( - cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, cipherHeader.iv, BlobCipherMetrics::KV_MEMORY); - Arena arena; - StringRef plaintext = cipher - .decrypt(data.begin() + BlobCipherEncryptHeader::headerSize, - sizeof(int) + h->len1 + h->len2, - cipherHeader, - arena) - ->toStringRef(); - h->op = *(int*)plaintext.begin(); - return Standalone(plaintext.substr(sizeof(int)), arena); + state Arena arena; + state StringRef plaintext; + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + state BlobCipherEncryptHeaderRef cipherHeaderRef = + BlobCipherEncryptHeaderRef::fromStringRef(StringRef(data.begin(), encryptHeaderSize)); + TextAndHeaderCipherKeys cipherKeys = + wait(getEncryptCipherKeys(self->db, cipherHeaderRef, BlobCipherMetrics::KV_MEMORY)); + DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey, + cipherKeys.cipherHeaderKey, + cipherHeaderRef.getIV(), + BlobCipherMetrics::KV_MEMORY); + plaintext = cipher.decrypt(data.begin() + encryptHeaderSize, h.len1 + h.len2, cipherHeaderRef, arena); + } else { + state BlobCipherEncryptHeader cipherHeader = *(BlobCipherEncryptHeader*)data.begin(); + TextAndHeaderCipherKeys cipherKeys = + wait(getEncryptCipherKeys(self->db, cipherHeader, BlobCipherMetrics::KV_MEMORY)); + DecryptBlobCipherAes256Ctr cipher( + cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, cipherHeader.iv, BlobCipherMetrics::KV_MEMORY); + plaintext = + cipher.decrypt(data.begin() + encryptHeaderSize, h.len1 + h.len2, cipherHeader, arena)->toStringRef(); + } + return Standalone(plaintext, arena); } ACTOR static Future recover(KeyValueStoreMemory* self, bool exactRecovery) { @@ -586,6 +648,7 @@ private: try { loop { + state bool encryptedOp = false; { Standalone data = wait(self->log->readNext(sizeof(OpHeader))); if (data.size() != sizeof(OpHeader)) { @@ -595,9 +658,11 @@ private: memset(&h, 0, sizeof(OpHeader)); memcpy(&h, data.begin(), data.size()); zeroFillSize = sizeof(OpHeader) - data.size() + h.len1 + h.len2 + 1; - if (h.op == OpEncrypted) { - // encryption header, plus the real (encrypted) op type - zeroFillSize += BlobCipherEncryptHeader::headerSize + sizeof(int); + if (isOpEncrypted(&h)) { + // encrypt header size + encryption header + // If it's a partial header we assume the header size is 0 (this is fine since we + // don't read the header in this case) + zeroFillSize += 0 + sizeof(uint16_t); } } TraceEvent("KVSMemRecoveryComplete", self->id) @@ -609,8 +674,13 @@ private: break; } h = *(OpHeader*)data.begin(); + encryptedOp = isOpEncrypted(&h); + // Reset the first bit to 0 so the op can be read properly + setEncryptFlag(&h, false); + ASSERT(h.op != OpEncrypted_Deprecated); } - state Standalone data = wait(readOpData(self, &h, &isZeroFilled, &zeroFillSize)); + state Standalone data = + wait(readOpData(self, h, &isZeroFilled, &zeroFillSize, encryptedOp)); if (zeroFillSize > 0) { TraceEvent("KVSMemRecoveryComplete", self->id) .detail("Reason", "data specified by header does not exist") diff --git a/fdbserver/RESTKmsConnector.actor.cpp b/fdbserver/RESTKmsConnector.actor.cpp index caf9458909..b73d66e09f 100644 --- a/fdbserver/RESTKmsConnector.actor.cpp +++ b/fdbserver/RESTKmsConnector.actor.cpp @@ -778,10 +778,10 @@ Future kmsRequestImpl(Reference ctx, ACTOR Future fetchEncryptionKeysByKeyIds(Reference ctx, KmsConnLookupEKsByKeyIdsReq req) { state KmsConnLookupEKsByKeyIdsRep reply; + try { bool refreshKmsUrls = shouldRefreshKmsUrls(ctx); StringRef requestBodyRef = getEncryptKeysByKeyIdsRequestBody(ctx, req, refreshKmsUrls, req.arena); - std::function>(Reference, Reference)> f = &parseEncryptCipherResponse; @@ -1060,11 +1060,13 @@ ACTOR Future procureValidationTokensFromFiles(Reference procureValidationTokens(Reference ctx) { std::string_view mode{ SERVER_KNOBS->REST_KMS_CONNECTOR_VALIDATION_TOKEN_MODE }; + TraceEvent("ProcureValidationTokensStart", ctx->uid); if (mode.compare("file") == 0) { wait(procureValidationTokensFromFiles(ctx, SERVER_KNOBS->REST_KMS_CONNECTOR_VALIDATION_TOKEN_DETAILS)); } else { throw not_implemented(); } + TraceEvent("ProcureValidationTokensDone", ctx->uid); return Void(); } diff --git a/fdbserver/Resolver.actor.cpp b/fdbserver/Resolver.actor.cpp index ff2eabc6e4..0ff41b12d9 100644 --- a/fdbserver/Resolver.actor.cpp +++ b/fdbserver/Resolver.actor.cpp @@ -26,7 +26,6 @@ #include "fdbclient/SystemData.h" #include "fdbserver/ApplyMetadataMutation.h" #include "fdbserver/ConflictSet.h" -#include "fdbserver/EncryptionOpsUtils.h" #include "fdbserver/IKeyValueStore.h" #include "fdbserver/Knobs.h" #include "fdbserver/LogSystem.h" diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp index 3102e273a1..6b7285d6eb 100644 --- a/fdbserver/RestoreLoader.actor.cpp +++ b/fdbserver/RestoreLoader.actor.cpp @@ -372,13 +372,10 @@ void handleRestoreSysInfoRequest(const RestoreSysInfoRequest& req, Reference _decryptMutation(MutationRef mutation, Database cx, Arena* arena) { ASSERT(mutation.isEncrypted()); + Reference const> dbInfo = cx->clientInfo; - state const BlobCipherEncryptHeader* header = mutation.encryptionHeader(); std::unordered_set cipherDetails; - cipherDetails.insert(header->cipherTextDetails); - if (header->cipherHeaderDetails.isValid()) { - cipherDetails.insert(header->cipherHeaderDetails); - } + mutation.updateEncryptCipherDetails(cipherDetails); std::unordered_map> getCipherKeysResult = wait(getEncryptCipherKeys(dbInfo, cipherDetails, BlobCipherMetrics::BACKUP)); return mutation.decrypt(getCipherKeysResult, *arena, BlobCipherMetrics::BACKUP); diff --git a/fdbserver/SimKmsConnector.actor.cpp b/fdbserver/SimKmsConnector.actor.cpp index e1950adfd9..6cecd6cbe0 100644 --- a/fdbserver/SimKmsConnector.actor.cpp +++ b/fdbserver/SimKmsConnector.actor.cpp @@ -134,6 +134,9 @@ ACTOR Future ekLookupByIds(Reference ctx, getEncryptDbgTraceKey(ENCRYPT_DBG_TRACE_RESULT_PREFIX, item.domainId.get(), itr->first), ""); } } else { + TraceEvent("SimKmsEKLookupByIdsKeyNotFound") + .detail("DomId", item.domainId) + .detail("BaseCipherId", item.baseCipherId); success = false; break; } diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index ec538ec61f..279b632610 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -332,9 +332,6 @@ class TestConfig : public BasicTestConfig { if (attrib == "disableRemoteKVS") { disableRemoteKVS = strcmp(value.c_str(), "true") == 0; } - if (attrib == "disableEncryption") { - disableEncryption = strcmp(value.c_str(), "true") == 0; - } if (attrib == "encryptModes") { std::stringstream ss(value); std::string token; @@ -410,9 +407,6 @@ public: bool disableHostname = false; // remote key value store is a child process spawned by the SS process to run the storage engine bool disableRemoteKVS = false; - // 7.2 cannot be downgraded to 7.1 or below after enabling encryption-at-rest. - // TODO: Remove this bool once the encryption knobs are removed - bool disableEncryption = false; // By default, encryption mode is set randomly (based on the tenant mode) // If provided, set using EncryptionAtRestMode::fromString std::vector encryptModes; @@ -493,7 +487,6 @@ public: .add("disableTss", &disableTss) .add("disableHostname", &disableHostname) .add("disableRemoteKVS", &disableRemoteKVS) - .add("disableEncryption", &disableEncryption) .add("encryptModes", &encryptModes) .add("simpleConfig", &simpleConfig) .add("generateFearless", &generateFearless) @@ -561,6 +554,11 @@ public: } } + bool excludedStorageEngineType(int storageEngineType) const { + return std::find(storageEngineExcludeTypes.begin(), storageEngineExcludeTypes.end(), storageEngineType) != + storageEngineExcludeTypes.end(); + } + TestConfig() = default; TestConfig(const BasicTestConfig& config) : BasicTestConfig(config) {} }; @@ -1296,14 +1294,6 @@ ACTOR Future restartSimulatedSystem(std::vector>* systemActor g_knobs.setKnob("remote_kv_store", KnobValueRef::create(bool{ false })); TraceEvent(SevDebug, "DisableRemoteKVS"); } - // TODO: Remove this code when encryption knobs are removed - if (testConfig->disableEncryption) { - g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ false })); - g_knobs.setKnob("enable_tlog_encryption", KnobValueRef::create(bool{ false })); - g_knobs.setKnob("enable_storage_server_encryption", KnobValueRef::create(bool{ false })); - g_knobs.setKnob("enable_blob_granule_encryption", KnobValueRef::create(bool{ false })); - TraceEvent(SevDebug, "DisableEncryption"); - } *pConnString = conn; *pTesterCount = testerCount; bool usingSSL = conn.toString().find(":tls") != std::string::npos || listenersPerProcess > 1; @@ -1596,35 +1586,54 @@ void SimulationConfig::setTenantMode(const TestConfig& testConfig) { } void SimulationConfig::setEncryptionAtRestMode(const TestConfig& testConfig) { - EncryptionAtRestMode encryptionMode = EncryptionAtRestMode::DISABLED; - // Only Redwood support encryption. Disable encryption if non-Redwood storage engine is explicitly specified. - bool disableEncryption = testConfig.disableEncryption || - (testConfig.storageEngineType.present() && testConfig.storageEngineType.get() != 3); - // TODO: Remove check on the ENABLE_ENCRYPTION knob once the EKP can start using the db config - if (!disableEncryption && (SERVER_KNOBS->ENABLE_ENCRYPTION || !testConfig.encryptModes.empty())) { - TenantMode tenantMode = db.tenantMode; - if (!testConfig.encryptModes.empty()) { - std::vector validEncryptModes; - // Get the subset of valid encrypt modes given the tenant mode - for (int i = 0; i < testConfig.encryptModes.size(); i++) { - EncryptionAtRestMode encryptMode = EncryptionAtRestMode::fromString(testConfig.encryptModes.at(i)); - if (encryptMode != EncryptionAtRestMode::DOMAIN_AWARE || tenantMode == TenantMode::REQUIRED) { - validEncryptModes.push_back(encryptMode); - } - } - if (validEncryptModes.size() > 0) { - encryptionMode = deterministicRandom()->randomChoice(validEncryptModes); - } - } else { - // TODO: These cases should only trigger with probability (BUGGIFY) once the server knob is removed - if (tenantMode == TenantMode::DISABLED || tenantMode == TenantMode::OPTIONAL_TENANT || BUGGIFY) { - // optional and disabled tenant modes currently only support cluster aware encryption - encryptionMode = EncryptionAtRestMode::CLUSTER_AWARE; - } else { - encryptionMode = EncryptionAtRestMode::DOMAIN_AWARE; - } + std::vector available; + std::vector probability; + if (!testConfig.encryptModes.empty()) { + // If encryptModes are specified explicitly, give them equal probability to be chosen. + available = std::vector(EncryptionAtRestMode::END, false); + probability = std::vector(EncryptionAtRestMode::END, 0); + for (auto& mode : testConfig.encryptModes) { + available[EncryptionAtRestMode::fromString(mode).mode] = true; + probability[EncryptionAtRestMode::fromString(mode).mode] = 1.0 / testConfig.encryptModes.size(); + } + } else { + // If encryptModes are not specified, give encryption higher chance to be enabled. + // The good thing is testing with encryption on doesn't loss test coverage for most of the other features. + available = std::vector(EncryptionAtRestMode::END, true); + probability = { 0.25, 0.5, 0.25 }; + // Only Redwood support encryption. Disable encryption if Redwood is not available. + if ((testConfig.storageEngineType.present() && testConfig.storageEngineType != 3) || + testConfig.excludedStorageEngineType(3)) { + available[(int)EncryptionAtRestMode::DOMAIN_AWARE] = false; + available[(int)EncryptionAtRestMode::CLUSTER_AWARE] = false; } } + // domain_aware mode is supported only with required tenant mode. + if (db.tenantMode != TenantMode::REQUIRED) { + available[(int)EncryptionAtRestMode::DOMAIN_AWARE] = false; + } + int lastAvailableMode = EncryptionAtRestMode::END; + double totalProbability = 0; + for (int mode = 0; mode < (int)EncryptionAtRestMode::END; mode++) { + if (available[mode]) { + lastAvailableMode = mode; + totalProbability += probability[mode]; + } + } + ASSERT(lastAvailableMode != EncryptionAtRestMode::END); // At least one mode available + double r = deterministicRandom()->random01() * totalProbability; + EncryptionAtRestMode encryptionMode; + for (int mode = 0;; mode++) { + if (available[mode] && (r < probability[mode] || mode == lastAvailableMode)) { + encryptionMode = (EncryptionAtRestMode::Mode)mode; + break; + } + r -= probability[mode]; + } + TraceEvent("SimulatedClusterEncryptionMode").detail("Mode", encryptionMode.toString()); + CODE_PROBE(encryptionMode == EncryptionAtRestMode::DISABLED, "Disabled encryption in simulation"); + CODE_PROBE(encryptionMode == EncryptionAtRestMode::CLUSTER_AWARE, "Enabled cluster-aware encryption in simulation"); + CODE_PROBE(encryptionMode == EncryptionAtRestMode::DOMAIN_AWARE, "Enabled domain-aware encryption in simulation"); set_config("encryption_at_rest_mode=" + encryptionMode.toString()); } @@ -1640,9 +1649,7 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) { storage_engine_type = testConfig.storageEngineType.get(); } else { // Continuously re-pick the storage engine type if it's the one we want to exclude - while (std::find(testConfig.storageEngineExcludeTypes.begin(), - testConfig.storageEngineExcludeTypes.end(), - storage_engine_type) != testConfig.storageEngineExcludeTypes.end()) { + while (testConfig.excludedStorageEngineType(storage_engine_type)) { storage_engine_type = deterministicRandom()->randomInt(0, 6); } } @@ -2132,19 +2139,6 @@ void setupSimulatedSystem(std::vector>* systemActors, simconfig.db.tLogWriteAntiQuorum = testConfig.logAntiQuorum; } - // TODO: remove knob hanlding once we move off from encrypption knobs to db config - if (simconfig.db.encryptionAtRestMode.mode == EncryptionAtRestMode::DISABLED) { - g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ false })); - CODE_PROBE(true, "Disabled encryption in simulation"); - } else { - g_knobs.setKnob("enable_encryption", KnobValueRef::create(bool{ true })); - CODE_PROBE(simconfig.db.encryptionAtRestMode.mode == EncryptionAtRestMode::CLUSTER_AWARE, - "Enabled cluster-aware encryption in simulation"); - CODE_PROBE(simconfig.db.encryptionAtRestMode.mode == EncryptionAtRestMode::DOMAIN_AWARE, - "Enabled domain-aware encryption in simulation"); - } - TraceEvent("SimulatedClusterEncryptionMode").detail("Mode", simconfig.db.encryptionAtRestMode.toString()); - g_simulator->blobGranulesEnabled = simconfig.db.blobGranulesEnabled; StatusObject startingConfigJSON = simconfig.db.toJSON(true); diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 59b34a63b0..40b4a63d50 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -854,7 +854,7 @@ ACTOR static Future processStatusFetcher( roles.addRole("consistency_scan", db->get().consistencyScan.get()); } - if (SERVER_KNOBS->ENABLE_ENCRYPTION && db->get().encryptKeyProxy.present()) { + if (db->get().encryptKeyProxy.present()) { roles.addRole("encrypt_key_proxy", db->get().encryptKeyProxy.get()); } diff --git a/fdbserver/StorageCache.actor.cpp b/fdbserver/StorageCache.actor.cpp index a097b5e414..ae884454db 100644 --- a/fdbserver/StorageCache.actor.cpp +++ b/fdbserver/StorageCache.actor.cpp @@ -1925,11 +1925,7 @@ ACTOR Future pullAsyncData(StorageCacheData* data) { cloneReader >> msg; if (msg.isEncrypted()) { if (!cipherKeys.present()) { - const BlobCipherEncryptHeader* header = msg.encryptionHeader(); - cipherDetails.insert(header->cipherTextDetails); - if (header->cipherHeaderDetails.isValid()) { - cipherDetails.insert(header->cipherHeaderDetails); - } + msg.updateEncryptCipherDetails(cipherDetails); collectingCipherKeys = true; } else { msg = msg.decrypt(cipherKeys.get(), cloneReader.arena(), BlobCipherMetrics::TLOG); diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp index 18cdb42902..2fa585d0e2 100644 --- a/fdbserver/TLogServer.actor.cpp +++ b/fdbserver/TLogServer.actor.cpp @@ -2437,15 +2437,7 @@ ACTOR Future getEncryptionAtRestMode(TLogData* self) { when(GetEncryptionAtRestModeResponse resp = wait(brokenPromiseToNever( self->dbInfo->get().clusterInterface.getEncryptionAtRestMode.getReply(req)))) { TraceEvent("GetEncryptionAtRestMode", self->dbgid).detail("Mode", resp.mode); - - // TODO: TLOG_ENCTYPTION KNOB shall be removed and db-config check should be sufficient to - // determine tlog (and cluster) encryption status - if ((EncryptionAtRestMode::Mode)resp.mode != EncryptionAtRestMode::Mode::DISABLED && - SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION) { - return EncryptionAtRestMode((EncryptionAtRestMode::Mode)resp.mode); - } else { - return EncryptionAtRestMode(); - } + return (EncryptionAtRestMode::Mode)resp.mode; } } } catch (Error& e) { diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 0aadc60607..049a8ee1f8 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -2815,6 +2815,13 @@ public: wait(self->keyProviderInitialized.getFuture()); ASSERT(self->keyProvider.isValid()); } + if (self->keyProvider->expectedEncodingType() != page->getEncodingType()) { + TraceEvent(SevWarnAlways, "RedwoodBTreeUnexpectedNodeEncoding") + .detail("PhysicalPageID", page->getPhysicalPageID()) + .detail("EncodingTypeFound", page->getEncodingType()) + .detail("EncodingTypeExpected", self->keyProvider->expectedEncodingType()); + throw unexpected_encoding_type(); + } ArenaPage::EncryptionKey k = wait(self->keyProvider->getEncryptionKey(page->getEncodingHeader())); page->encryptionKey = k; } @@ -2883,6 +2890,17 @@ public: try { page->postReadHeader(pageIDs.front()); if (page->isEncrypted()) { + if (!self->keyProvider.isValid()) { + wait(self->keyProviderInitialized.getFuture()); + ASSERT(self->keyProvider.isValid()); + } + if (self->keyProvider->expectedEncodingType() != page->getEncodingType()) { + TraceEvent(SevWarnAlways, "RedwoodBTreeUnexpectedNodeEncoding") + .detail("PhysicalPageID", page->getPhysicalPageID()) + .detail("EncodingTypeFound", page->getEncodingType()) + .detail("EncodingTypeExpected", self->keyProvider->expectedEncodingType()); + throw unexpected_encoding_type(); + } ArenaPage::EncryptionKey k = wait(self->keyProvider->getEncryptionKey(page->getEncodingHeader())); page->encryptionKey = k; } @@ -3595,7 +3613,7 @@ public: // The next section explicitly cancels all pending operations held in the pager debug_printf("DWALPager(%s) shutdown kill ioLock\n", self->filename.c_str()); - self->ioLock->kill(); + self->ioLock->halt(); debug_printf("DWALPager(%s) shutdown cancel recovery\n", self->filename.c_str()); self->recoverFuture.cancel(); @@ -3657,7 +3675,14 @@ public: } else { g_network->getDiskBytes(parentDirectory(filename), free, total); } - int64_t pagerSize = header.pageCount * physicalPageSize; + + // Size of redwood data file. Note that filePageCountPending is used here instead of filePageCount. This is + // because it is always >= filePageCount and accounts for file size changes which will complete soon. + int64_t pagerPhysicalSize = filePageCountPending * physicalPageSize; + + // Size of the pager, which can be less than the data file size. All pages within this size are either in use + // in a data structure or accounted for in one of the pager's free page lists. + int64_t pagerLogicalSize = header.pageCount * physicalPageSize; // It is not exactly known how many pages on the delayed free list are usable as of right now. It could be // known, if each commit delayed entries that were freeable were shuffled from the delayed free queue to the @@ -3668,12 +3693,17 @@ public: // Amount of space taken up by the free list queues themselves, as if we were to pop and use // items on the free lists the space the items are stored in would also become usable int64_t reusableQueueSpace = (freeList.numPages + delayedFreeList.numPages) * physicalPageSize; - int64_t reusable = reusablePageSpace + reusableQueueSpace; + + // Pager slack is the space at the end of the pager's logical size until the end of the pager file's size. + // These pages will be used if needed without growing the file size. + int64_t reusablePagerSlackSpace = pagerPhysicalSize - pagerLogicalSize; + + int64_t reusable = reusablePageSpace + reusableQueueSpace + reusablePagerSlackSpace; // Space currently in used by old page versions have have not yet been freed due to the remap cleanup window. int64_t temp = remapQueue.numEntries * physicalPageSize; - return StorageBytes(free, total, pagerSize - reusable, free + reusable, temp); + return StorageBytes(free, total, pagerPhysicalSize, free + reusable, temp); } int64_t getPageCacheCount() override { return pageCache.getCount(); } @@ -5255,6 +5285,7 @@ public: .detail("InstanceName", self->m_pager->getName()) .detail("UsingEncodingType", self->m_encodingType) .detail("ExistingEncodingType", self->m_header.encodingType); + throw unexpected_encoding_type(); } // Verify if encryption mode and encoding type in the header are consistent. // This check can also fail in case of authentication mode mismatch. @@ -6231,18 +6262,6 @@ private: metrics.pageRead += 1; metrics.pageReadExt += (id.size() - 1); - // If BTree encryption is enabled, pages read must be encrypted using the desired encryption type - if (self->m_enforceEncodingType && (page->getEncodingType() != self->m_encodingType)) { - Error e = unexpected_encoding_type(); - TraceEvent(SevWarnAlways, "RedwoodBTreeUnexpectedNodeEncoding") - .error(e) - .detail("PhysicalPageID", page->getPhysicalPageID()) - .detail("IsEncrypted", page->isEncrypted()) - .detail("EncodingTypeFound", page->getEncodingType()) - .detail("EncodingTypeExpected", self->m_encodingType); - throw e; - } - return std::move(page); } @@ -11448,8 +11467,8 @@ TEST_CASE("/redwood/correctness/EnforceEncodingType") { {}, // encryptionMode reopenEncodingType, encryptionKeyProviders.at(reopenEncodingType)); - wait(kvs->init()); try { + wait(kvs->init()); Optional v = wait(kvs->readValue("foo"_sr)); UNREACHABLE(); } catch (Error& e) { diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index a042b32280..357207e687 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -2255,14 +2255,6 @@ int main(int argc, char* argv[]) { } } } - g_knobs.setKnob("enable_encryption", - KnobValue::create(ini.GetBoolValue("META", "enableEncryption", false))); - g_knobs.setKnob("enable_tlog_encryption", - KnobValue::create(ini.GetBoolValue("META", "enableTLogEncryption", false))); - g_knobs.setKnob("enable_storage_server_encryption", - KnobValue::create(ini.GetBoolValue("META", "enableStorageServerEncryption", false))); - g_knobs.setKnob("enable_blob_granule_encryption", - KnobValue::create(ini.GetBoolValue("META", "enableBlobGranuleEncryption", false))); g_knobs.setKnob("enable_blob_granule_compression", KnobValue::create(ini.GetBoolValue("META", "enableBlobGranuleEncryption", false))); g_knobs.setKnob("encrypt_header_auth_token_enabled", @@ -2270,6 +2262,11 @@ int main(int argc, char* argv[]) { g_knobs.setKnob("encrypt_header_auth_token_algo", KnobValue::create((int)ini.GetLongValue( "META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO))); + g_knobs.setKnob("enable_configurable_encryption", + KnobValue::create(ini.GetBoolValue("META", + "enableConfigurableEncryption", + CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION))); + g_knobs.setKnob( "shard_encode_location_metadata", KnobValue::create(ini.GetBoolValue("META", "enableShardEncodeLocationMetadata", false))); diff --git a/fdbserver/include/fdbserver/BlobGranuleServerCommon.actor.h b/fdbserver/include/fdbserver/BlobGranuleServerCommon.actor.h index c52ad6e967..5a04573107 100644 --- a/fdbserver/include/fdbserver/BlobGranuleServerCommon.actor.h +++ b/fdbserver/include/fdbserver/BlobGranuleServerCommon.actor.h @@ -171,6 +171,7 @@ ACTOR Future> getRestoreStatus(Database db, KeyRange ACTOR Future> getRestoreArg(Database db, KeyRangeRef range); ACTOR Future getRestoreTargetVersion(Database db, KeyRangeRef range, Version defaultVersion); ACTOR Future getManifestVersion(Database db); +ACTOR Future getMutationLogUrl(); #include "flow/unactorcompiler.h" #endif diff --git a/fdbserver/include/fdbserver/EncryptionOpsUtils.h b/fdbserver/include/fdbserver/EncryptionOpsUtils.h deleted file mode 100644 index 8b40caa8ce..0000000000 --- a/fdbserver/include/fdbserver/EncryptionOpsUtils.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * EncryptionOpUtils.h - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FDBSERVER_ENCRYPTION_OPS_UTIL_H -#define FDBSERVER_ENCRYPTION_OPS_UTIL_H -#pragma once - -#include "fdbserver/Knobs.h" -#include "fdbclient/CommitProxyInterface.h" - -typedef enum { TLOG_ENCRYPTION = 0, STORAGE_SERVER_ENCRYPTION = 1, BLOB_GRANULE_ENCRYPTION = 2 } EncryptOperationType; - -inline bool isEncryptionOpSupported(EncryptOperationType operation_type) { - // We would check against dbInfo.isEncryptionEnabled instead, but the dbInfo may not be available before - // ClusterController broadcast the dbInfo to workers. Before the broadcast encryption may appear to be disabled - // when it should be enabled. Moving the encryption switch to DB config could fix the issue. - if (!SERVER_KNOBS->ENABLE_ENCRYPTION) { - return false; - } - - if (operation_type == TLOG_ENCRYPTION) { - return SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION; - } else if (operation_type == STORAGE_SERVER_ENCRYPTION) { - return SERVER_KNOBS->ENABLE_STORAGE_SERVER_ENCRYPTION; - } else if (operation_type == BLOB_GRANULE_ENCRYPTION) { - bool supported = SERVER_KNOBS->ENABLE_BLOB_GRANULE_ENCRYPTION && SERVER_KNOBS->BG_METADATA_SOURCE == "tenant"; - ASSERT((supported && SERVER_KNOBS->ENABLE_ENCRYPTION) || !supported); - return supported; - } else { - return false; - } -} - -#endif // FDBSERVER_ENCRYPTION_OPS_UTIL_H diff --git a/fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h b/fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h index 488666c7e4..87c426b269 100644 --- a/fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h +++ b/fdbserver/include/fdbserver/IPageEncryptionKeyProvider.actor.h @@ -18,6 +18,7 @@ * limitations under the License. */ +#include "fdbclient/Knobs.h" #include "fdbclient/TenantManagement.actor.h" #include "fdbrpc/TenantInfo.h" #if defined(NO_INTELLISENSE) && !defined(FDBSERVER_IPAGEENCRYPTIONKEYPROVIDER_ACTOR_G_H) @@ -31,7 +32,6 @@ #include "fdbclient/SystemData.h" #include "fdbclient/Tenant.h" -#include "fdbserver/EncryptionOpsUtils.h" #include "fdbserver/IPager.h" #include "fdbserver/Knobs.h" #include "fdbserver/ServerDBInfo.h" @@ -44,6 +44,7 @@ #include #include #include +#include #include "flow/actorcompiler.h" // This must be the last #include. @@ -158,6 +159,22 @@ public: uint8_t xorWith; }; +namespace { +template +int64_t getEncryptionDomainIdFromAesEncryptionHeader(const void* encodingHeader) { + using Encoder = typename ArenaPage::AESEncryptionEncoder; + using EncodingHeader = typename Encoder::Header; + ASSERT(encodingHeader != nullptr); + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader); + return headerRef.getCipherDetails().textCipherDetails.encryptDomainId; + } else { + const BlobCipherEncryptHeader& header = reinterpret_cast(encodingHeader)->encryption; + return header.cipherTextDetails.encryptDomainId; + } +} +} // anonymous namespace + // Key provider to provider cipher keys randomly from a pre-generated pool. It does not maintain encryption domains. // Use for testing. template 0; } Future getEncryptionKey(const void* encodingHeader) override { - using Header = typename ArenaPage::AESEncryptionEncoder::Header; - const Header* h = reinterpret_cast(encodingHeader); + using Encoder = typename ArenaPage::AESEncryptionEncoder; EncryptionKey s; - s.aesKey.cipherTextKey = - getCipherKey(h->encryption.cipherTextDetails.encryptDomainId, h->encryption.cipherTextDetails.baseCipherId); - s.aesKey.cipherHeaderKey = getCipherKey(h->encryption.cipherHeaderDetails.encryptDomainId, - h->encryption.cipherHeaderDetails.baseCipherId); + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + const BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader); + EncryptHeaderCipherDetails details = headerRef.getCipherDetails(); + ASSERT(details.textCipherDetails.isValid()); + s.aesKey.cipherTextKey = + getCipherKey(details.textCipherDetails.encryptDomainId, details.textCipherDetails.baseCipherId); + if (details.headerCipherDetails.present()) { + ASSERT(details.headerCipherDetails.get().isValid()); + s.aesKey.cipherHeaderKey = getCipherKey(details.headerCipherDetails.get().encryptDomainId, + details.headerCipherDetails.get().baseCipherId); + } + } else { + const typename Encoder::Header* h = reinterpret_cast(encodingHeader); + s.aesKey.cipherTextKey = getCipherKey(h->encryption.cipherTextDetails.encryptDomainId, + h->encryption.cipherTextDetails.baseCipherId); + if (h->encryption.cipherHeaderDetails.isValid()) { + s.aesKey.cipherHeaderKey = getCipherKey(h->encryption.cipherHeaderDetails.encryptDomainId, + h->encryption.cipherHeaderDetails.baseCipherId); + } + } return s; } Future getLatestEncryptionKey(int64_t domainId) override { domainId = checkDomainId(domainId); EncryptionKey s; - s.aesKey.cipherTextKey = getCipherKey(domainId, deterministicRandom()->randomInt(0, NUM_CIPHER)); + s.aesKey.cipherTextKey = getCipherKey(domainId, deterministicRandom()->randomInt(1, NUM_CIPHER + 1)); s.aesKey.cipherHeaderKey = - getCipherKey(ENCRYPT_HEADER_DOMAIN_ID, deterministicRandom()->randomInt(0, NUM_CIPHER)); + getCipherKey(ENCRYPT_HEADER_DOMAIN_ID, deterministicRandom()->randomInt(1, NUM_CIPHER + 1)); return s; } @@ -222,10 +254,7 @@ public: } int64_t getEncryptionDomainIdFromHeader(const void* encodingHeader) override { - ASSERT(encodingHeader != nullptr); - using Header = typename ArenaPage::AESEncryptionEncoder::Header; - const Header* h = reinterpret_cast(encodingHeader); - return h->encryption.cipherTextDetails.encryptDomainId; + return getEncryptionDomainIdFromAesEncryptionHeader(encodingHeader); } private: @@ -260,11 +289,12 @@ private: Reference getCipherKey(EncryptCipherDomainId domainId, EncryptCipherBaseKeyId cipherId) { // Create a new cipher key by replacing the domain id. + ASSERT(cipherId > 0 && cipherId <= NUM_CIPHER); return makeReference(domainId, cipherId, - cipherKeys[cipherId]->rawBaseCipher(), + cipherKeys[cipherId - 1]->rawBaseCipher(), AES_256_KEY_LENGTH, - cipherKeys[cipherId]->getSalt(), + cipherKeys[cipherId - 1]->getSalt(), std::numeric_limits::max() /* refreshAt */, std::numeric_limits::max() /* expireAt */); } @@ -282,7 +312,8 @@ template class AESEncryptionKeyProvider : public IPageEncryptionKeyProvider { public: - using EncodingHeader = typename ArenaPage::AESEncryptionEncoder::Header; + using Encoder = typename ArenaPage::AESEncryptionEncoder; + using EncodingHeader = typename Encoder::Header; const StringRef systemKeysPrefix = systemKeys.begin; @@ -303,9 +334,17 @@ public: } ACTOR static Future getEncryptionKey(AESEncryptionKeyProvider* self, const void* encodingHeader) { - const BlobCipherEncryptHeader& header = reinterpret_cast(encodingHeader)->encryption; - TextAndHeaderCipherKeys cipherKeys = - wait(getEncryptCipherKeys(self->db, header, BlobCipherMetrics::KV_REDWOOD)); + state TextAndHeaderCipherKeys cipherKeys; + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + BlobCipherEncryptHeaderRef headerRef = Encoder::getEncryptionHeaderRef(encodingHeader); + TextAndHeaderCipherKeys cks = + wait(getEncryptCipherKeys(self->db, headerRef, BlobCipherMetrics::KV_REDWOOD)); + cipherKeys = cks; + } else { + const BlobCipherEncryptHeader& header = reinterpret_cast(encodingHeader)->encryption; + TextAndHeaderCipherKeys cks = wait(getEncryptCipherKeys(self->db, header, BlobCipherMetrics::KV_REDWOOD)); + cipherKeys = cks; + } EncryptionKey encryptionKey; encryptionKey.aesKey = cipherKeys; return encryptionKey; @@ -355,9 +394,7 @@ public: } int64_t getEncryptionDomainIdFromHeader(const void* encodingHeader) override { - ASSERT(encodingHeader != nullptr); - const BlobCipherEncryptHeader& header = reinterpret_cast(encodingHeader)->encryption; - return header.cipherTextDetails.encryptDomainId; + return getEncryptionDomainIdFromAesEncryptionHeader(encodingHeader); } private: diff --git a/fdbserver/include/fdbserver/IPager.h b/fdbserver/include/fdbserver/IPager.h index 8c06cb75cb..43f81ce941 100644 --- a/fdbserver/include/fdbserver/IPager.h +++ b/fdbserver/include/fdbserver/IPager.h @@ -19,6 +19,7 @@ */ #pragma once +#include "fdbclient/Knobs.h" #ifndef FDBSERVER_IPAGER_H #define FDBSERVER_IPAGER_H @@ -387,23 +388,40 @@ public: // By default, xxhash is used to checksum the page. But ff authentication is enabled (such as when we are using // aes256-ctr-hmac-sha256 encryption scheme), the auth tag plays the role of a checksum while assuring authenticity // of the data. xxhash checksum is not needed in this case. + // + // To support configurable encryption, which may come with variable size encryption header, we assume the encryption + // header size is no larger than that of BlobCipherEncryptHeader. This is true for current supported encryption + // header format types. Moving forward, the plan is to make IPager support variable size encoding header, and let + // Redwood rebuild a page when it tries to in-place update the page, but the reserved buffer for the encoding header + // is not large enough. + // TODO(yiwu): Cleanup the old encryption header, and update headerSize to be the maximum size of the supported + // encryption header format type. + // TODO(yiwu): Support variable size encoding header. template ::type = true> struct AESEncryptionEncoder { struct AESEncryptionEncodingHeader { - BlobCipherEncryptHeader encryption; XXH64_hash_t checksum; + union { + BlobCipherEncryptHeader encryption; + uint8_t encryptionHeaderBuf[0]; // for configurable encryption + }; }; struct AESEncryptionWithAuthEncodingHeader { - BlobCipherEncryptHeader encryption; + union { + BlobCipherEncryptHeader encryption; + uint8_t encryptionHeaderBuf[0]; // for configurable encryption + }; }; using Header = typename std::conditional::type; + static constexpr size_t headerSize = sizeof(Header); + static void encode(void* header, const TextAndHeaderCipherKeys& cipherKeys, uint8_t* payload, @@ -415,7 +433,19 @@ public: getEncryptAuthTokenMode(ENCRYPT_HEADER_AUTH_TOKEN_MODE_SINGLE), BlobCipherMetrics::KV_REDWOOD); Arena arena; - StringRef ciphertext = cipher.encrypt(payload, len, &h->encryption, arena)->toStringRef(); + StringRef ciphertext; + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + BlobCipherEncryptHeaderRef headerRef; + ciphertext = cipher.encrypt(payload, len, &headerRef, arena); + Standalone serializedHeader = BlobCipherEncryptHeaderRef::toStringRef(headerRef); + ASSERT(serializedHeader.size() <= headerSize); + memcpy(h->encryptionHeaderBuf, serializedHeader.begin(), serializedHeader.size()); + if (serializedHeader.size() < headerSize) { + memset(h->encryptionHeaderBuf + serializedHeader.size(), 0, headerSize - serializedHeader.size()); + } + } else { + ciphertext = cipher.encrypt(payload, len, &h->encryption, arena)->toStringRef(); + } ASSERT_EQ(len, ciphertext.size()); memcpy(payload, ciphertext.begin(), len); if constexpr (encodingType == AESEncryption) { @@ -423,6 +453,13 @@ public: } } + static BlobCipherEncryptHeaderRef getEncryptionHeaderRef(const void* header) { + ASSERT(CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION); + const Header* h = reinterpret_cast(header); + return BlobCipherEncryptHeaderRef::fromStringRef( + StringRef(h->encryptionHeaderBuf, headerSize - (h->encryptionHeaderBuf - (const uint8_t*)h))); + } + static void decode(void* header, const TextAndHeaderCipherKeys& cipherKeys, uint8_t* payload, @@ -434,10 +471,22 @@ public: throw page_decoding_failed(); } } - DecryptBlobCipherAes256Ctr cipher( - cipherKeys.cipherTextKey, cipherKeys.cipherHeaderKey, h->encryption.iv, BlobCipherMetrics::KV_REDWOOD); Arena arena; - StringRef plaintext = cipher.decrypt(payload, len, h->encryption, arena)->toStringRef(); + StringRef plaintext; + if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { + BlobCipherEncryptHeaderRef headerRef = getEncryptionHeaderRef(header); + DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey, + cipherKeys.cipherHeaderKey, + headerRef.getIV(), + BlobCipherMetrics::KV_REDWOOD); + plaintext = cipher.decrypt(payload, len, headerRef, arena); + } else { + DecryptBlobCipherAes256Ctr cipher(cipherKeys.cipherTextKey, + cipherKeys.cipherHeaderKey, + h->encryption.iv, + BlobCipherMetrics::KV_REDWOOD); + plaintext = cipher.decrypt(payload, len, h->encryption, arena)->toStringRef(); + } ASSERT_EQ(len, plaintext.size()); memcpy(payload, plaintext.begin(), len); } diff --git a/fdbserver/include/fdbserver/ProxyCommitData.actor.h b/fdbserver/include/fdbserver/ProxyCommitData.actor.h index 11c2e20181..0c3cf8c186 100644 --- a/fdbserver/include/fdbserver/ProxyCommitData.actor.h +++ b/fdbserver/include/fdbserver/ProxyCommitData.actor.h @@ -19,8 +19,6 @@ */ #pragma once -#include "fdbserver/EncryptionOpsUtils.h" -#include #if defined(NO_INTELLISENSE) && !defined(FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H) #define FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H #include "fdbserver/ProxyCommitData.actor.g.h" diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 0b68c51d77..632fd2cf0f 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -159,6 +159,7 @@ bool canReplyWith(Error e) { return false; } } + } // namespace #define PERSIST_PREFIX "\xff\xff" @@ -854,7 +855,6 @@ public: void clearTenants(StringRef startTenant, StringRef endTenant, Version version); void checkTenantEntry(Version version, TenantInfo tenant); - KeyRangeRef clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena); std::vector getStorageServerShards(KeyRangeRef range); @@ -2078,6 +2078,7 @@ ACTOR Future waitForMinVersion(StorageServer* data, Version version) { void StorageServer::checkTenantEntry(Version version, TenantInfo tenantInfo) { if (tenantInfo.hasTenant()) { + ASSERT(version == latestVersion || (version >= tenantMap.oldestVersion && version <= this->version.get())); auto view = tenantMap.at(version); auto itr = view.find(tenantInfo.tenantId); if (itr == view.end()) { @@ -3020,11 +3021,7 @@ ACTOR Future> getChangeFeedMutations(Stor if (doFilterMutations || !req.encrypted) { for (auto& m : decodedMutations.back().first) { if (m.isEncrypted()) { - const BlobCipherEncryptHeader* header = m.encryptionHeader(); - cipherDetails.insert(header->cipherTextDetails); - if (header->cipherHeaderDetails.isValid()) { - cipherDetails.insert(header->cipherHeaderDetails); - } + m.updateEncryptCipherDetails(cipherDetails); } } } @@ -4038,17 +4035,6 @@ ACTOR Future readRange(StorageServer* data, return result; } -KeyRangeRef StorageServer::clampRangeToTenant(KeyRangeRef range, TenantInfo const& tenantInfo, Arena& arena) { - if (tenantInfo.hasTenant()) { - return KeyRangeRef(range.begin.startsWith(tenantInfo.prefix.get()) ? range.begin : tenantInfo.prefix.get(), - range.end.startsWith(tenantInfo.prefix.get()) - ? range.end - : allKeys.end.withPrefix(tenantInfo.prefix.get(), arena)); - } else { - return range; - } -} - ACTOR Future findKey(StorageServer* data, KeySelectorRef sel, Version version, @@ -4250,7 +4236,7 @@ ACTOR Future getKeyValuesQ(StorageServer* data, GetKeyValuesRequest req) throw wrong_shard_server(); } - KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena); + KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena); state int offset1 = 0; state int offset2; @@ -5415,7 +5401,7 @@ ACTOR Future getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe throw wrong_shard_server(); } - KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena); + KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena); state int offset1 = 0; state int offset2; @@ -5616,7 +5602,7 @@ ACTOR Future getKeyValuesStreamQ(StorageServer* data, GetKeyValuesStreamRe throw wrong_shard_server(); } - KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena); + KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena); state int offset1 = 0; state int offset2; @@ -5806,7 +5792,7 @@ ACTOR Future getKeyQ(StorageServer* data, GetKeyRequest req) { state uint64_t changeCounter = data->shardChangeCounter; KeyRange shard = getShardKeyRange(data, req.sel); - KeyRangeRef searchRange = data->clampRangeToTenant(shard, req.tenantInfo, req.arena); + KeyRangeRef searchRange = TenantAPI::clampRangeToTenant(shard, req.tenantInfo, req.arena); state int offset; Key absoluteKey = wait(findKey(data, req.sel, version, searchRange, &offset, req.spanContext, req.options)); @@ -6854,9 +6840,7 @@ ACTOR Future fetchChangeFeedApplier(StorageServer* data, data->counters.feedBytesFetched += remoteResult.expectedSize(); data->fetchKeysBytesBudget -= remoteResult.expectedSize(); - if (data->fetchKeysBytesBudget <= 0) { - data->fetchKeysBudgetUsed.set(true); - } + data->fetchKeysBudgetUsed.set(data->fetchKeysBytesBudget <= 0); wait(yield()); } } catch (Error& e) { @@ -7574,16 +7558,15 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { metricReporter.addFetchedBytes(expectedBlockSize, this_block.size()); // Write this_block to storage + state int sinceYield = 0; state KeyValueRef* kvItr = this_block.begin(); for (; kvItr != this_block.end(); ++kvItr) { data->storage.writeKeyValue(*kvItr); - wait(yield()); - } - - kvItr = this_block.begin(); - for (; kvItr != this_block.end(); ++kvItr) { data->byteSampleApplySet(*kvItr, invalidVersion); - wait(yield()); + if (++sinceYield > 1000) { + wait(yield()); + sinceYield = 0; + } } ASSERT(this_block.readThrough.present() || this_block.size()); @@ -7592,9 +7575,7 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { this_block = RangeResult(); data->fetchKeysBytesBudget -= expectedBlockSize; - if (data->fetchKeysBytesBudget <= 0) { - data->fetchKeysBudgetUsed.set(true); - } + data->fetchKeysBudgetUsed.set(data->fetchKeysBytesBudget <= 0); } } catch (Error& e) { if (e.code() != error_code_end_of_stream && e.code() != error_code_connection_failed && @@ -9257,11 +9238,7 @@ ACTOR Future update(StorageServer* data, bool* pReceivedUpdate) { } if (msg.isEncrypted()) { if (!cipherKeys.present()) { - const BlobCipherEncryptHeader* header = msg.encryptionHeader(); - cipherDetails.insert(header->cipherTextDetails); - if (header->cipherHeaderDetails.isValid()) { - cipherDetails.insert(header->cipherHeaderDetails); - } + msg.updateEncryptCipherDetails(cipherDetails); collectingCipherKeys = true; } else { msg = msg.decrypt(cipherKeys.get(), eager.arena, BlobCipherMetrics::TLOG); @@ -9668,6 +9645,8 @@ ACTOR Future createCheckpoint(StorageServer* data, CheckpointMetaData meta ACTOR Future updateStorage(StorageServer* data) { state UnlimitedCommitBytes unlimitedCommitBytes = UnlimitedCommitBytes::False; + state Future durableDelay = Void(); + loop { unlimitedCommitBytes = UnlimitedCommitBytes::False; ASSERT(data->durableVersion.get() == data->storageVersion()); @@ -9678,7 +9657,19 @@ ACTOR Future updateStorage(StorageServer* data) { wait(delay(endTime - now(), TaskPriority::UpdateStorage)); } } - wait(data->desiredOldestVersion.whenAtLeast(data->storageVersion() + 1)); + + // If the fetch keys budget is not used up then we have already waited for the storage commit delay so + // wait for either a new mutation version or the budget to be used up. + // Otherwise, don't wait at all. + if (!data->fetchKeysBudgetUsed.get()) { + wait(data->desiredOldestVersion.whenAtLeast(data->storageVersion() + 1) || + data->fetchKeysBudgetUsed.onChange()); + } + + // Yield to TaskPriority::UpdateStorage in case more mutations have arrived but were not processed yet. + // If the fetch keys budget has already been used up, then we likely arrived here without waiting the + // full post storage commit delay, so this will allow the update actor to process some mutations + // before we proceed. wait(delay(0, TaskPriority::UpdateStorage)); state Promise durableInProgress; @@ -9783,6 +9774,17 @@ ACTOR Future updateStorage(StorageServer* data) { break; } + // Allow data fetch to use an additional bytesLeft but don't penalize fetch budget if bytesLeft is negative + if (bytesLeft > 0) { + data->fetchKeysBytesBudget += bytesLeft; + data->fetchKeysBudgetUsed.set(data->fetchKeysBytesBudget <= 0); + + // Dependng on how negative the fetchKeys budget was it could still be used up + if (!data->fetchKeysBudgetUsed.get()) { + wait(durableDelay || data->fetchKeysBudgetUsed.onChange()); + } + } + if (addedRanges) { TraceEvent(SevVerbose, "SSAddKVSRangeMetaData", data->thisServerID) .detail("NewDurableVersion", newOldestVersion) @@ -9882,11 +9884,10 @@ ACTOR Future updateStorage(StorageServer* data) { wait(data->storage.canCommit()); state Future durable = data->storage.commit(); ++data->counters.kvCommits; - state Future durableDelay = Void(); - if (bytesLeft > 0) { - durableDelay = delay(SERVER_KNOBS->STORAGE_COMMIT_INTERVAL, TaskPriority::UpdateStorage); - } + // If the mutation bytes budget was not fully used then wait some time before the next commit + durableDelay = + (bytesLeft > 0) ? delay(SERVER_KNOBS->STORAGE_COMMIT_INTERVAL, TaskPriority::UpdateStorage) : Void(); wait(ioTimeoutError(durable, SERVER_KNOBS->MAX_STORAGE_COMMIT_TIME, "StorageCommit")); data->storageCommitLatencyHistogram->sampleSeconds(now() - beforeStorageCommit); @@ -11711,7 +11712,7 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, // If the storage server dies while something that uses self is still on the stack, // we want that actor to complete before we terminate and that memory goes out of scope - self.ssLock->kill(); + self.ssLock->halt(); state Error err = e; if (storageServerTerminated(self, persistentData, err)) { @@ -11804,7 +11805,7 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, throw internal_error(); } catch (Error& e) { - self.ssLock->kill(); + self.ssLock->halt(); if (self.byteSampleRecovery.isValid()) { self.byteSampleRecovery.cancel(); diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index d845fdea92..2dd6f607fe 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -1388,8 +1388,6 @@ std::map> testSpecGlobalKey [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableHostname", ""); } }, { "disableRemoteKVS", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedRemoteKVS", ""); } }, - { "disableEncryption", - [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedEncryption", ""); } }, { "allowDefaultTenant", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDefaultTenant", ""); } } }; diff --git a/fdbserver/workloads/AuthzSecurity.actor.cpp b/fdbserver/workloads/AuthzSecurity.actor.cpp index 1e6039c98c..1b773a595b 100644 --- a/fdbserver/workloads/AuthzSecurity.actor.cpp +++ b/fdbserver/workloads/AuthzSecurity.actor.cpp @@ -54,14 +54,15 @@ struct AuthzSecurityWorkload : TestWorkload { WipedString signedTokenAnotherTenant; Standalone tLogConfigKey; PerfIntCounter crossTenantGetPositive, crossTenantGetNegative, crossTenantCommitPositive, crossTenantCommitNegative, - publicNonTenantRequestPositive, tLogReadNegative; + publicNonTenantRequestPositive, tLogReadNegative, keyLocationLeakNegative; std::vector(Database cx)>> testFunctions; AuthzSecurityWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), crossTenantGetPositive("CrossTenantGetPositive"), crossTenantGetNegative("CrossTenantGetNegative"), crossTenantCommitPositive("CrossTenantCommitPositive"), crossTenantCommitNegative("CrossTenantCommitNegative"), - publicNonTenantRequestPositive("PublicNonTenantRequestPositive"), tLogReadNegative("TLogReadNegative") { + publicNonTenantRequestPositive("PublicNonTenantRequestPositive"), tLogReadNegative("TLogReadNegative"), + keyLocationLeakNegative("KeyLocationLeakNegative") { testDuration = getOption(options, "testDuration"_sr, 10.0); transactionsPerSecond = getOption(options, "transactionsPerSecond"_sr, 500.0) / clientCount; actorCount = getOption(options, "actorsPerClient"_sr, transactionsPerSecond / 5); @@ -81,6 +82,7 @@ struct AuthzSecurityWorkload : TestWorkload { testFunctions.push_back( [this](Database cx) { return testPublicNonTenantRequestsAllowedWithoutTokens(this, cx); }); testFunctions.push_back([this](Database cx) { return testTLogReadDisallowed(this, cx); }); + testFunctions.push_back([this](Database cx) { return testKeyLocationLeakDisallowed(this, cx); }); } Future setup(Database const& cx) override { @@ -108,7 +110,8 @@ struct AuthzSecurityWorkload : TestWorkload { clients.clear(); return errors == 0 && crossTenantGetPositive.getValue() > 0 && crossTenantGetNegative.getValue() > 0 && crossTenantCommitPositive.getValue() > 0 && crossTenantCommitNegative.getValue() > 0 && - publicNonTenantRequestPositive.getValue() > 0 && tLogReadNegative.getValue() > 0; + publicNonTenantRequestPositive.getValue() > 0 && tLogReadNegative.getValue() > 0 && + keyLocationLeakNegative.getValue() > 0; } void getMetrics(std::vector& m) override { @@ -118,6 +121,7 @@ struct AuthzSecurityWorkload : TestWorkload { m.push_back(crossTenantCommitNegative.getMetric()); m.push_back(publicNonTenantRequestPositive.getMetric()); m.push_back(tLogReadNegative.getMetric()); + m.push_back(keyLocationLeakNegative.getMetric()); } void setAuthToken(Transaction& tr, StringRef token) { @@ -400,6 +404,68 @@ struct AuthzSecurityWorkload : TestWorkload { return Void(); } + ACTOR static Future testKeyLocationLeakDisallowed(AuthzSecurityWorkload* self, Database cx) { + state Key key = self->randomString(); + state Value value = self->randomString(); + state Version v1 = + wait(setAndCommitKeyValueAndGetVersion(self, cx, self->tenant, self->signedToken, key, value)); + state Version v2 = wait(setAndCommitKeyValueAndGetVersion( + self, cx, self->anotherTenant, self->signedTokenAnotherTenant, key, value)); + + { + GetKeyServerLocationsReply rep = + wait(basicLoadBalance(cx->getCommitProxies(UseProvisionalProxies::False), + &CommitProxyInterface::getKeyServersLocations, + GetKeyServerLocationsRequest(SpanContext(), + TenantInfo(self->tenant->id(), self->signedToken), + key, + Optional(), + 100, + false, + v2, + Arena()))); + for (auto const& [range, ssIfaces] : rep.results) { + if (!range.begin.startsWith(self->tenant->prefix())) { + TraceEvent(SevError, "AuthzSecurityKeyRangeLeak") + .detail("TenantId", self->tenant->id()) + .detail("LeakingRangeBegin", range.begin.printable()); + } + if (!range.end.startsWith(self->tenant->prefix())) { + TraceEvent(SevError, "AuthzSecurityKeyRangeLeak") + .detail("TenantId", self->tenant->id()) + .detail("LeakingRangeEnd", range.end.printable()); + } + } + } + { + GetKeyServerLocationsReply rep = wait(basicLoadBalance( + cx->getCommitProxies(UseProvisionalProxies::False), + &CommitProxyInterface::getKeyServersLocations, + GetKeyServerLocationsRequest(SpanContext(), + TenantInfo(self->anotherTenant->id(), self->signedTokenAnotherTenant), + key, + Optional(), + 100, + false, + v2, + Arena()))); + for (auto const& [range, ssIfaces] : rep.results) { + if (!range.begin.startsWith(self->anotherTenant->prefix())) { + TraceEvent(SevError, "AuthzSecurityKeyRangeLeak") + .detail("TenantId", self->anotherTenant->id()) + .detail("LeakingRangeBegin", range.begin.printable()); + } + if (!range.end.startsWith(self->anotherTenant->prefix())) { + TraceEvent(SevError, "AuthzSecurityKeyRangeLeak") + .detail("TenantId", self->anotherTenant->id()) + .detail("LeakingRangeEnd", range.end.printable()); + } + } + } + ++self->keyLocationLeakNegative; + return Void(); + } + ACTOR static Future runTestClient(AuthzSecurityWorkload* self, Database cx) { state double lastTime = now(); state double delay = self->actorCount / self->transactionsPerSecond; diff --git a/fdbserver/workloads/BlobGranuleRangesWorkload.actor.cpp b/fdbserver/workloads/BlobGranuleRangesWorkload.actor.cpp index 4f5be79a1b..637bbe9b6e 100644 --- a/fdbserver/workloads/BlobGranuleRangesWorkload.actor.cpp +++ b/fdbserver/workloads/BlobGranuleRangesWorkload.actor.cpp @@ -553,8 +553,8 @@ struct BlobGranuleRangesWorkload : TestWorkload { ASSERT(!fail6); bool fail7 = wait(cx->blobbifyRange(KeyRangeRef(middleKey, activeRange.end), self->tenant)); - ASSERT(!fail7); + ASSERT(!fail7); bool fail8 = wait(cx->blobbifyRange(KeyRangeRef(middleKey, middleKey2), self->tenant)); ASSERT(!fail8); @@ -681,6 +681,15 @@ struct BlobGranuleRangesWorkload : TestWorkload { return Void(); } + ACTOR Future blobbifyBlockingUnit(Database cx, BlobGranuleRangesWorkload* self, KeyRange range) { + bool setSuccess = wait(cx->blobbifyRangeBlocking(range, self->tenant)); + ASSERT(setSuccess); + bool verifySuccess = wait(self->isRangeActive(cx, range, self->tenant)); + ASSERT(verifySuccess); + + return Void(); + } + enum UnitTestTypes { VERIFY_RANGE_UNIT, VERIFY_RANGE_GAP_UNIT, @@ -688,7 +697,8 @@ struct BlobGranuleRangesWorkload : TestWorkload { BLOBBIFY_IDEMPOTENT, RE_BLOBBIFY, ADJACENT_PURGE, - OP_COUNT = 6 /* keep this last */ + BLOBBIFY_BLOCKING_UNIT, + OP_COUNT = 7 /* keep this last */ }; ACTOR Future blobGranuleRangesUnitTests(Database cx, BlobGranuleRangesWorkload* self) { @@ -732,6 +742,8 @@ struct BlobGranuleRangesWorkload : TestWorkload { wait(self->reBlobbifyUnit(cx, self, range)); } else if (op == ADJACENT_PURGE) { wait(self->adjacentPurge(cx, self, range)); + } else if (op == BLOBBIFY_BLOCKING_UNIT) { + wait(self->blobbifyBlockingUnit(cx, self, range)); } else { ASSERT(false); } diff --git a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp index 5123294217..e5f0c98b42 100644 --- a/fdbserver/workloads/BlobGranuleVerifier.actor.cpp +++ b/fdbserver/workloads/BlobGranuleVerifier.actor.cpp @@ -168,27 +168,13 @@ struct BlobGranuleVerifierWorkload : TestWorkload { } } - // FIXME: run the actual FDBCLI command instead of copy/pasting its implementation // Sets the whole user keyspace to be blobified ACTOR Future setUpBlobRange(Database cx) { - state Reference tr = makeReference(cx); - loop { - try { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - tr->set(blobRangeChangeKey, deterministicRandom()->randomUniqueID().toString()); - wait(krmSetRange(tr, blobRangeKeys.begin, KeyRange(normalKeys), "1"_sr)); - wait(tr->commit()); - if (BGV_DEBUG) { - printf("Successfully set up blob granule range for normalKeys\n"); - } - TraceEvent("BlobGranuleVerifierSetup"); - return Void(); - } catch (Error& e) { - wait(tr->onError(e)); - } - } + bool success = wait(cx->blobbifyRange(normalKeys)); + ASSERT(success); + return Void(); } + void disableFailureInjectionWorkloads(std::set& out) const override { out.emplace("Attrition"); } Future setup(Database const& cx) override { return _setup(cx, this); } diff --git a/fdbserver/workloads/BlobRestoreWorkload.actor.cpp b/fdbserver/workloads/BlobRestoreWorkload.actor.cpp index 3c5081c9b1..3f4b22e326 100644 --- a/fdbserver/workloads/BlobRestoreWorkload.actor.cpp +++ b/fdbserver/workloads/BlobRestoreWorkload.actor.cpp @@ -25,7 +25,6 @@ #include "fdbclient/BackupContainer.h" #include "fdbclient/BackupContainerFileSystem.h" #include "fdbclient/FDBTypes.h" -#include "fdbclient/Knobs.h" #include "fdbclient/SystemData.h" #include "fdbserver/workloads/workloads.actor.h" #include "fdbserver/BlobGranuleServerCommon.actor.h" @@ -73,6 +72,10 @@ struct BlobRestoreWorkload : TestWorkload { if (self->performRestore_) { fmt::print("Perform blob restore\n"); + // disable manifest backup and log truncation + KnobValueRef knobFalse = KnobValueRef::create(bool{ false }); + IKnobCollection::getMutableGlobalKnobCollection().setKnob("blob_manifest_backup", knobFalse); + wait(store(result, self->extraDb_->blobRestore(normalKeys, {}))); state std::vector> futures; @@ -178,8 +181,9 @@ struct BlobRestoreWorkload : TestWorkload { if (src[i].value != dest[i].value) { fmt::print("Value mismatch at {}\n", i); TraceEvent(SevError, "TestFailure") - .detail("Reason", "Key Mismatch") + .detail("Reason", "Value Mismatch") .detail("Index", i) + .detail("Key", src[i].key.printable()) .detail("SrcValue", src[i].value.printable()) .detail("DestValue", dest[i].value.printable()); return false; diff --git a/fdbserver/workloads/EncryptionOps.actor.cpp b/fdbserver/workloads/EncryptionOps.actor.cpp index c08593a1ce..68a35111a3 100644 --- a/fdbserver/workloads/EncryptionOps.actor.cpp +++ b/fdbserver/workloads/EncryptionOps.actor.cpp @@ -120,7 +120,6 @@ struct EncryptionOpsWorkload : TestWorkload { std::unique_ptr buff; int enableTTLTest; - Arena arena; std::unique_ptr metrics; EncryptCipherDomainId minDomainId; @@ -279,7 +278,8 @@ struct EncryptionOpsWorkload : TestWorkload { int len, const EncryptAuthTokenMode authMode, const EncryptAuthTokenAlgo authAlgo, - BlobCipherEncryptHeader* header) { + BlobCipherEncryptHeader* header, + Arena& arena) { uint8_t iv[AES_256_IV_LENGTH]; deterministicRandom()->randomBytes(&iv[0], AES_256_IV_LENGTH); EncryptBlobCipherAes265Ctr encryptor( @@ -304,7 +304,8 @@ struct EncryptionOpsWorkload : TestWorkload { int len, const EncryptAuthTokenMode authMode, const EncryptAuthTokenAlgo authAlgo, - BlobCipherEncryptHeaderRef* headerRef) { + BlobCipherEncryptHeaderRef* headerRef, + Arena& arena) { uint8_t iv[AES_256_IV_LENGTH]; deterministicRandom()->randomBytes(&iv[0], AES_256_IV_LENGTH); EncryptBlobCipherAes265Ctr encryptor( @@ -345,7 +346,8 @@ struct EncryptionOpsWorkload : TestWorkload { int len, const BlobCipherEncryptHeader& header, uint8_t* originalPayload, - Reference orgCipherKey) { + Reference orgCipherKey, + Arena& arena) { ASSERT_EQ(header.flags.headerVersion, EncryptBlobCipherAes265Ctr::ENCRYPT_HEADER_VERSION); ASSERT_EQ(header.flags.encryptMode, ENCRYPT_CIPHER_MODE_AES_256_CTR); @@ -379,7 +381,8 @@ struct EncryptionOpsWorkload : TestWorkload { int len, const Standalone& headerStr, uint8_t* originalPayload, - Reference orgCipherKey) { + Reference orgCipherKey, + Arena& arena) { BlobCipherEncryptHeaderRef headerRef = BlobCipherEncryptHeaderRef::fromStringRef(headerStr); ASSERT_EQ(headerRef.flagsVersion(), CLIENT_KNOBS->ENCRYPT_HEADER_FLAGS_VERSION); @@ -436,6 +439,7 @@ struct EncryptionOpsWorkload : TestWorkload { setupCipherEssentials(); for (int i = 0; i < numIterations; i++) { + Arena tmpArena; bool updateBaseCipher = deterministicRandom()->randomInt(1, 100) < 5; // Step-1: Encryption key derivation, caching the cipher for later use @@ -482,23 +486,23 @@ struct EncryptionOpsWorkload : TestWorkload { try { BlobCipherEncryptHeader header; - Reference encrypted = - doEncryption(cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &header); + Reference encrypted = doEncryption( + cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &header, tmpArena); // Decrypt the payload - parses the BlobCipherEncryptHeader, fetch corresponding cipherKey and // decrypt - doDecryption(encrypted, dataLen, header, buff.get(), cipherKey); + doDecryption(encrypted, dataLen, header, buff.get(), cipherKey, tmpArena); if (CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION) { BlobCipherEncryptHeaderRef headerRef; - StringRef encrypted = - doEncryption(cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &headerRef); + StringRef encrypted = doEncryption( + cipherKey, headerCipherKey, buff.get(), dataLen, authMode, authAlgo, &headerRef, tmpArena); // simulate 'header' on-disk read, serialize buffer and deserialize on decryption Standalone serHeader = BlobCipherEncryptHeaderRef::toStringRef(headerRef); // Decrypt the payload - parses the BlobCipherEncryptHeader, fetch corresponding cipherKey and // decrypt - doDecryption(encrypted, dataLen, serHeader, buff.get(), cipherKey); + doDecryption(encrypted, dataLen, serHeader, buff.get(), cipherKey, tmpArena); } } catch (Error& e) { TraceEvent("Failed") diff --git a/fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp b/fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp index 26b08a77b7..4c8d0ec9bf 100644 --- a/fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp +++ b/fdbserver/workloads/MetaclusterManagementWorkload.actor.cpp @@ -299,22 +299,18 @@ struct MetaclusterManagementWorkload : TestWorkload { } catch (Error& e) { if (e.code() == error_code_conflicting_restore) { ASSERT(retried); - CODE_PROBE(true, "MetaclusterRestore: 2 restores on the same cluster simultaneously"); + CODE_PROBE(true, "MetaclusterManagementWorkload: timed out restore conflicts with retried restore"); continue; } throw; } } - - ASSERT(dataDb->registered); - if (!dryRun) { - dataDb->detached = false; - } } catch (Error& e) { if (e.code() == error_code_cluster_not_found) { ASSERT(!dataDb->registered); return Void(); } + TraceEvent(SevError, "RestoreClusterFailure").error(e).detail("ClusterName", clusterName); ASSERT(false); } diff --git a/fdbserver/workloads/SaveAndKill.actor.cpp b/fdbserver/workloads/SaveAndKill.actor.cpp index ae2bb54362..681fe413cf 100644 --- a/fdbserver/workloads/SaveAndKill.actor.cpp +++ b/fdbserver/workloads/SaveAndKill.actor.cpp @@ -79,13 +79,8 @@ struct SaveAndKillWorkload : TestWorkload { if (cx->defaultTenant.present()) { ini.SetValue("META", "defaultTenant", cx->defaultTenant.get().toString().c_str()); } - - ini.SetBoolValue("META", "enableEncryption", SERVER_KNOBS->ENABLE_ENCRYPTION); - ini.SetBoolValue("META", "enableTLogEncryption", SERVER_KNOBS->ENABLE_TLOG_ENCRYPTION); - ini.SetBoolValue("META", "enableStorageServerEncryption", SERVER_KNOBS->ENABLE_STORAGE_SERVER_ENCRYPTION); - ini.SetBoolValue("META", "enableBlobGranuleEncryption", SERVER_KNOBS->ENABLE_BLOB_GRANULE_ENCRYPTION); ini.SetBoolValue("META", "enableShardEncodeLocationMetadata", SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA); - + ini.SetBoolValue("META", "enableConfigurableEncryption", CLIENT_KNOBS->ENABLE_CONFIGURABLE_ENCRYPTION); ini.SetBoolValue("META", "encryptHeaderAuthTokenEnabled", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ENABLED); ini.SetLongValue("META", "encryptHeaderAuthTokenAlgo", FLOW_KNOBS->ENCRYPT_HEADER_AUTH_TOKEN_ALGO); diff --git a/flow/Arena.cpp b/flow/Arena.cpp index b23ee5b50a..61d1a51015 100644 --- a/flow/Arena.cpp +++ b/flow/Arena.cpp @@ -22,6 +22,8 @@ #include "flow/UnitTest.h" +#include "flow/config.h" + // We don't align memory properly, and we need to tell lsan about that. extern "C" const char* __lsan_default_options(void) { return "use_unaligned=1"; @@ -998,7 +1000,7 @@ TEST_CASE("/flow/Arena/OptionalMap") { } TEST_CASE("/flow/Arena/Secure") { -#ifndef ADDRESS_SANITIZER +#ifndef USE_SANITIZER // Note: Assumptions underlying this unit test are speculative. // Disable for a build configuration or entirely if deemed flaky. // As of writing, below equivalency of (buf == newBuf) holds except for ASAN builds. @@ -1051,6 +1053,6 @@ TEST_CASE("/flow/Arena/Secure") { } } fmt::print("Total iterations: {}, # of times check passed: {}\n", totalIters, samePtrCount); -#endif // ADDRESS_SANITIZER +#endif // USE_SANITIZER return Void(); } diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp index 39c21e2fa4..fcced5a76b 100644 --- a/flow/Platform.actor.cpp +++ b/flow/Platform.actor.cpp @@ -651,7 +651,11 @@ void getDiskBytes(std::string const& directory, int64_t& free, int64_t& total) { #endif free = std::min((uint64_t)std::numeric_limits::max(), buf.f_bavail * blockSize); - total = std::min((uint64_t)std::numeric_limits::max(), buf.f_blocks * blockSize); + + // f_blocks is the total fs space but (f_bfree - f_bavail) is space only available to privileged users + // so that amount will be subtracted from the reported total since FDB can't use it. + total = std::min((uint64_t)std::numeric_limits::max(), + (buf.f_blocks - (buf.f_bfree - buf.f_bavail)) * blockSize); #elif defined(_WIN32) std::string fullPath = abspath(directory); diff --git a/flow/include/flow/PriorityMultiLock.actor.h b/flow/include/flow/PriorityMultiLock.actor.h index ec1fd98cd9..e525b58066 100644 --- a/flow/include/flow/PriorityMultiLock.actor.h +++ b/flow/include/flow/PriorityMultiLock.actor.h @@ -88,7 +88,7 @@ public: : PriorityMultiLock(concurrency, parseStringToVector(weights, ',')) {} PriorityMultiLock(int concurrency, std::vector weightsByPriority) - : concurrency(concurrency), available(concurrency), waiting(0), totalPendingWeights(0) { + : concurrency(concurrency), available(concurrency), waiting(0), totalPendingWeights(0), killed(false) { priorities.resize(weightsByPriority.size()); for (int i = 0; i < priorities.size(); ++i) { @@ -102,6 +102,9 @@ public: ~PriorityMultiLock() { kill(); } Future lock(int priority = 0) { + if (killed) + throw broken_promise(); + Priority& p = priorities[priority]; Queue& q = p.queue; @@ -135,17 +138,33 @@ public: return w.lockPromise.getFuture(); } - void kill() { - pml_debug_printf("kill %s\n", toString().c_str()); + // Halt stops the PML from handing out any new locks but leaves waiters and runners alone. + // Existing and new waiters will not see an error, they will just never get a lock. + // Can be safely called multiple times. + void halt() { + pml_debug_printf("halt %s\n", toString().c_str()); brokenOnDestruct.reset(); - // handleRelease will not free up any execution slots when it ends via cancel - fRunner.cancel(); - available = 0; + if (fRunner.isValid()) { + fRunner.cancel(); + // Adjust available and concurrency so that if all runners finish the available + available -= concurrency; + concurrency = 0; + } waitingPriorities.clear(); - for (auto& p : priorities) { - p.queue.clear(); + } + + // Halt, then make existing and new waiters get a broken_promise error. + // Can be safely called multiple times. + void kill() { + if (!killed) { + // Killed must be set first because waiters which ignore exceptions could call wait again immediately. + killed = true; + halt(); + for (auto& p : priorities) { + p.queue.clear(); + } } } @@ -231,6 +250,7 @@ private: Future fRunner; AsyncTrigger wakeRunner; Promise brokenOnDestruct; + bool killed; ACTOR static void handleRelease(Reference self, Priority* priority, Future holder) { pml_debug_printf("%f handleRelease self=%p start\n", now(), self.getPtr()); diff --git a/tests/BlobGranuleFileUnit.toml b/tests/BlobGranuleFileUnit.toml index 52f43fb3f9..df5b28f1af 100644 --- a/tests/BlobGranuleFileUnit.toml +++ b/tests/BlobGranuleFileUnit.toml @@ -2,8 +2,6 @@ encryptModes = ['domain_aware', 'cluster_aware'] [[knobs]] -enable_encryption = true -enable_blob_file_encryption = true enable_blob_file_compression = true [[test]] diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b24d9da57f..c1be591aa2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -333,15 +333,15 @@ if(WITH_PYTHON) add_fdb_test( TEST_FILES restarting/from_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-1.toml restarting/from_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-1.toml + restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-2.toml) add_fdb_test( TEST_FILES restarting/from_7.2.0_until_7.3.0/VersionVectorDisableRestart-1.toml restarting/from_7.2.0_until_7.3.0/VersionVectorDisableRestart-2.toml) add_fdb_test( TEST_FILES restarting/from_7.2.0_until_7.3.0/VersionVectorEnableRestart-1.toml restarting/from_7.2.0_until_7.3.0/VersionVectorEnableRestart-2.toml) - add_fdb_test( - TEST_FILES restarting/from_7.2.0/DrUpgradeRestart-1.toml - restarting/from_7.2.0/DrUpgradeRestart-2.toml) add_fdb_test( TEST_FILES restarting/from_7.2.4_until_7.3.0/UpgradeAndBackupRestore-1.toml restarting/from_7.2.4_until_7.3.0/UpgradeAndBackupRestore-2.toml) @@ -351,6 +351,9 @@ if(WITH_PYTHON) add_fdb_test( TEST_FILES restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-1.toml restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.0/DrUpgradeRestart-1.toml + restarting/from_7.3.0/DrUpgradeRestart-2.toml) add_fdb_test( TEST_FILES restarting/from_7.3.0/UpgradeAndBackupRestore-1.toml restarting/from_7.3.0/UpgradeAndBackupRestore-2.toml) diff --git a/tests/TestRunner/local_cluster.py b/tests/TestRunner/local_cluster.py index cadfaf138c..7e4e286361 100644 --- a/tests/TestRunner/local_cluster.py +++ b/tests/TestRunner/local_cluster.py @@ -114,6 +114,7 @@ logdir = {logdir} {bg_knob_line} {encrypt_knob_line1} {encrypt_knob_line2} +{encrypt_knob_line3} {tls_config} {authz_public_key_config} {custom_config} @@ -256,13 +257,14 @@ logdir = {logdir} bg_knob_line = "" encrypt_knob_line1 = "" encrypt_knob_line2 = "" + encrypt_knob_line3 = "" if self.use_legacy_conf_syntax: conf_template = conf_template.replace("-", "_") if self.blob_granules_enabled: bg_knob_line = "knob_bg_url=file://" + str(self.data) + "/fdbblob/" if self.enable_encryption_at_rest: - encrypt_knob_line1 = "knob_enable_encryption=true" encrypt_knob_line2 = "knob_kms_connector_type=FDBPerfKmsConnector" + encrypt_knob_line3 = "knob_enable_configurable_encryption=true" f.write( conf_template.format( etcdir=self.etc, @@ -273,6 +275,7 @@ logdir = {logdir} bg_knob_line=bg_knob_line, encrypt_knob_line1=encrypt_knob_line1, encrypt_knob_line2=encrypt_knob_line2, + encrypt_knob_line3=encrypt_knob_line3, tls_config=self.tls_conf_string(), authz_public_key_config=self.authz_public_key_conf_string(), optional_tls=":tls" if self.tls_config is not None else "", diff --git a/tests/fast/BackupCorrectnessWithEKPKeyFetchFailures.toml b/tests/fast/BackupCorrectnessWithEKPKeyFetchFailures.toml index da8c6457eb..f161d72b6c 100644 --- a/tests/fast/BackupCorrectnessWithEKPKeyFetchFailures.toml +++ b/tests/fast/BackupCorrectnessWithEKPKeyFetchFailures.toml @@ -4,9 +4,6 @@ tenantModes = ['required'] allowCreatingTenants = false encryptModes = ['domain_aware'] -[[knobs]] -enable_encryption = true - [[test]] testTitle = 'BackupAndRestoreWithEKPKeyFetchFailures' clearAfterTest = false diff --git a/tests/fast/BackupCorrectnessWithTenantDeletion.toml b/tests/fast/BackupCorrectnessWithTenantDeletion.toml index ce2356cb6d..c7708c29ae 100644 --- a/tests/fast/BackupCorrectnessWithTenantDeletion.toml +++ b/tests/fast/BackupCorrectnessWithTenantDeletion.toml @@ -3,6 +3,9 @@ allowDefaultTenant = false tenantModes = ['required'] allowCreatingTenants = false +[[knobs]] +simulation_enable_snapshot_encryption_checks = false + [[test]] testTitle = 'BackupAndRestoreWithTenantDeletion' clearAfterTest = false diff --git a/tests/fast/EncryptKeyProxyTest.toml b/tests/fast/EncryptKeyProxyTest.toml index 6cd3a6f255..82e053763e 100644 --- a/tests/fast/EncryptKeyProxyTest.toml +++ b/tests/fast/EncryptKeyProxyTest.toml @@ -2,9 +2,6 @@ testClass = "Encryption" encryptModes = ['domain_aware', 'cluster_aware'] -[[knobs]] -enable_encryption = true - [[test]] testTitle = 'EncryptKeyProxy' diff --git a/tests/fast/EncryptedBackupCorrectness.toml b/tests/fast/EncryptedBackupCorrectness.toml index daa69ff699..7682dfd36a 100644 --- a/tests/fast/EncryptedBackupCorrectness.toml +++ b/tests/fast/EncryptedBackupCorrectness.toml @@ -5,10 +5,6 @@ tenantModes = ['required'] encryptModes = ['domain_aware'] [[knobs]] -enable_encryption = true -enable_tlog_encryption = true -enable_storage_server_encryption = false -enable_blob_granule_encryption = true max_write_transaction_life_versions = 5000000 [[test]] diff --git a/tests/fast/EncryptionOps.toml b/tests/fast/EncryptionOps.toml index 693e15f319..439244cb08 100644 --- a/tests/fast/EncryptionOps.toml +++ b/tests/fast/EncryptionOps.toml @@ -1,7 +1,7 @@ [configuration] buggify = false testClass = "Encryption" -disableEncryption = true +encryptModes = ['disabled'] [[knobs]] enable_configurable_encryption = true diff --git a/tests/fast/IncrementalBackupWithEKPKeyFetchFailures.toml b/tests/fast/IncrementalBackupWithEKPKeyFetchFailures.toml index b3055c1f4b..f40d7565d8 100644 --- a/tests/fast/IncrementalBackupWithEKPKeyFetchFailures.toml +++ b/tests/fast/IncrementalBackupWithEKPKeyFetchFailures.toml @@ -4,9 +4,6 @@ tenantModes = ['required'] allowCreatingTenants = false encryptModes = ['domain_aware'] -[[knobs]] -enable_encryption = true - [[test]] testTitle = 'SubmitBackup' simBackupAgents = 'BackupToFile' diff --git a/tests/fast/IncrementalBackupWithTenantDeletion.toml b/tests/fast/IncrementalBackupWithTenantDeletion.toml index 702d0a0a43..a304bf8efa 100644 --- a/tests/fast/IncrementalBackupWithTenantDeletion.toml +++ b/tests/fast/IncrementalBackupWithTenantDeletion.toml @@ -3,6 +3,9 @@ allowDefaultTenant = false tenantModes = ['required'] allowCreatingTenants = false +[[knobs]] +simulation_enable_snapshot_encryption_checks = false + [[test]] testTitle = 'SubmitBackup' simBackupAgents = 'BackupToFile' diff --git a/tests/restarting/from_7.2.0/DrUpgradeRestart-1.toml b/tests/restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-1.toml similarity index 100% rename from tests/restarting/from_7.2.0/DrUpgradeRestart-1.toml rename to tests/restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-1.toml diff --git a/tests/restarting/from_7.2.0/DrUpgradeRestart-2.toml b/tests/restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-2.toml similarity index 100% rename from tests/restarting/from_7.2.0/DrUpgradeRestart-2.toml rename to tests/restarting/from_7.2.0_until_7.3.0/DrUpgradeRestart-2.toml diff --git a/tests/restarting/from_7.3.0/DrUpgradeRestart-1.toml b/tests/restarting/from_7.3.0/DrUpgradeRestart-1.toml new file mode 100644 index 0000000000..5d1e6389cd --- /dev/null +++ b/tests/restarting/from_7.3.0/DrUpgradeRestart-1.toml @@ -0,0 +1,28 @@ +[configuration] +extraDatabaseMode = "Local" +# In 7.2, DR is not supported in required tenant mode +allowDefaultTenant = false +encryptModes = ['disabled'] + +[[test]] +testTitle = "DrUpgrade" +clearAfterTest = false +simBackupAgents = "BackupToDB" + + [[test.workload]] + testName = "Cycle" + nodeCount = 30000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + + [[test.workload]] + testName = "BackupToDBUpgrade" + backupAfter = 10.0 + stopDifferentialAfter = 50.0 + backupRangesCount = -1 + + [[test.workload]] + testName = "SaveAndKill" + restartInfoLocation = "simfdb/restartInfo.ini" + testDuration = 40.0 diff --git a/tests/restarting/from_7.3.0/DrUpgradeRestart-2.toml b/tests/restarting/from_7.3.0/DrUpgradeRestart-2.toml new file mode 100644 index 0000000000..9a883ee2f4 --- /dev/null +++ b/tests/restarting/from_7.3.0/DrUpgradeRestart-2.toml @@ -0,0 +1,22 @@ +[configuration] +extraDatabaseMode = "Local" + +[[test]] +testTitle = "DrUpgrade" +runSetup = false +clearAfterTest = false +simBackupAgents = "BackupToDB" +waitForQuiescenceBegin = false + + [[test.workload]] + testName = "Cycle" + nodeCount = 30000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + + [[test.workload]] + testName = "BackupToDBUpgrade" + backupAfter = 10.0 + backupRangesCount = -1 + stopDifferentialAfter = 70.0 diff --git a/tests/restarting/to_7.1.0_until_7.2.0/ConfigureStorageMigrationTestRestart-1.toml b/tests/restarting/to_7.1.0_until_7.2.0/ConfigureStorageMigrationTestRestart-1.toml index 50164841d1..8f37f2d6bf 100644 --- a/tests/restarting/to_7.1.0_until_7.2.0/ConfigureStorageMigrationTestRestart-1.toml +++ b/tests/restarting/to_7.1.0_until_7.2.0/ConfigureStorageMigrationTestRestart-1.toml @@ -2,9 +2,9 @@ extraMachineCountDC = 2 maxTLogVersion=6 disableHostname=true -disableEncryption=true storageEngineExcludeTypes=[3, 4, 5] tenantModes=['disabled'] +encryptModes=['disabled'] [[knobs]] # This can be removed once the lower bound of this downgrade test is a version that understands the new protocol diff --git a/tests/restarting/to_7.1.0_until_7.2.0/CycleTestRestart-1.toml b/tests/restarting/to_7.1.0_until_7.2.0/CycleTestRestart-1.toml index 54f8a7994a..be2a3eb2d4 100644 --- a/tests/restarting/to_7.1.0_until_7.2.0/CycleTestRestart-1.toml +++ b/tests/restarting/to_7.1.0_until_7.2.0/CycleTestRestart-1.toml @@ -3,7 +3,7 @@ storageEngineExcludeTypes = [3, 5] maxTLogVersion = 6 disableTss = true disableHostname = true -disableEncryption = true +encryptModes = ['disabled'] tenantModes=['disabled'] [[knobs]] diff --git a/tests/restarting/to_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-1.toml b/tests/restarting/to_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-1.toml index 9e315bd45f..6525d53ff9 100644 --- a/tests/restarting/to_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-1.toml +++ b/tests/restarting/to_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-1.toml @@ -2,7 +2,7 @@ extraMachineCountDC = 2 maxTLogVersion=6 disableHostname=true -disableEncryption=true +encryptModes = ['disabled'] storageEngineExcludeTypes=[4] tenantModes=['disabled'] diff --git a/tests/restarting/to_7.2.0_until_7.3.0/CycleTestRestart-1.toml b/tests/restarting/to_7.2.0_until_7.3.0/CycleTestRestart-1.toml index 2d5877824c..bc74a69624 100644 --- a/tests/restarting/to_7.2.0_until_7.3.0/CycleTestRestart-1.toml +++ b/tests/restarting/to_7.2.0_until_7.3.0/CycleTestRestart-1.toml @@ -2,7 +2,7 @@ maxTLogVersion = 6 disableTss = true disableHostname = true -disableEncryption = true +encryptModes = ['disabled'] tenantModes=['disabled'] [[test]] diff --git a/tests/slow/BlobGranuleCorrectness.toml b/tests/slow/BlobGranuleCorrectness.toml index bc44be7fc0..95573249fd 100644 --- a/tests/slow/BlobGranuleCorrectness.toml +++ b/tests/slow/BlobGranuleCorrectness.toml @@ -9,7 +9,6 @@ encryptModes = ['domain_aware', 'cluster_aware'] [[knobs]] bg_metadata_source = "tenant" bg_key_tuple_truncate_offset = 1 -enable_encryption = true enable_configurable_encryption = true [[test]] diff --git a/tests/slow/BlobGranuleCorrectnessClean.toml b/tests/slow/BlobGranuleCorrectnessClean.toml index cfdab46df5..76e2c0390b 100644 --- a/tests/slow/BlobGranuleCorrectnessClean.toml +++ b/tests/slow/BlobGranuleCorrectnessClean.toml @@ -6,7 +6,6 @@ encryptModes = ['domain_aware', 'cluster_aware'] [[knobs]] bg_metadata_source = "tenant" -enable_encryption = true enable_configurable_encryption = true [[test]]