diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py index 2cff6c3cbc..fbae72d36c 100644 --- a/bindings/bindingtester/known_testers.py +++ b/bindings/bindingtester/known_testers.py @@ -58,8 +58,8 @@ _java_cmd = 'java -ea -cp %s:%s com.apple.foundationdb.test.' % ( # We could set min_api_version lower on some of these if the testers were updated to support them testers = { - 'python': Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES), - 'python3': Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES), + 'python': Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True), + 'python3': Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True), 'ruby': Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 2040, 23, MAX_API_VERSION), 'java': Tester('java', _java_cmd + 'StackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), 'java_async': Tester('java', _java_cmd + 'AsyncStackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), diff --git a/bindings/python/fdb/__init__.py b/bindings/python/fdb/__init__.py index 0054e72808..413c81249a 100644 --- a/bindings/python/fdb/__init__.py +++ b/bindings/python/fdb/__init__.py @@ -88,6 +88,7 @@ def api_version(ver): 'predicates', 'Future', 'Database', + 'Tenant', 'Transaction', 'KeyValue', 'KeySelector', diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index 837d3937c6..023e85ae95 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -34,6 +34,7 @@ import traceback import fdb from fdb import six +from fdb.tuple import pack, unpack _network_thread = None _network_thread_reentrant_lock = threading.RLock() @@ -198,9 +199,10 @@ def transactional(*tr_args, **tr_kwargs): one of two actions, depending on the type of the parameter passed to the function at call time. - If given a Database, a Transaction will be created and passed into - the wrapped code in place of the Database. After the function is - complete, the newly created transaction will be committed. + If given a Database or Tenant, a Transaction will be created and + passed into the wrapped code in place of the Database or Tenant. + After the function is complete, the newly created transaction + will be committed. It is important to note that the wrapped method may be called multiple times in the event of a commit failure, until the commit @@ -943,128 +945,114 @@ class FormerFuture(_FDBBase): except: pass - -class Database(_FDBBase): - def __init__(self, dpointer): - self.dpointer = dpointer - self.options = _DatabaseOptions(self) - - def __del__(self): - # print('Destroying database 0x%x' % self.dpointer) - self.capi.fdb_database_destroy(self.dpointer) - +class _TransactionCreator(_FDBBase): def get(self, key): - return Database.__database_getitem(self, key) + return _TransactionCreator.__creator_getitem(self, key) def __getitem__(self, key): if isinstance(key, slice): return self.get_range(key.start, key.stop, reverse=(key.step == -1)) - return Database.__database_getitem(self, key) + return _TransactionCreator.__creator_getitem(self, key) def get_key(self, key_selector): - return Database.__database_get_key(self, key_selector) + return _TransactionCreator.__creator_get_key(self, key_selector) def get_range(self, begin, end, limit=0, reverse=False, streaming_mode=StreamingMode.want_all): - return Database.__database_get_range(self, begin, end, limit, reverse, streaming_mode) + return _TransactionCreator.__creator_get_range(self, begin, end, limit, reverse, streaming_mode) def get_range_startswith(self, prefix, *args, **kwargs): - return Database.__database_get_range_startswith(self, prefix, *args, **kwargs) + return _TransactionCreator.__creator_get_range_startswith(self, prefix, *args, **kwargs) def set(self, key, value): - Database.__database_setitem(self, key, value) + _TransactionCreator.__creator_setitem(self, key, value) def __setitem__(self, key, value): - Database.__database_setitem(self, key, value) + _TransactionCreator.__creator_setitem(self, key, value) def clear(self, key): - Database.__database_delitem(self, key) + _TransactionCreator.__creator_delitem(self, key) def clear_range(self, begin, end): - Database.__database_delitem(self, slice(begin, end)) + _TransactionCreator.__creator_delitem(self, slice(begin, end)) def __delitem__(self, key_or_slice): - Database.__database_delitem(self, key_or_slice) + _TransactionCreator.__creator_delitem(self, key_or_slice) def clear_range_startswith(self, prefix): - Database.__database_clear_range_startswith(self, prefix) + _TransactionCreator.__creator_clear_range_startswith(self, prefix) def get_and_watch(self, key): - return Database.__database_get_and_watch(self, key) + return _TransactionCreator.__creator_get_and_watch(self, key) def set_and_watch(self, key, value): - return Database.__database_set_and_watch(self, key, value) + return _TransactionCreator.__creator_set_and_watch(self, key, value) def clear_and_watch(self, key): - return Database.__database_clear_and_watch(self, key) + return _TransactionCreator.__creator_clear_and_watch(self, key) def create_transaction(self): - pointer = ctypes.c_void_p() - self.capi.fdb_database_create_transaction(self.dpointer, ctypes.byref(pointer)) - return Transaction(pointer.value, self) - - def _set_option(self, option, param, length): - self.capi.fdb_database_set_option(self.dpointer, option, param, length) + pass def _atomic_operation(self, opcode, key, param): - Database.__database_atomic_operation(self, opcode, key, param) + _TransactionCreator.__creator_atomic_operation(self, opcode, key, param) #### Transaction implementations #### @staticmethod @transactional - def __database_getitem(tr, key): + def __creator_getitem(tr, key): return tr[key].value @staticmethod @transactional - def __database_get_key(tr, key_selector): + def __creator_get_key(tr, key_selector): return tr.get_key(key_selector).value @staticmethod @transactional - def __database_get_range(tr, begin, end, limit, reverse, streaming_mode): + def __creator_get_range(tr, begin, end, limit, reverse, streaming_mode): return tr.get_range(begin, end, limit, reverse, streaming_mode).to_list() @staticmethod @transactional - def __database_get_range_startswith(tr, prefix, *args, **kwargs): + def __creator_get_range_startswith(tr, prefix, *args, **kwargs): return tr.get_range_startswith(prefix, *args, **kwargs).to_list() @staticmethod @transactional - def __database_setitem(tr, key, value): + def __creator_setitem(tr, key, value): tr[key] = value @staticmethod @transactional - def __database_clear_range_startswith(tr, prefix): + def __creator_clear_range_startswith(tr, prefix): tr.clear_range_startswith(prefix) @staticmethod @transactional - def __database_get_and_watch(tr, key): + def __creator_get_and_watch(tr, key): v = tr.get(key) return v, tr.watch(key) @staticmethod @transactional - def __database_set_and_watch(tr, key, value): + def __creator_set_and_watch(tr, key, value): tr.set(key, value) return tr.watch(key) @staticmethod @transactional - def __database_clear_and_watch(tr, key): + def __creator_clear_and_watch(tr, key): del tr[key] return tr.watch(key) @staticmethod @transactional - def __database_delitem(tr, key_or_slice): + def __creator_delitem(tr, key_or_slice): del tr[key_or_slice] @staticmethod @transactional - def __database_atomic_operation(tr, opcode, key, param): + def __creator_atomic_operation(tr, opcode, key, param): tr._atomic_operation(opcode, key, param) # Asynchronous transactions @@ -1074,11 +1062,11 @@ class Database(_FDBBase): From = asyncio.From coroutine = asyncio.coroutine - class Database: + class TransactionCreator: @staticmethod @transactional @coroutine - def __database_getitem(tr, key): + def __creator_getitem(tr, key): # raise Return(( yield From( tr[key] ) )) raise Return(tr[key]) yield None @@ -1086,26 +1074,26 @@ class Database(_FDBBase): @staticmethod @transactional @coroutine - def __database_get_key(tr, key_selector): + def __creator_get_key(tr, key_selector): raise Return(tr.get_key(key_selector)) yield None @staticmethod @transactional @coroutine - def __database_get_range(tr, begin, end, limit, reverse, streaming_mode): + def __creator_get_range(tr, begin, end, limit, reverse, streaming_mode): raise Return((yield From(tr.get_range(begin, end, limit, reverse, streaming_mode).to_list()))) @staticmethod @transactional @coroutine - def __database_get_range_startswith(tr, prefix, *args, **kwargs): + def __creator_get_range_startswith(tr, prefix, *args, **kwargs): raise Return((yield From(tr.get_range_startswith(prefix, *args, **kwargs).to_list()))) @staticmethod @transactional @coroutine - def __database_setitem(tr, key, value): + def __creator_setitem(tr, key, value): tr[key] = value raise Return() yield None @@ -1113,7 +1101,7 @@ class Database(_FDBBase): @staticmethod @transactional @coroutine - def __database_clear_range_startswith(tr, prefix): + def __creator_clear_range_startswith(tr, prefix): tr.clear_range_startswith(prefix) raise Return() yield None @@ -1121,7 +1109,7 @@ class Database(_FDBBase): @staticmethod @transactional @coroutine - def __database_get_and_watch(tr, key): + def __creator_get_and_watch(tr, key): v = tr.get(key) raise Return(v, tr.watch(key)) yield None @@ -1129,7 +1117,7 @@ class Database(_FDBBase): @staticmethod @transactional @coroutine - def __database_set_and_watch(tr, key, value): + def __creator_set_and_watch(tr, key, value): tr.set(key, value) raise Return(tr.watch(key)) yield None @@ -1137,7 +1125,7 @@ class Database(_FDBBase): @staticmethod @transactional @coroutine - def __database_clear_and_watch(tr, key): + def __creator_clear_and_watch(tr, key): del tr[key] raise Return(tr.watch(key)) yield None @@ -1145,7 +1133,7 @@ class Database(_FDBBase): @staticmethod @transactional @coroutine - def __database_delitem(tr, key_or_slice): + def __creator_delitem(tr, key_or_slice): del tr[key_or_slice] raise Return() yield None @@ -1153,11 +1141,101 @@ class Database(_FDBBase): @staticmethod @transactional @coroutine - def __database_atomic_operation(tr, opcode, key, param): + def __creator_atomic_operation(tr, opcode, key, param): tr._atomic_operation(opcode, key, param) raise Return() yield None - return Database + return TransactionCreator + +def process_tenant_name(name): + if isinstance(name, tuple): + return pack(name) + elif isinstance(name, bytes): + return name + else: + raise TypeError('Tenant name must be of type ' + bytes.__name__ + ' or of type ' + tuple.__name__) + +class Database(_TransactionCreator): + def __init__(self, dpointer): + self.dpointer = dpointer + self.options = _DatabaseOptions(self) + + def __del__(self): + # print('Destroying database 0x%x' % self.dpointer) + self.capi.fdb_database_destroy(self.dpointer) + + def _set_option(self, option, param, length): + self.capi.fdb_database_set_option(self.dpointer, option, param, length) + + def open_tenant(self, name): + tname = process_tenant_name(name) + pointer = ctypes.c_void_p() + self.capi.fdb_database_open_tenant(self.dpointer, tname, len(tname), ctypes.byref(pointer)) + return Tenant(pointer.value) + + def create_transaction(self): + pointer = ctypes.c_void_p() + self.capi.fdb_database_create_transaction(self.dpointer, ctypes.byref(pointer)) + return Transaction(pointer.value, self) + + def allocate_tenant(self, name): + Database.__database_allocate_tenant(self, process_tenant_name(name), []) + + def delete_tenant(self, name): + Database.__database_delete_tenant(self, process_tenant_name(name), []) + + # Attempt to allocate a tenant in the cluster. If the tenant already exists, + # this function will return a tenant_already_exists error. If the tenant is created + # concurrently, then this function may return success even if another caller creates + # it. + # + # The existence_check_marker is expected to be an empty list. This function will + # modify the list after completing the existence check to avoid checking for existence + # on retries. This allows the operation to be idempotent. + @staticmethod + @transactional + def __database_allocate_tenant(tr, name, existence_check_marker): + tr.options.set_special_key_space_enable_writes() + key = b'\xff\xff/management/tenant_map/%s' % name + if not existence_check_marker: + existing_tenant = tr[key].wait() + existence_check_marker.append(None) + if existing_tenant != None: + raise fdb.FDBError(2132) # tenant_already_exists + tr[key] = b'' + + # Attempt to remove a tenant in the cluster. If the tenant doesn't exist, this + # function will return a tenant_not_found error. If the tenant is deleted + # concurrently, then this function may return success even if another caller deletes + # it. + # + # The existence_check_marker is expected to be an empty list. This function will + # modify the list after completing the existence check to avoid checking for existence + # on retries. This allows the operation to be idempotent. + @staticmethod + @transactional + def __database_delete_tenant(tr, name, existence_check_marker): + tr.options.set_special_key_space_enable_writes() + key = b'\xff\xff/management/tenant_map/%s' % name + if not existence_check_marker: + existing_tenant = tr[key].wait() + existence_check_marker.append(None) + if existing_tenant == None: + raise fdb.FDBError(2131) # tenant_not_found + del tr[key] + + +class Tenant(_TransactionCreator): + def __init__(self, tpointer): + self.tpointer = tpointer + + def __del__(self): + self.capi.fdb_tenant_destroy(self.tpointer) + + def create_transaction(self): + pointer = ctypes.c_void_p() + self.capi.fdb_tenant_create_transaction(self.tpointer, ctypes.byref(pointer)) + return Transaction(pointer.value, self) fill_operations() @@ -1458,6 +1536,10 @@ def init_c_api(): _capi.fdb_database_destroy.argtypes = [ctypes.c_void_p] _capi.fdb_database_destroy.restype = None + _capi.fdb_database_open_tenant.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.POINTER(ctypes.c_void_p)] + _capi.fdb_database_open_tenant.restype = ctypes.c_int + _capi.fdb_database_open_tenant.errcheck = check_error_code + _capi.fdb_database_create_transaction.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)] _capi.fdb_database_create_transaction.restype = ctypes.c_int _capi.fdb_database_create_transaction.errcheck = check_error_code @@ -1466,6 +1548,13 @@ def init_c_api(): _capi.fdb_database_set_option.restype = ctypes.c_int _capi.fdb_database_set_option.errcheck = check_error_code + _capi.fdb_tenant_destroy.argtypes = [ctypes.c_void_p] + _capi.fdb_tenant_destroy.restype = None + + _capi.fdb_tenant_create_transaction.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)] + _capi.fdb_tenant_create_transaction.restype = ctypes.c_int + _capi.fdb_tenant_create_transaction.errcheck = check_error_code + _capi.fdb_transaction_destroy.argtypes = [ctypes.c_void_p] _capi.fdb_transaction_destroy.restype = None @@ -1686,10 +1775,10 @@ def init(event_model=None): raise asyncio.Return(self) return it() FDBRange.iterate = iterate - AT = Database.declare_asynchronous_transactions() + AT = _TransactionCreator.declare_asynchronous_transactions() for name in dir(AT): - if name.startswith("_Database__database_"): - setattr(Database, name, getattr(AT, name)) + if name.startswith("__TransactionCreator__creator_"): + setattr(_TransactionCreator, name, getattr(AT, name)) def to_list(self): if self._mode == StreamingMode.iterator: diff --git a/bindings/python/tests/tenant_tests.py b/bindings/python/tests/tenant_tests.py new file mode 100755 index 0000000000..9f35620b6a --- /dev/null +++ b/bindings/python/tests/tenant_tests.py @@ -0,0 +1,123 @@ +#!/usr/bin/python +# +# tenant_tests.py +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import fdb +import sys +import json +from fdb.tuple import pack + +if __name__ == '__main__': + fdb.api_version(710) + +def test_tenant_tuple_name(db): + tuplename=(b'test', b'level', b'hierarchy', 3, 1.24, 'str') + db.allocate_tenant(tuplename) + + tenant=db.open_tenant(tuplename) + tenant[b'foo'] = b'bar' + + assert tenant[b'foo'] == b'bar' + + del tenant[b'foo'] + db.delete_tenant(tuplename) + +def cleanup_tenant(db, tenant_name): + try: + tenant = db.open_tenant(tenant_name) + del tenant[:] + db.delete_tenant(tenant_name) + except fdb.FDBError as e: + if e.code == 2131: # tenant not found + pass + else: + raise + +def test_tenant_operations(db): + cleanup_tenant(db, b'tenant1') + cleanup_tenant(db, b'tenant2') + + db.allocate_tenant(b'tenant1') + db.allocate_tenant(b'tenant2') + + tenant1 = db.open_tenant(b'tenant1') + tenant2 = db.open_tenant(b'tenant2') + + db[b'tenant_test_key'] = b'no_tenant' + tenant1[b'tenant_test_key'] = b'tenant1' + tenant2[b'tenant_test_key'] = b'tenant2' + + tenant1_entry = db[b'\xff\xff/management/tenant_map/tenant1'] + tenant1_json = json.loads(tenant1_entry) + prefix1 = tenant1_json['prefix'].encode('utf8') + + tenant2_entry = db[b'\xff\xff/management/tenant_map/tenant2'] + tenant2_json = json.loads(tenant2_entry) + prefix2 = tenant2_json['prefix'].encode('utf8') + + assert tenant1[b'tenant_test_key'] == b'tenant1' + assert db[prefix1 + b'tenant_test_key'] == b'tenant1' + assert tenant2[b'tenant_test_key'] == b'tenant2' + assert db[prefix2 + b'tenant_test_key'] == b'tenant2' + assert db[b'tenant_test_key'] == b'no_tenant' + + tr1 = tenant1.create_transaction() + try: + del tr1[:] + tr1.commit().wait() + except fdb.FDBError as e: + tr.on_error(e).wait() + + assert tenant1[b'tenant_test_key'] == None + assert db[prefix1 + b'tenant_test_key'] == None + assert tenant2[b'tenant_test_key'] == b'tenant2' + assert db[prefix2 + b'tenant_test_key'] == b'tenant2' + assert db[b'tenant_test_key'] == b'no_tenant' + + db.delete_tenant(b'tenant1') + try: + tenant1[b'tenant_test_key'] + assert False + except fdb.FDBError as e: + assert e.code == 2131 # tenant not found + + del tenant2[:] + db.delete_tenant(b'tenant2') + + assert db[prefix1 + b'tenant_test_key'] == None + assert db[prefix2 + b'tenant_test_key'] == None + assert db[b'tenant_test_key'] == b'no_tenant' + + del db[b'tenant_test_key'] + + assert db[b'tenant_test_key'] == None + +def test_tenants(db): + test_tenant_tuple_name(db) + test_tenant_operations(db) + +# Expect a cluster file as input. This test will write to the FDB cluster, so +# be aware of potential side effects. +if __name__ == '__main__': + clusterFile = sys.argv[1] + db = fdb.open(clusterFile) + db.options.set_transaction_timeout(2000) # 2 seconds + db.options.set_transaction_retry_limit(3) + + test_tenants(db) diff --git a/bindings/python/tests/tester.py b/bindings/python/tests/tester.py index 6aa41dea4a..7f8d794207 100644 --- a/bindings/python/tests/tester.py +++ b/bindings/python/tests/tester.py @@ -49,6 +49,7 @@ from cancellation_timeout_tests import test_db_retry_limits from cancellation_timeout_tests import test_combinations from size_limit_tests import test_size_limit_option, test_get_approximate_size +from tenant_tests import test_tenants random.seed(0) @@ -112,12 +113,13 @@ class Stack: class Instruction: - def __init__(self, tr, stack, op, index, isDatabase=False, isSnapshot=False): + def __init__(self, tr, stack, op, index, isDatabase=False, isTenant=False, isSnapshot=False): self.tr = tr self.stack = stack self.op = op self.index = index self.isDatabase = isDatabase + self.isTenant = isTenant self.isSnapshot = isSnapshot def pop(self, count=None, with_idx=False): @@ -277,6 +279,7 @@ class Tester: def __init__(self, db, prefix): self.db = db + self.tenant = None self.instructions = self.db[fdb.tuple.range((prefix,))] @@ -317,7 +320,8 @@ class Tester: def new_transaction(self): with Tester.tr_map_lock: - Tester.tr_map[self.tr_name] = self.db.create_transaction() + tr_source = self.tenant if self.tenant is not None else self.db + Tester.tr_map[self.tr_name] = tr_source.create_transaction() def switch_transaction(self, name): self.tr_name = name @@ -335,18 +339,22 @@ class Tester: # print("%d. Instruction is %s" % (idx, op)) isDatabase = op.endswith(six.u('_DATABASE')) + isTenant = op.endswith(six.u('_TENANT')) isSnapshot = op.endswith(six.u('_SNAPSHOT')) if isDatabase: op = op[:-9] obj = self.db + elif isTenant: + op = op[:-7] + obj = self.tenant if self.tenant else self.db elif isSnapshot: op = op[:-9] obj = self.current_transaction().snapshot else: obj = self.current_transaction() - inst = Instruction(obj, self.stack, op, idx, isDatabase, isSnapshot) + inst = Instruction(obj, self.stack, op, idx, isDatabase, isTenant, isSnapshot) try: if inst.op == six.u("PUSH"): @@ -583,6 +591,19 @@ class Tester: prefix = inst.pop() Tester.wait_empty(self.db, prefix) inst.push(b"WAITED_FOR_EMPTY") + elif inst.op == six.u("TENANT_CREATE"): + name = inst.pop() + self.db.allocate_tenant(name) + inst.push(b"RESULT_NOT_PRESENT") + elif inst.op == six.u("TENANT_DELETE"): + name = inst.pop() + self.db.delete_tenant(name) + inst.push(b"RESULT_NOT_PRESENT") + elif inst.op == six.u("TENANT_SET_ACTIVE"): + name = inst.pop() + self.tenant = self.db.open_tenant(name) + elif inst.op == six.u("TENANT_CLEAR_ACTIVE"): + self.tenant = None elif inst.op == six.u("UNIT_TESTS"): try: test_db_options(db) @@ -600,6 +621,8 @@ class Tester: test_size_limit_option(db) test_get_approximate_size(db) + test_tenants(db) + except fdb.FDBError as e: print("Unit tests failed: %s" % e.description) traceback.print_exc() diff --git a/documentation/sphinx/requirements.txt b/documentation/sphinx/requirements.txt index 67ca207628..06e23ea6d3 100644 --- a/documentation/sphinx/requirements.txt +++ b/documentation/sphinx/requirements.txt @@ -3,3 +3,4 @@ setuptools>=20.10.0,<=57.4.0 sphinx==1.5.6 sphinx-bootstrap-theme==0.4.8 docutils==0.16 +Jinja2==3.0.3 diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index f3af667e0c..5dab3e49c6 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -7,7 +7,7 @@ .. |database-type| replace:: ``Database`` .. |database-class| replace:: :class:`Database` .. |database-auto| replace:: the :func:`@fdb.transactional ` decorator -.. |tenant-type| replace:: FIXME +.. |tenant-type| replace:: :class:`Tenant` .. |transaction-class| replace:: :class:`Transaction` .. |get-key-func| replace:: :func:`Transaction.get_key` .. |get-range-func| replace:: :func:`Transaction.get_range` @@ -316,9 +316,29 @@ A |database-blurb1| |database-blurb2| Returns a new :class:`Transaction` object. Consider using the :func:`@fdb.transactional ` decorator to create transactions instead, since it will automatically provide you with appropriate retry behavior. +.. method:: Database.open_tenant(tenant_name) + + Opens an existing tenant to be used for running transactions and returns it as a :class`Tenant` object. + + The tenant name can be either a byte string or a tuple. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name. + .. |sync-read| replace:: This read is fully synchronous. .. |sync-write| replace:: This change will be committed immediately, and is fully synchronous. +.. method:: Database.allocate_tenant(tenant_name): + + Creates a new tenant in the cluster. |sync-write| + + The tenant name can be either a byte string or a tuple and cannot start with the ``\xff`` byte. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name. + +.. method:: Database.delete_tenant(tenant_name): + + Delete a tenant from the cluster. |sync-write| + + The tenant name can be either a byte string or a tuple. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name. + + It is an error to delete a tenant that still has data. To delete a non-empty tenant, first clear all of the keys in the tenant. + .. method:: Database.get(key) Returns the value associated with the specified key in the database (or ``None`` if the key does not exist). |sync-read| @@ -460,6 +480,17 @@ Database options .. method:: Database.options.set_snapshot_ryw_disable() |option-db-snapshot-ryw-disable-blurb| + +Tenant objects +============== + +.. class:: Tenant + +|tenant-blurb1| + +.. method:: Tenant.create_transaction() + + Returns a new :class:`Transaction` object. Consider using the :func:`@fdb.transactional ` decorator to create transactions instead, since it will automatically provide you with appropriate retry behavior. .. _api-python-transactional-decorator: @@ -479,9 +510,9 @@ Transactional decoration The ``@fdb.transactional`` decorator makes ``simple_function`` a transactional function. All functions using this decorator must have an argument **named** ``tr``. This specially named argument is passed a transaction that the function can use to do reads and writes. - A caller of a transactionally decorated function can pass a :class:`Database` instead of a transaction for the ``tr`` parameter. Then a transaction will be created automatically, and automatically committed before returning to the caller. The decorator will retry calling the decorated function until the transaction successfully commits. + A caller of a transactionally decorated function can pass a :class:`Database` or :class:`Tenant` instead of a transaction for the ``tr`` parameter. Then a transaction will be created automatically, and automatically committed before returning to the caller. The decorator will retry calling the decorated function until the transaction successfully commits. - If ``db`` is a :class:`Database`, a call like :: + If ``db`` is a :class:`Database` or :class:`Tenant`, a call like :: simple_function(db, 'a', 'b') @@ -744,7 +775,7 @@ Committing .. decorator:: transactional() - The ``transactional`` decorator makes it easy to write transactional functions which accept either a :class:`Database` or a :class:`Transaction` as a parameter and automatically commit. See :func:`@fdb.transactional ` for explanation and examples. + The ``transactional`` decorator makes it easy to write transactional functions which accept a :class:`Database`, :class`Tenant`, or :class:`Transaction` as a parameter and automatically commit. See :func:`@fdb.transactional ` for explanation and examples. .. method :: Transaction.commit() @@ -754,7 +785,7 @@ Committing |commit-outstanding-reads-blurb| - .. note :: Consider using the :func:`@fdb.transactional ` decorator, which not only calls :meth:`Database.create_transaction` and :meth:`Transaction.commit()` for you but also implements the required error handling and retry logic for transactions. + .. note :: Consider using the :func:`@fdb.transactional ` decorator, which not only calls :meth:`Database.create_transaction` or :meth`Tenant.create_transaction` and :meth:`Transaction.commit()` for you but also implements the required error handling and retry logic for transactions. .. warning :: |used-during-commit-blurb| diff --git a/documentation/sphinx/source/backups.rst b/documentation/sphinx/source/backups.rst index 740da17706..9dcf5ffd8b 100644 --- a/documentation/sphinx/source/backups.rst +++ b/documentation/sphinx/source/backups.rst @@ -155,6 +155,12 @@ Here is a complete list of valid parameters: **Example**: The URL parameter *header=x-amz-storage-class:REDUCED_REDUNDANCY* would send the HTTP header required to use the reduced redundancy storage option in the S3 API. +Signing Protocol +================= + +AWS signature version 4 is the default signing protocol choice. This boolean knob ``--knob_http_request_aws_v4_header`` can be used to select between v4 style and v2 style signatures. +If the knob is set to ``true`` then v4 signature will be used and if set to ``false`` then v2 signature will be used. + .. _blob-credential-files: Blob Credential Files diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h index a33032b183..5ad5c53b1b 100644 --- a/fdbbackup/FileConverter.h +++ b/fdbbackup/FileConverter.h @@ -46,6 +46,7 @@ enum { OPT_HEX_KEY_PREFIX, OPT_BEGIN_VERSION_FILTER, OPT_END_VERSION_FILTER, + OPT_KNOB, OPT_HELP }; @@ -72,6 +73,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP }, { OPT_HEX_KEY_PREFIX, "--hex-prefix", SO_REQ_SEP }, { OPT_BEGIN_VERSION_FILTER, "--begin-version-filter", SO_REQ_SEP }, { OPT_END_VERSION_FILTER, "--end-version-filter", SO_REQ_SEP }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, { OPT_HELP, "-?", SO_NONE }, { OPT_HELP, "-h", SO_NONE }, { OPT_HELP, "--help", SO_NONE }, diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp index 93c6cdc084..7e851bf6e0 100644 --- a/fdbbackup/FileDecoder.actor.cpp +++ b/fdbbackup/FileDecoder.actor.cpp @@ -26,17 +26,21 @@ #include #include "fdbbackup/BackupTLSConfig.h" +#include "fdbclient/BuildFlags.h" +#include "fdbbackup/FileConverter.h" #include "fdbclient/BackupAgent.actor.h" #include "fdbclient/BackupContainer.h" -#include "fdbbackup/FileConverter.h" #include "fdbclient/CommitTransaction.h" #include "fdbclient/FDBTypes.h" +#include "fdbclient/IKnobCollection.h" +#include "fdbclient/Knobs.h" #include "fdbclient/MutationList.h" +#include "flow/ArgParseUtil.h" #include "flow/IRandom.h" #include "flow/Trace.h" #include "flow/flow.h" #include "flow/serialize.h" -#include "fdbclient/BuildFlags.h" + #include "flow/actorcompiler.h" // has to be last include #define SevDecodeInfo SevVerbose @@ -73,11 +77,13 @@ void printDecodeUsage() { " --list-only Print file list and exit.\n" " -k KEY_PREFIX Use the prefix for filtering mutations\n" " --hex-prefix HEX_PREFIX\n" - " The prefix specified in HEX format, e.g., \\x05\\x01.\n" + " The prefix specified in HEX format, e.g., \"\\\\x05\\\\x01\".\n" " --begin-version-filter BEGIN_VERSION\n" " The version range's begin version (inclusive) for filtering.\n" " --end-version-filter END_VERSION\n" " The version range's end version (exclusive) for filtering.\n" + " --knob-KNOBNAME KNOBVALUE\n" + " Changes a knob value. KNOBNAME should be lowercase." "\n"; return; } @@ -97,6 +103,8 @@ struct DecodeParams { Version beginVersionFilter = 0; Version endVersionFilter = std::numeric_limits::max(); + std::vector> knobs; + // Returns if [begin, end) overlap with the filter range bool overlap(Version begin, Version end) const { // Filter [100, 200), [50,75) [200, 300) @@ -130,8 +138,39 @@ struct DecodeParams { if (!prefix.empty()) { s.append(", KeyPrefix: ").append(printable(KeyRef(prefix))); } + for (const auto& [knob, value] : knobs) { + s.append(", KNOB-").append(knob).append(" = ").append(value); + } return s; } + + void updateKnobs() { + auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection(); + for (const auto& [knobName, knobValueString] : knobs) { + try { + auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString); + g_knobs.setKnob(knobName, knobValue); + } catch (Error& e) { + if (e.code() == error_code_invalid_option_value) { + std::cerr << "WARNING: Invalid value '" << knobValueString << "' for knob option '" << knobName + << "'\n"; + TraceEvent(SevWarnAlways, "InvalidKnobValue") + .detail("Knob", printable(knobName)) + .detail("Value", printable(knobValueString)); + } else { + std::cerr << "ERROR: Failed to set knob option '" << knobName << "': " << e.what() << "\n"; + TraceEvent(SevError, "FailedToSetKnob") + .errorUnsuppressed(e) + .detail("Knob", printable(knobName)) + .detail("Value", printable(knobValueString)); + throw; + } + } + } + + // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs + g_knobs.initialize(Randomize::True, IsSimulated::False); + } }; // Decode an ASCII string, e.g., "\x15\x1b\x19\x04\xaf\x0c\x28\x0a", @@ -256,6 +295,16 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) { param->tlsConfig.blobCredentials.push_back(args->OptionArg()); break; + case OPT_KNOB: { + Optional knobName = extractPrefixedArgument("--knob", args->OptionSyntax()); + if (!knobName.present()) { + std::cerr << "ERROR: unable to parse knob option '" << args->OptionSyntax() << "'\n"; + return FDB_EXIT_ERROR; + } + param->knobs.emplace_back(knobName.get(), args->OptionArg()); + break; + } + #ifndef TLS_DISABLED case TLSConfig::OPT_TLS_PLUGIN: args->OptionArg(); @@ -552,6 +601,9 @@ int main(int argc, char** argv) { StringRef url(param.container_url); setupNetwork(0, UseMetrics::True); + // Must be called after setupNetwork() to be effective + param.updateKnobs(); + TraceEvent::setNetworkThread(); openTraceFile(NetworkAddress(), 10 << 20, 500 << 20, param.log_dir, "decode", param.trace_log_group); param.tlsConfig.setupBlobCredentials(); diff --git a/fdbclient/JSONDoc.h b/fdbclient/JSONDoc.h index 2fdeb7ba66..39a1b388ee 100644 --- a/fdbclient/JSONDoc.h +++ b/fdbclient/JSONDoc.h @@ -22,6 +22,7 @@ #include "fdbclient/json_spirit/json_spirit_writer_template.h" #include "fdbclient/json_spirit/json_spirit_reader_template.h" +#include "flow/Error.h" // JSONDoc is a convenient reader/writer class for manipulating JSON documents using "paths". // Access is done using a "path", which is a string of dot-separated diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp index b63b930ee8..a4fa95616a 100644 --- a/fdbclient/S3BlobStore.actor.cpp +++ b/fdbclient/S3BlobStore.actor.cpp @@ -573,7 +573,7 @@ ACTOR Future updateSecret_impl(Reference b) { JSONDoc accounts(doc.last().get_obj()); if (accounts.has(credentialsFileKey, false) && accounts.last().type() == json_spirit::obj_type) { JSONDoc account(accounts.last()); - S3BlobStoreEndpoint::Credentials creds; + S3BlobStoreEndpoint::Credentials creds = b->credentials.get(); if (b->lookupKey) { std::string apiKey; if (account.tryGet("api_key", apiKey))