Merge branch 'main' into s3_sdk_credentials
This commit is contained in:
Josh Slocum
commit
5684fcace4
|
|
@ -58,8 +58,8 @@ _java_cmd = 'java -ea -cp %s:%s com.apple.foundationdb.test.' % (
|
|||
|
||||
# We could set min_api_version lower on some of these if the testers were updated to support them
|
||||
testers = {
|
||||
'python': Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES),
|
||||
'python3': Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES),
|
||||
'python': Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True),
|
||||
'python3': Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True),
|
||||
'ruby': Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 2040, 23, MAX_API_VERSION),
|
||||
'java': Tester('java', _java_cmd + 'StackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES),
|
||||
'java_async': Tester('java', _java_cmd + 'AsyncStackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES),
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ def api_version(ver):
|
|||
'predicates',
|
||||
'Future',
|
||||
'Database',
|
||||
'Tenant',
|
||||
'Transaction',
|
||||
'KeyValue',
|
||||
'KeySelector',
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import traceback
|
|||
|
||||
import fdb
|
||||
from fdb import six
|
||||
from fdb.tuple import pack, unpack
|
||||
|
||||
_network_thread = None
|
||||
_network_thread_reentrant_lock = threading.RLock()
|
||||
|
|
@ -198,9 +199,10 @@ def transactional(*tr_args, **tr_kwargs):
|
|||
one of two actions, depending on the type of the parameter passed
|
||||
to the function at call time.
|
||||
|
||||
If given a Database, a Transaction will be created and passed into
|
||||
the wrapped code in place of the Database. After the function is
|
||||
complete, the newly created transaction will be committed.
|
||||
If given a Database or Tenant, a Transaction will be created and
|
||||
passed into the wrapped code in place of the Database or Tenant.
|
||||
After the function is complete, the newly created transaction
|
||||
will be committed.
|
||||
|
||||
It is important to note that the wrapped method may be called
|
||||
multiple times in the event of a commit failure, until the commit
|
||||
|
|
@ -943,128 +945,114 @@ class FormerFuture(_FDBBase):
|
|||
except:
|
||||
pass
|
||||
|
||||
|
||||
class Database(_FDBBase):
|
||||
def __init__(self, dpointer):
|
||||
self.dpointer = dpointer
|
||||
self.options = _DatabaseOptions(self)
|
||||
|
||||
def __del__(self):
|
||||
# print('Destroying database 0x%x' % self.dpointer)
|
||||
self.capi.fdb_database_destroy(self.dpointer)
|
||||
|
||||
class _TransactionCreator(_FDBBase):
|
||||
def get(self, key):
|
||||
return Database.__database_getitem(self, key)
|
||||
return _TransactionCreator.__creator_getitem(self, key)
|
||||
|
||||
def __getitem__(self, key):
|
||||
if isinstance(key, slice):
|
||||
return self.get_range(key.start, key.stop, reverse=(key.step == -1))
|
||||
return Database.__database_getitem(self, key)
|
||||
return _TransactionCreator.__creator_getitem(self, key)
|
||||
|
||||
def get_key(self, key_selector):
|
||||
return Database.__database_get_key(self, key_selector)
|
||||
return _TransactionCreator.__creator_get_key(self, key_selector)
|
||||
|
||||
def get_range(self, begin, end, limit=0, reverse=False, streaming_mode=StreamingMode.want_all):
|
||||
return Database.__database_get_range(self, begin, end, limit, reverse, streaming_mode)
|
||||
return _TransactionCreator.__creator_get_range(self, begin, end, limit, reverse, streaming_mode)
|
||||
|
||||
def get_range_startswith(self, prefix, *args, **kwargs):
|
||||
return Database.__database_get_range_startswith(self, prefix, *args, **kwargs)
|
||||
return _TransactionCreator.__creator_get_range_startswith(self, prefix, *args, **kwargs)
|
||||
|
||||
def set(self, key, value):
|
||||
Database.__database_setitem(self, key, value)
|
||||
_TransactionCreator.__creator_setitem(self, key, value)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
Database.__database_setitem(self, key, value)
|
||||
_TransactionCreator.__creator_setitem(self, key, value)
|
||||
|
||||
def clear(self, key):
|
||||
Database.__database_delitem(self, key)
|
||||
_TransactionCreator.__creator_delitem(self, key)
|
||||
|
||||
def clear_range(self, begin, end):
|
||||
Database.__database_delitem(self, slice(begin, end))
|
||||
_TransactionCreator.__creator_delitem(self, slice(begin, end))
|
||||
|
||||
def __delitem__(self, key_or_slice):
|
||||
Database.__database_delitem(self, key_or_slice)
|
||||
_TransactionCreator.__creator_delitem(self, key_or_slice)
|
||||
|
||||
def clear_range_startswith(self, prefix):
|
||||
Database.__database_clear_range_startswith(self, prefix)
|
||||
_TransactionCreator.__creator_clear_range_startswith(self, prefix)
|
||||
|
||||
def get_and_watch(self, key):
|
||||
return Database.__database_get_and_watch(self, key)
|
||||
return _TransactionCreator.__creator_get_and_watch(self, key)
|
||||
|
||||
def set_and_watch(self, key, value):
|
||||
return Database.__database_set_and_watch(self, key, value)
|
||||
return _TransactionCreator.__creator_set_and_watch(self, key, value)
|
||||
|
||||
def clear_and_watch(self, key):
|
||||
return Database.__database_clear_and_watch(self, key)
|
||||
return _TransactionCreator.__creator_clear_and_watch(self, key)
|
||||
|
||||
def create_transaction(self):
|
||||
pointer = ctypes.c_void_p()
|
||||
self.capi.fdb_database_create_transaction(self.dpointer, ctypes.byref(pointer))
|
||||
return Transaction(pointer.value, self)
|
||||
|
||||
def _set_option(self, option, param, length):
|
||||
self.capi.fdb_database_set_option(self.dpointer, option, param, length)
|
||||
pass
|
||||
|
||||
def _atomic_operation(self, opcode, key, param):
|
||||
Database.__database_atomic_operation(self, opcode, key, param)
|
||||
_TransactionCreator.__creator_atomic_operation(self, opcode, key, param)
|
||||
|
||||
#### Transaction implementations ####
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_getitem(tr, key):
|
||||
def __creator_getitem(tr, key):
|
||||
return tr[key].value
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_get_key(tr, key_selector):
|
||||
def __creator_get_key(tr, key_selector):
|
||||
return tr.get_key(key_selector).value
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_get_range(tr, begin, end, limit, reverse, streaming_mode):
|
||||
def __creator_get_range(tr, begin, end, limit, reverse, streaming_mode):
|
||||
return tr.get_range(begin, end, limit, reverse, streaming_mode).to_list()
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_get_range_startswith(tr, prefix, *args, **kwargs):
|
||||
def __creator_get_range_startswith(tr, prefix, *args, **kwargs):
|
||||
return tr.get_range_startswith(prefix, *args, **kwargs).to_list()
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_setitem(tr, key, value):
|
||||
def __creator_setitem(tr, key, value):
|
||||
tr[key] = value
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_clear_range_startswith(tr, prefix):
|
||||
def __creator_clear_range_startswith(tr, prefix):
|
||||
tr.clear_range_startswith(prefix)
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_get_and_watch(tr, key):
|
||||
def __creator_get_and_watch(tr, key):
|
||||
v = tr.get(key)
|
||||
return v, tr.watch(key)
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_set_and_watch(tr, key, value):
|
||||
def __creator_set_and_watch(tr, key, value):
|
||||
tr.set(key, value)
|
||||
return tr.watch(key)
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_clear_and_watch(tr, key):
|
||||
def __creator_clear_and_watch(tr, key):
|
||||
del tr[key]
|
||||
return tr.watch(key)
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_delitem(tr, key_or_slice):
|
||||
def __creator_delitem(tr, key_or_slice):
|
||||
del tr[key_or_slice]
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_atomic_operation(tr, opcode, key, param):
|
||||
def __creator_atomic_operation(tr, opcode, key, param):
|
||||
tr._atomic_operation(opcode, key, param)
|
||||
|
||||
# Asynchronous transactions
|
||||
|
|
@ -1074,11 +1062,11 @@ class Database(_FDBBase):
|
|||
From = asyncio.From
|
||||
coroutine = asyncio.coroutine
|
||||
|
||||
class Database:
|
||||
class TransactionCreator:
|
||||
@staticmethod
|
||||
@transactional
|
||||
@coroutine
|
||||
def __database_getitem(tr, key):
|
||||
def __creator_getitem(tr, key):
|
||||
# raise Return(( yield From( tr[key] ) ))
|
||||
raise Return(tr[key])
|
||||
yield None
|
||||
|
|
@ -1086,26 +1074,26 @@ class Database(_FDBBase):
|
|||
@staticmethod
|
||||
@transactional
|
||||
@coroutine
|
||||
def __database_get_key(tr, key_selector):
|
||||
def __creator_get_key(tr, key_selector):
|
||||
raise Return(tr.get_key(key_selector))
|
||||
yield None
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
@coroutine
|
||||
def __database_get_range(tr, begin, end, limit, reverse, streaming_mode):
|
||||
def __creator_get_range(tr, begin, end, limit, reverse, streaming_mode):
|
||||
raise Return((yield From(tr.get_range(begin, end, limit, reverse, streaming_mode).to_list())))
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
@coroutine
|
||||
def __database_get_range_startswith(tr, prefix, *args, **kwargs):
|
||||
def __creator_get_range_startswith(tr, prefix, *args, **kwargs):
|
||||
raise Return((yield From(tr.get_range_startswith(prefix, *args, **kwargs).to_list())))
|
||||
|
||||
@staticmethod
|
||||
@transactional
|
||||
@coroutine
|
||||
def __database_setitem(tr, key, value):
|
||||
def __creator_setitem(tr, key, value):
|
||||
tr[key] = value
|
||||
raise Return()
|
||||
yield None
|
||||
|
|
@ -1113,7 +1101,7 @@ class Database(_FDBBase):
|
|||
@staticmethod
|
||||
@transactional
|
||||
@coroutine
|
||||
def __database_clear_range_startswith(tr, prefix):
|
||||
def __creator_clear_range_startswith(tr, prefix):
|
||||
tr.clear_range_startswith(prefix)
|
||||
raise Return()
|
||||
yield None
|
||||
|
|
@ -1121,7 +1109,7 @@ class Database(_FDBBase):
|
|||
@staticmethod
|
||||
@transactional
|
||||
@coroutine
|
||||
def __database_get_and_watch(tr, key):
|
||||
def __creator_get_and_watch(tr, key):
|
||||
v = tr.get(key)
|
||||
raise Return(v, tr.watch(key))
|
||||
yield None
|
||||
|
|
@ -1129,7 +1117,7 @@ class Database(_FDBBase):
|
|||
@staticmethod
|
||||
@transactional
|
||||
@coroutine
|
||||
def __database_set_and_watch(tr, key, value):
|
||||
def __creator_set_and_watch(tr, key, value):
|
||||
tr.set(key, value)
|
||||
raise Return(tr.watch(key))
|
||||
yield None
|
||||
|
|
@ -1137,7 +1125,7 @@ class Database(_FDBBase):
|
|||
@staticmethod
|
||||
@transactional
|
||||
@coroutine
|
||||
def __database_clear_and_watch(tr, key):
|
||||
def __creator_clear_and_watch(tr, key):
|
||||
del tr[key]
|
||||
raise Return(tr.watch(key))
|
||||
yield None
|
||||
|
|
@ -1145,7 +1133,7 @@ class Database(_FDBBase):
|
|||
@staticmethod
|
||||
@transactional
|
||||
@coroutine
|
||||
def __database_delitem(tr, key_or_slice):
|
||||
def __creator_delitem(tr, key_or_slice):
|
||||
del tr[key_or_slice]
|
||||
raise Return()
|
||||
yield None
|
||||
|
|
@ -1153,11 +1141,101 @@ class Database(_FDBBase):
|
|||
@staticmethod
|
||||
@transactional
|
||||
@coroutine
|
||||
def __database_atomic_operation(tr, opcode, key, param):
|
||||
def __creator_atomic_operation(tr, opcode, key, param):
|
||||
tr._atomic_operation(opcode, key, param)
|
||||
raise Return()
|
||||
yield None
|
||||
return Database
|
||||
return TransactionCreator
|
||||
|
||||
def process_tenant_name(name):
|
||||
if isinstance(name, tuple):
|
||||
return pack(name)
|
||||
elif isinstance(name, bytes):
|
||||
return name
|
||||
else:
|
||||
raise TypeError('Tenant name must be of type ' + bytes.__name__ + ' or of type ' + tuple.__name__)
|
||||
|
||||
class Database(_TransactionCreator):
|
||||
def __init__(self, dpointer):
|
||||
self.dpointer = dpointer
|
||||
self.options = _DatabaseOptions(self)
|
||||
|
||||
def __del__(self):
|
||||
# print('Destroying database 0x%x' % self.dpointer)
|
||||
self.capi.fdb_database_destroy(self.dpointer)
|
||||
|
||||
def _set_option(self, option, param, length):
|
||||
self.capi.fdb_database_set_option(self.dpointer, option, param, length)
|
||||
|
||||
def open_tenant(self, name):
|
||||
tname = process_tenant_name(name)
|
||||
pointer = ctypes.c_void_p()
|
||||
self.capi.fdb_database_open_tenant(self.dpointer, tname, len(tname), ctypes.byref(pointer))
|
||||
return Tenant(pointer.value)
|
||||
|
||||
def create_transaction(self):
|
||||
pointer = ctypes.c_void_p()
|
||||
self.capi.fdb_database_create_transaction(self.dpointer, ctypes.byref(pointer))
|
||||
return Transaction(pointer.value, self)
|
||||
|
||||
def allocate_tenant(self, name):
|
||||
Database.__database_allocate_tenant(self, process_tenant_name(name), [])
|
||||
|
||||
def delete_tenant(self, name):
|
||||
Database.__database_delete_tenant(self, process_tenant_name(name), [])
|
||||
|
||||
# Attempt to allocate a tenant in the cluster. If the tenant already exists,
|
||||
# this function will return a tenant_already_exists error. If the tenant is created
|
||||
# concurrently, then this function may return success even if another caller creates
|
||||
# it.
|
||||
#
|
||||
# The existence_check_marker is expected to be an empty list. This function will
|
||||
# modify the list after completing the existence check to avoid checking for existence
|
||||
# on retries. This allows the operation to be idempotent.
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_allocate_tenant(tr, name, existence_check_marker):
|
||||
tr.options.set_special_key_space_enable_writes()
|
||||
key = b'\xff\xff/management/tenant_map/%s' % name
|
||||
if not existence_check_marker:
|
||||
existing_tenant = tr[key].wait()
|
||||
existence_check_marker.append(None)
|
||||
if existing_tenant != None:
|
||||
raise fdb.FDBError(2132) # tenant_already_exists
|
||||
tr[key] = b''
|
||||
|
||||
# Attempt to remove a tenant in the cluster. If the tenant doesn't exist, this
|
||||
# function will return a tenant_not_found error. If the tenant is deleted
|
||||
# concurrently, then this function may return success even if another caller deletes
|
||||
# it.
|
||||
#
|
||||
# The existence_check_marker is expected to be an empty list. This function will
|
||||
# modify the list after completing the existence check to avoid checking for existence
|
||||
# on retries. This allows the operation to be idempotent.
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_delete_tenant(tr, name, existence_check_marker):
|
||||
tr.options.set_special_key_space_enable_writes()
|
||||
key = b'\xff\xff/management/tenant_map/%s' % name
|
||||
if not existence_check_marker:
|
||||
existing_tenant = tr[key].wait()
|
||||
existence_check_marker.append(None)
|
||||
if existing_tenant == None:
|
||||
raise fdb.FDBError(2131) # tenant_not_found
|
||||
del tr[key]
|
||||
|
||||
|
||||
class Tenant(_TransactionCreator):
|
||||
def __init__(self, tpointer):
|
||||
self.tpointer = tpointer
|
||||
|
||||
def __del__(self):
|
||||
self.capi.fdb_tenant_destroy(self.tpointer)
|
||||
|
||||
def create_transaction(self):
|
||||
pointer = ctypes.c_void_p()
|
||||
self.capi.fdb_tenant_create_transaction(self.tpointer, ctypes.byref(pointer))
|
||||
return Transaction(pointer.value, self)
|
||||
|
||||
|
||||
fill_operations()
|
||||
|
|
@ -1458,6 +1536,10 @@ def init_c_api():
|
|||
_capi.fdb_database_destroy.argtypes = [ctypes.c_void_p]
|
||||
_capi.fdb_database_destroy.restype = None
|
||||
|
||||
_capi.fdb_database_open_tenant.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.POINTER(ctypes.c_void_p)]
|
||||
_capi.fdb_database_open_tenant.restype = ctypes.c_int
|
||||
_capi.fdb_database_open_tenant.errcheck = check_error_code
|
||||
|
||||
_capi.fdb_database_create_transaction.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
|
||||
_capi.fdb_database_create_transaction.restype = ctypes.c_int
|
||||
_capi.fdb_database_create_transaction.errcheck = check_error_code
|
||||
|
|
@ -1466,6 +1548,13 @@ def init_c_api():
|
|||
_capi.fdb_database_set_option.restype = ctypes.c_int
|
||||
_capi.fdb_database_set_option.errcheck = check_error_code
|
||||
|
||||
_capi.fdb_tenant_destroy.argtypes = [ctypes.c_void_p]
|
||||
_capi.fdb_tenant_destroy.restype = None
|
||||
|
||||
_capi.fdb_tenant_create_transaction.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
|
||||
_capi.fdb_tenant_create_transaction.restype = ctypes.c_int
|
||||
_capi.fdb_tenant_create_transaction.errcheck = check_error_code
|
||||
|
||||
_capi.fdb_transaction_destroy.argtypes = [ctypes.c_void_p]
|
||||
_capi.fdb_transaction_destroy.restype = None
|
||||
|
||||
|
|
@ -1686,10 +1775,10 @@ def init(event_model=None):
|
|||
raise asyncio.Return(self)
|
||||
return it()
|
||||
FDBRange.iterate = iterate
|
||||
AT = Database.declare_asynchronous_transactions()
|
||||
AT = _TransactionCreator.declare_asynchronous_transactions()
|
||||
for name in dir(AT):
|
||||
if name.startswith("_Database__database_"):
|
||||
setattr(Database, name, getattr(AT, name))
|
||||
if name.startswith("__TransactionCreator__creator_"):
|
||||
setattr(_TransactionCreator, name, getattr(AT, name))
|
||||
|
||||
def to_list(self):
|
||||
if self._mode == StreamingMode.iterator:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,123 @@
|
|||
#!/usr/bin/python
|
||||
#
|
||||
# tenant_tests.py
|
||||
#
|
||||
# This source file is part of the FoundationDB open source project
|
||||
#
|
||||
# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import fdb
|
||||
import sys
|
||||
import json
|
||||
from fdb.tuple import pack
|
||||
|
||||
if __name__ == '__main__':
|
||||
fdb.api_version(710)
|
||||
|
||||
def test_tenant_tuple_name(db):
|
||||
tuplename=(b'test', b'level', b'hierarchy', 3, 1.24, 'str')
|
||||
db.allocate_tenant(tuplename)
|
||||
|
||||
tenant=db.open_tenant(tuplename)
|
||||
tenant[b'foo'] = b'bar'
|
||||
|
||||
assert tenant[b'foo'] == b'bar'
|
||||
|
||||
del tenant[b'foo']
|
||||
db.delete_tenant(tuplename)
|
||||
|
||||
def cleanup_tenant(db, tenant_name):
|
||||
try:
|
||||
tenant = db.open_tenant(tenant_name)
|
||||
del tenant[:]
|
||||
db.delete_tenant(tenant_name)
|
||||
except fdb.FDBError as e:
|
||||
if e.code == 2131: # tenant not found
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
def test_tenant_operations(db):
|
||||
cleanup_tenant(db, b'tenant1')
|
||||
cleanup_tenant(db, b'tenant2')
|
||||
|
||||
db.allocate_tenant(b'tenant1')
|
||||
db.allocate_tenant(b'tenant2')
|
||||
|
||||
tenant1 = db.open_tenant(b'tenant1')
|
||||
tenant2 = db.open_tenant(b'tenant2')
|
||||
|
||||
db[b'tenant_test_key'] = b'no_tenant'
|
||||
tenant1[b'tenant_test_key'] = b'tenant1'
|
||||
tenant2[b'tenant_test_key'] = b'tenant2'
|
||||
|
||||
tenant1_entry = db[b'\xff\xff/management/tenant_map/tenant1']
|
||||
tenant1_json = json.loads(tenant1_entry)
|
||||
prefix1 = tenant1_json['prefix'].encode('utf8')
|
||||
|
||||
tenant2_entry = db[b'\xff\xff/management/tenant_map/tenant2']
|
||||
tenant2_json = json.loads(tenant2_entry)
|
||||
prefix2 = tenant2_json['prefix'].encode('utf8')
|
||||
|
||||
assert tenant1[b'tenant_test_key'] == b'tenant1'
|
||||
assert db[prefix1 + b'tenant_test_key'] == b'tenant1'
|
||||
assert tenant2[b'tenant_test_key'] == b'tenant2'
|
||||
assert db[prefix2 + b'tenant_test_key'] == b'tenant2'
|
||||
assert db[b'tenant_test_key'] == b'no_tenant'
|
||||
|
||||
tr1 = tenant1.create_transaction()
|
||||
try:
|
||||
del tr1[:]
|
||||
tr1.commit().wait()
|
||||
except fdb.FDBError as e:
|
||||
tr.on_error(e).wait()
|
||||
|
||||
assert tenant1[b'tenant_test_key'] == None
|
||||
assert db[prefix1 + b'tenant_test_key'] == None
|
||||
assert tenant2[b'tenant_test_key'] == b'tenant2'
|
||||
assert db[prefix2 + b'tenant_test_key'] == b'tenant2'
|
||||
assert db[b'tenant_test_key'] == b'no_tenant'
|
||||
|
||||
db.delete_tenant(b'tenant1')
|
||||
try:
|
||||
tenant1[b'tenant_test_key']
|
||||
assert False
|
||||
except fdb.FDBError as e:
|
||||
assert e.code == 2131 # tenant not found
|
||||
|
||||
del tenant2[:]
|
||||
db.delete_tenant(b'tenant2')
|
||||
|
||||
assert db[prefix1 + b'tenant_test_key'] == None
|
||||
assert db[prefix2 + b'tenant_test_key'] == None
|
||||
assert db[b'tenant_test_key'] == b'no_tenant'
|
||||
|
||||
del db[b'tenant_test_key']
|
||||
|
||||
assert db[b'tenant_test_key'] == None
|
||||
|
||||
def test_tenants(db):
|
||||
test_tenant_tuple_name(db)
|
||||
test_tenant_operations(db)
|
||||
|
||||
# Expect a cluster file as input. This test will write to the FDB cluster, so
|
||||
# be aware of potential side effects.
|
||||
if __name__ == '__main__':
|
||||
clusterFile = sys.argv[1]
|
||||
db = fdb.open(clusterFile)
|
||||
db.options.set_transaction_timeout(2000) # 2 seconds
|
||||
db.options.set_transaction_retry_limit(3)
|
||||
|
||||
test_tenants(db)
|
||||
|
|
@ -49,6 +49,7 @@ from cancellation_timeout_tests import test_db_retry_limits
|
|||
from cancellation_timeout_tests import test_combinations
|
||||
|
||||
from size_limit_tests import test_size_limit_option, test_get_approximate_size
|
||||
from tenant_tests import test_tenants
|
||||
|
||||
random.seed(0)
|
||||
|
||||
|
|
@ -112,12 +113,13 @@ class Stack:
|
|||
|
||||
|
||||
class Instruction:
|
||||
def __init__(self, tr, stack, op, index, isDatabase=False, isSnapshot=False):
|
||||
def __init__(self, tr, stack, op, index, isDatabase=False, isTenant=False, isSnapshot=False):
|
||||
self.tr = tr
|
||||
self.stack = stack
|
||||
self.op = op
|
||||
self.index = index
|
||||
self.isDatabase = isDatabase
|
||||
self.isTenant = isTenant
|
||||
self.isSnapshot = isSnapshot
|
||||
|
||||
def pop(self, count=None, with_idx=False):
|
||||
|
|
@ -277,6 +279,7 @@ class Tester:
|
|||
|
||||
def __init__(self, db, prefix):
|
||||
self.db = db
|
||||
self.tenant = None
|
||||
|
||||
self.instructions = self.db[fdb.tuple.range((prefix,))]
|
||||
|
||||
|
|
@ -317,7 +320,8 @@ class Tester:
|
|||
|
||||
def new_transaction(self):
|
||||
with Tester.tr_map_lock:
|
||||
Tester.tr_map[self.tr_name] = self.db.create_transaction()
|
||||
tr_source = self.tenant if self.tenant is not None else self.db
|
||||
Tester.tr_map[self.tr_name] = tr_source.create_transaction()
|
||||
|
||||
def switch_transaction(self, name):
|
||||
self.tr_name = name
|
||||
|
|
@ -335,18 +339,22 @@ class Tester:
|
|||
# print("%d. Instruction is %s" % (idx, op))
|
||||
|
||||
isDatabase = op.endswith(six.u('_DATABASE'))
|
||||
isTenant = op.endswith(six.u('_TENANT'))
|
||||
isSnapshot = op.endswith(six.u('_SNAPSHOT'))
|
||||
|
||||
if isDatabase:
|
||||
op = op[:-9]
|
||||
obj = self.db
|
||||
elif isTenant:
|
||||
op = op[:-7]
|
||||
obj = self.tenant if self.tenant else self.db
|
||||
elif isSnapshot:
|
||||
op = op[:-9]
|
||||
obj = self.current_transaction().snapshot
|
||||
else:
|
||||
obj = self.current_transaction()
|
||||
|
||||
inst = Instruction(obj, self.stack, op, idx, isDatabase, isSnapshot)
|
||||
inst = Instruction(obj, self.stack, op, idx, isDatabase, isTenant, isSnapshot)
|
||||
|
||||
try:
|
||||
if inst.op == six.u("PUSH"):
|
||||
|
|
@ -583,6 +591,19 @@ class Tester:
|
|||
prefix = inst.pop()
|
||||
Tester.wait_empty(self.db, prefix)
|
||||
inst.push(b"WAITED_FOR_EMPTY")
|
||||
elif inst.op == six.u("TENANT_CREATE"):
|
||||
name = inst.pop()
|
||||
self.db.allocate_tenant(name)
|
||||
inst.push(b"RESULT_NOT_PRESENT")
|
||||
elif inst.op == six.u("TENANT_DELETE"):
|
||||
name = inst.pop()
|
||||
self.db.delete_tenant(name)
|
||||
inst.push(b"RESULT_NOT_PRESENT")
|
||||
elif inst.op == six.u("TENANT_SET_ACTIVE"):
|
||||
name = inst.pop()
|
||||
self.tenant = self.db.open_tenant(name)
|
||||
elif inst.op == six.u("TENANT_CLEAR_ACTIVE"):
|
||||
self.tenant = None
|
||||
elif inst.op == six.u("UNIT_TESTS"):
|
||||
try:
|
||||
test_db_options(db)
|
||||
|
|
@ -600,6 +621,8 @@ class Tester:
|
|||
test_size_limit_option(db)
|
||||
test_get_approximate_size(db)
|
||||
|
||||
test_tenants(db)
|
||||
|
||||
except fdb.FDBError as e:
|
||||
print("Unit tests failed: %s" % e.description)
|
||||
traceback.print_exc()
|
||||
|
|
|
|||
|
|
@ -3,3 +3,4 @@ setuptools>=20.10.0,<=57.4.0
|
|||
sphinx==1.5.6
|
||||
sphinx-bootstrap-theme==0.4.8
|
||||
docutils==0.16
|
||||
Jinja2==3.0.3
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
.. |database-type| replace:: ``Database``
|
||||
.. |database-class| replace:: :class:`Database`
|
||||
.. |database-auto| replace:: the :func:`@fdb.transactional <transactional>` decorator
|
||||
.. |tenant-type| replace:: FIXME
|
||||
.. |tenant-type| replace:: :class:`Tenant`
|
||||
.. |transaction-class| replace:: :class:`Transaction`
|
||||
.. |get-key-func| replace:: :func:`Transaction.get_key`
|
||||
.. |get-range-func| replace:: :func:`Transaction.get_range`
|
||||
|
|
@ -316,9 +316,29 @@ A |database-blurb1| |database-blurb2|
|
|||
|
||||
Returns a new :class:`Transaction` object. Consider using the :func:`@fdb.transactional <transactional>` decorator to create transactions instead, since it will automatically provide you with appropriate retry behavior.
|
||||
|
||||
.. method:: Database.open_tenant(tenant_name)
|
||||
|
||||
Opens an existing tenant to be used for running transactions and returns it as a :class`Tenant` object.
|
||||
|
||||
The tenant name can be either a byte string or a tuple. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name.
|
||||
|
||||
.. |sync-read| replace:: This read is fully synchronous.
|
||||
.. |sync-write| replace:: This change will be committed immediately, and is fully synchronous.
|
||||
|
||||
.. method:: Database.allocate_tenant(tenant_name):
|
||||
|
||||
Creates a new tenant in the cluster. |sync-write|
|
||||
|
||||
The tenant name can be either a byte string or a tuple and cannot start with the ``\xff`` byte. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name.
|
||||
|
||||
.. method:: Database.delete_tenant(tenant_name):
|
||||
|
||||
Delete a tenant from the cluster. |sync-write|
|
||||
|
||||
The tenant name can be either a byte string or a tuple. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name.
|
||||
|
||||
It is an error to delete a tenant that still has data. To delete a non-empty tenant, first clear all of the keys in the tenant.
|
||||
|
||||
.. method:: Database.get(key)
|
||||
|
||||
Returns the value associated with the specified key in the database (or ``None`` if the key does not exist). |sync-read|
|
||||
|
|
@ -460,6 +480,17 @@ Database options
|
|||
.. method:: Database.options.set_snapshot_ryw_disable()
|
||||
|
||||
|option-db-snapshot-ryw-disable-blurb|
|
||||
|
||||
Tenant objects
|
||||
==============
|
||||
|
||||
.. class:: Tenant
|
||||
|
||||
|tenant-blurb1|
|
||||
|
||||
.. method:: Tenant.create_transaction()
|
||||
|
||||
Returns a new :class:`Transaction` object. Consider using the :func:`@fdb.transactional <transactional>` decorator to create transactions instead, since it will automatically provide you with appropriate retry behavior.
|
||||
|
||||
.. _api-python-transactional-decorator:
|
||||
|
||||
|
|
@ -479,9 +510,9 @@ Transactional decoration
|
|||
|
||||
The ``@fdb.transactional`` decorator makes ``simple_function`` a transactional function. All functions using this decorator must have an argument **named** ``tr``. This specially named argument is passed a transaction that the function can use to do reads and writes.
|
||||
|
||||
A caller of a transactionally decorated function can pass a :class:`Database` instead of a transaction for the ``tr`` parameter. Then a transaction will be created automatically, and automatically committed before returning to the caller. The decorator will retry calling the decorated function until the transaction successfully commits.
|
||||
A caller of a transactionally decorated function can pass a :class:`Database` or :class:`Tenant` instead of a transaction for the ``tr`` parameter. Then a transaction will be created automatically, and automatically committed before returning to the caller. The decorator will retry calling the decorated function until the transaction successfully commits.
|
||||
|
||||
If ``db`` is a :class:`Database`, a call like ::
|
||||
If ``db`` is a :class:`Database` or :class:`Tenant`, a call like ::
|
||||
|
||||
simple_function(db, 'a', 'b')
|
||||
|
||||
|
|
@ -744,7 +775,7 @@ Committing
|
|||
|
||||
.. decorator:: transactional()
|
||||
|
||||
The ``transactional`` decorator makes it easy to write transactional functions which accept either a :class:`Database` or a :class:`Transaction` as a parameter and automatically commit. See :func:`@fdb.transactional <transactional>` for explanation and examples.
|
||||
The ``transactional`` decorator makes it easy to write transactional functions which accept a :class:`Database`, :class`Tenant`, or :class:`Transaction` as a parameter and automatically commit. See :func:`@fdb.transactional <transactional>` for explanation and examples.
|
||||
|
||||
.. method :: Transaction.commit()
|
||||
|
||||
|
|
@ -754,7 +785,7 @@ Committing
|
|||
|
||||
|commit-outstanding-reads-blurb|
|
||||
|
||||
.. note :: Consider using the :func:`@fdb.transactional <transactional>` decorator, which not only calls :meth:`Database.create_transaction` and :meth:`Transaction.commit()` for you but also implements the required error handling and retry logic for transactions.
|
||||
.. note :: Consider using the :func:`@fdb.transactional <transactional>` decorator, which not only calls :meth:`Database.create_transaction` or :meth`Tenant.create_transaction` and :meth:`Transaction.commit()` for you but also implements the required error handling and retry logic for transactions.
|
||||
|
||||
.. warning :: |used-during-commit-blurb|
|
||||
|
||||
|
|
|
|||
|
|
@ -155,6 +155,12 @@ Here is a complete list of valid parameters:
|
|||
|
||||
**Example**: The URL parameter *header=x-amz-storage-class:REDUCED_REDUNDANCY* would send the HTTP header required to use the reduced redundancy storage option in the S3 API.
|
||||
|
||||
Signing Protocol
|
||||
=================
|
||||
|
||||
AWS signature version 4 is the default signing protocol choice. This boolean knob ``--knob_http_request_aws_v4_header`` can be used to select between v4 style and v2 style signatures.
|
||||
If the knob is set to ``true`` then v4 signature will be used and if set to ``false`` then v2 signature will be used.
|
||||
|
||||
.. _blob-credential-files:
|
||||
|
||||
Blob Credential Files
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ enum {
|
|||
OPT_HEX_KEY_PREFIX,
|
||||
OPT_BEGIN_VERSION_FILTER,
|
||||
OPT_END_VERSION_FILTER,
|
||||
OPT_KNOB,
|
||||
OPT_HELP
|
||||
};
|
||||
|
||||
|
|
@ -72,6 +73,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
|
|||
{ OPT_HEX_KEY_PREFIX, "--hex-prefix", SO_REQ_SEP },
|
||||
{ OPT_BEGIN_VERSION_FILTER, "--begin-version-filter", SO_REQ_SEP },
|
||||
{ OPT_END_VERSION_FILTER, "--end-version-filter", SO_REQ_SEP },
|
||||
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
|
||||
{ OPT_HELP, "-?", SO_NONE },
|
||||
{ OPT_HELP, "-h", SO_NONE },
|
||||
{ OPT_HELP, "--help", SO_NONE },
|
||||
|
|
|
|||
|
|
@ -26,17 +26,21 @@
|
|||
#include <vector>
|
||||
|
||||
#include "fdbbackup/BackupTLSConfig.h"
|
||||
#include "fdbclient/BuildFlags.h"
|
||||
#include "fdbbackup/FileConverter.h"
|
||||
#include "fdbclient/BackupAgent.actor.h"
|
||||
#include "fdbclient/BackupContainer.h"
|
||||
#include "fdbbackup/FileConverter.h"
|
||||
#include "fdbclient/CommitTransaction.h"
|
||||
#include "fdbclient/FDBTypes.h"
|
||||
#include "fdbclient/IKnobCollection.h"
|
||||
#include "fdbclient/Knobs.h"
|
||||
#include "fdbclient/MutationList.h"
|
||||
#include "flow/ArgParseUtil.h"
|
||||
#include "flow/IRandom.h"
|
||||
#include "flow/Trace.h"
|
||||
#include "flow/flow.h"
|
||||
#include "flow/serialize.h"
|
||||
#include "fdbclient/BuildFlags.h"
|
||||
|
||||
#include "flow/actorcompiler.h" // has to be last include
|
||||
|
||||
#define SevDecodeInfo SevVerbose
|
||||
|
|
@ -73,11 +77,13 @@ void printDecodeUsage() {
|
|||
" --list-only Print file list and exit.\n"
|
||||
" -k KEY_PREFIX Use the prefix for filtering mutations\n"
|
||||
" --hex-prefix HEX_PREFIX\n"
|
||||
" The prefix specified in HEX format, e.g., \\x05\\x01.\n"
|
||||
" The prefix specified in HEX format, e.g., \"\\\\x05\\\\x01\".\n"
|
||||
" --begin-version-filter BEGIN_VERSION\n"
|
||||
" The version range's begin version (inclusive) for filtering.\n"
|
||||
" --end-version-filter END_VERSION\n"
|
||||
" The version range's end version (exclusive) for filtering.\n"
|
||||
" --knob-KNOBNAME KNOBVALUE\n"
|
||||
" Changes a knob value. KNOBNAME should be lowercase."
|
||||
"\n";
|
||||
return;
|
||||
}
|
||||
|
|
@ -97,6 +103,8 @@ struct DecodeParams {
|
|||
Version beginVersionFilter = 0;
|
||||
Version endVersionFilter = std::numeric_limits<Version>::max();
|
||||
|
||||
std::vector<std::pair<std::string, std::string>> knobs;
|
||||
|
||||
// Returns if [begin, end) overlap with the filter range
|
||||
bool overlap(Version begin, Version end) const {
|
||||
// Filter [100, 200), [50,75) [200, 300)
|
||||
|
|
@ -130,8 +138,39 @@ struct DecodeParams {
|
|||
if (!prefix.empty()) {
|
||||
s.append(", KeyPrefix: ").append(printable(KeyRef(prefix)));
|
||||
}
|
||||
for (const auto& [knob, value] : knobs) {
|
||||
s.append(", KNOB-").append(knob).append(" = ").append(value);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
void updateKnobs() {
|
||||
auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
|
||||
for (const auto& [knobName, knobValueString] : knobs) {
|
||||
try {
|
||||
auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
|
||||
g_knobs.setKnob(knobName, knobValue);
|
||||
} catch (Error& e) {
|
||||
if (e.code() == error_code_invalid_option_value) {
|
||||
std::cerr << "WARNING: Invalid value '" << knobValueString << "' for knob option '" << knobName
|
||||
<< "'\n";
|
||||
TraceEvent(SevWarnAlways, "InvalidKnobValue")
|
||||
.detail("Knob", printable(knobName))
|
||||
.detail("Value", printable(knobValueString));
|
||||
} else {
|
||||
std::cerr << "ERROR: Failed to set knob option '" << knobName << "': " << e.what() << "\n";
|
||||
TraceEvent(SevError, "FailedToSetKnob")
|
||||
.errorUnsuppressed(e)
|
||||
.detail("Knob", printable(knobName))
|
||||
.detail("Value", printable(knobValueString));
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
|
||||
g_knobs.initialize(Randomize::True, IsSimulated::False);
|
||||
}
|
||||
};
|
||||
|
||||
// Decode an ASCII string, e.g., "\x15\x1b\x19\x04\xaf\x0c\x28\x0a",
|
||||
|
|
@ -256,6 +295,16 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
|
|||
param->tlsConfig.blobCredentials.push_back(args->OptionArg());
|
||||
break;
|
||||
|
||||
case OPT_KNOB: {
|
||||
Optional<std::string> knobName = extractPrefixedArgument("--knob", args->OptionSyntax());
|
||||
if (!knobName.present()) {
|
||||
std::cerr << "ERROR: unable to parse knob option '" << args->OptionSyntax() << "'\n";
|
||||
return FDB_EXIT_ERROR;
|
||||
}
|
||||
param->knobs.emplace_back(knobName.get(), args->OptionArg());
|
||||
break;
|
||||
}
|
||||
|
||||
#ifndef TLS_DISABLED
|
||||
case TLSConfig::OPT_TLS_PLUGIN:
|
||||
args->OptionArg();
|
||||
|
|
@ -552,6 +601,9 @@ int main(int argc, char** argv) {
|
|||
StringRef url(param.container_url);
|
||||
setupNetwork(0, UseMetrics::True);
|
||||
|
||||
// Must be called after setupNetwork() to be effective
|
||||
param.updateKnobs();
|
||||
|
||||
TraceEvent::setNetworkThread();
|
||||
openTraceFile(NetworkAddress(), 10 << 20, 500 << 20, param.log_dir, "decode", param.trace_log_group);
|
||||
param.tlsConfig.setupBlobCredentials();
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
|
||||
#include "fdbclient/json_spirit/json_spirit_writer_template.h"
|
||||
#include "fdbclient/json_spirit/json_spirit_reader_template.h"
|
||||
#include "flow/Error.h"
|
||||
|
||||
// JSONDoc is a convenient reader/writer class for manipulating JSON documents using "paths".
|
||||
// Access is done using a "path", which is a string of dot-separated
|
||||
|
|
|
|||
|
|
@ -573,7 +573,7 @@ ACTOR Future<Void> updateSecret_impl(Reference<S3BlobStoreEndpoint> b) {
|
|||
JSONDoc accounts(doc.last().get_obj());
|
||||
if (accounts.has(credentialsFileKey, false) && accounts.last().type() == json_spirit::obj_type) {
|
||||
JSONDoc account(accounts.last());
|
||||
S3BlobStoreEndpoint::Credentials creds;
|
||||
S3BlobStoreEndpoint::Credentials creds = b->credentials.get();
|
||||
if (b->lookupKey) {
|
||||
std::string apiKey;
|
||||
if (account.tryGet("api_key", apiKey))
|
||||
|
|
|
|||
Loading…
Reference in New Issue