Merge branch 'main' of github.com:apple/foundationdb into jfu-grv-cache-multi-threaded
This commit is contained in:
Jon Fu
commit
4677b59fd8
|
|
@ -202,7 +202,7 @@ class TestRunner(object):
|
|||
self.args.types = list(reduce(lambda t1, t2: filter(t1.__contains__, t2), map(lambda tester: tester.types, self.testers)))
|
||||
|
||||
self.args.no_directory_snapshot_ops = self.args.no_directory_snapshot_ops or any([not tester.directory_snapshot_ops_enabled for tester in self.testers])
|
||||
self.args.no_tenants = self.args.no_tenants or any([not tester.tenants_enabled for tester in self.testers])
|
||||
self.args.no_tenants = self.args.no_tenants or any([not tester.tenants_enabled for tester in self.testers]) or self.args.api_version < 710
|
||||
|
||||
def print_test(self):
|
||||
test_instructions = self._generate_test()
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ set(SRCS
|
|||
fdb/locality.py
|
||||
fdb/six.py
|
||||
fdb/subspace_impl.py
|
||||
fdb/tenant_management.py
|
||||
fdb/tuple.py
|
||||
README.rst
|
||||
MANIFEST.in)
|
||||
|
|
|
|||
|
|
@ -100,6 +100,9 @@ def api_version(ver):
|
|||
|
||||
_add_symbols(fdb.impl, list)
|
||||
|
||||
if ver >= 710:
|
||||
import fdb.tenant_management
|
||||
|
||||
if ver < 610:
|
||||
globals()["init"] = getattr(fdb.impl, "init")
|
||||
globals()["open"] = getattr(fdb.impl, "open_v609")
|
||||
|
|
|
|||
|
|
@ -1178,52 +1178,6 @@ class Database(_TransactionCreator):
|
|||
self.capi.fdb_database_create_transaction(self.dpointer, ctypes.byref(pointer))
|
||||
return Transaction(pointer.value, self)
|
||||
|
||||
def allocate_tenant(self, name):
|
||||
Database.__database_allocate_tenant(self, process_tenant_name(name), [])
|
||||
|
||||
def delete_tenant(self, name):
|
||||
Database.__database_delete_tenant(self, process_tenant_name(name), [])
|
||||
|
||||
# Attempt to allocate a tenant in the cluster. If the tenant already exists,
|
||||
# this function will return a tenant_already_exists error. If the tenant is created
|
||||
# concurrently, then this function may return success even if another caller creates
|
||||
# it.
|
||||
#
|
||||
# The existence_check_marker is expected to be an empty list. This function will
|
||||
# modify the list after completing the existence check to avoid checking for existence
|
||||
# on retries. This allows the operation to be idempotent.
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_allocate_tenant(tr, name, existence_check_marker):
|
||||
tr.options.set_special_key_space_enable_writes()
|
||||
key = b'\xff\xff/management/tenant_map/%s' % name
|
||||
if not existence_check_marker:
|
||||
existing_tenant = tr[key].wait()
|
||||
existence_check_marker.append(None)
|
||||
if existing_tenant != None:
|
||||
raise fdb.FDBError(2132) # tenant_already_exists
|
||||
tr[key] = b''
|
||||
|
||||
# Attempt to remove a tenant in the cluster. If the tenant doesn't exist, this
|
||||
# function will return a tenant_not_found error. If the tenant is deleted
|
||||
# concurrently, then this function may return success even if another caller deletes
|
||||
# it.
|
||||
#
|
||||
# The existence_check_marker is expected to be an empty list. This function will
|
||||
# modify the list after completing the existence check to avoid checking for existence
|
||||
# on retries. This allows the operation to be idempotent.
|
||||
@staticmethod
|
||||
@transactional
|
||||
def __database_delete_tenant(tr, name, existence_check_marker):
|
||||
tr.options.set_special_key_space_enable_writes()
|
||||
key = b'\xff\xff/management/tenant_map/%s' % name
|
||||
if not existence_check_marker:
|
||||
existing_tenant = tr[key].wait()
|
||||
existence_check_marker.append(None)
|
||||
if existing_tenant == None:
|
||||
raise fdb.FDBError(2131) # tenant_not_found
|
||||
del tr[key]
|
||||
|
||||
|
||||
class Tenant(_TransactionCreator):
|
||||
def __init__(self, tpointer):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
#
|
||||
# tenant_management.py
|
||||
#
|
||||
# This source file is part of the FoundationDB open source project
|
||||
#
|
||||
# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# FoundationDB Python API
|
||||
|
||||
"""Documentation for this API can be found at
|
||||
https://apple.github.io/foundationdb/api-python.html"""
|
||||
|
||||
from fdb import impl as _impl
|
||||
|
||||
_tenant_map_prefix = b'\xff\xff/management/tenant_map/'
|
||||
|
||||
# If the existence_check_marker is an empty list, then check whether the tenant exists.
|
||||
# After the check, append an item to the existence_check_marker list so that subsequent
|
||||
# calls to this function will not perform the existence check.
|
||||
#
|
||||
# If the existence_check_marker is a non-empty list, return None.
|
||||
def _check_tenant_existence(tr, key, existence_check_marker, force_maybe_commited):
|
||||
if not existence_check_marker:
|
||||
existing_tenant = tr[key].wait()
|
||||
existence_check_marker.append(None)
|
||||
if force_maybe_commited:
|
||||
raise _impl.FDBError(1021) # maybe_committed
|
||||
return existing_tenant != None
|
||||
|
||||
return None
|
||||
|
||||
# Attempt to create a tenant in the cluster. If existence_check_marker is an empty
|
||||
# list, then this function will check if the tenant already exists and fail if it does.
|
||||
# Once the existence check is completed, it will not be done again if this function
|
||||
# retries. As a result, this function may return successfully if the tenant is created
|
||||
# by someone else concurrently. This behavior allows the operation to be idempotent with
|
||||
# respect to retries.
|
||||
#
|
||||
# If the existence_check_marker is a non-empty list, then the existence check is skipped.
|
||||
@_impl.transactional
|
||||
def _create_tenant_impl(tr, tenant_name, existence_check_marker, force_existence_check_maybe_committed=False):
|
||||
tr.options.set_special_key_space_enable_writes()
|
||||
key = b'%s%s' % (_tenant_map_prefix, tenant_name)
|
||||
|
||||
if _check_tenant_existence(tr, key, existence_check_marker, force_existence_check_maybe_committed) is True:
|
||||
raise _impl.FDBError(2132) # tenant_already_exists
|
||||
|
||||
tr[key] = b''
|
||||
|
||||
# Attempt to delete a tenant from the cluster. If existence_check_marker is an empty
|
||||
# list, then this function will check if the tenant already exists and fail if it does
|
||||
# not. Once the existence check is completed, it will not be done again if this function
|
||||
# retries. As a result, this function may return successfully if the tenant is deleted
|
||||
# by someone else concurrently. This behavior allows the operation to be idempotent with
|
||||
# respect to retries.
|
||||
#
|
||||
# If the existence_check_marker is a non-empty list, then the existence check is skipped.
|
||||
@_impl.transactional
|
||||
def _delete_tenant_impl(tr, tenant_name, existence_check_marker, force_existence_check_maybe_committed=False):
|
||||
tr.options.set_special_key_space_enable_writes()
|
||||
key = b'%s%s' % (_tenant_map_prefix, tenant_name)
|
||||
|
||||
if _check_tenant_existence(tr, key, existence_check_marker, force_existence_check_maybe_committed) is False:
|
||||
raise _impl.FDBError(2131) # tenant_not_found
|
||||
|
||||
del tr[key]
|
||||
|
||||
def create_tenant(db_or_tr, tenant_name):
|
||||
tenant_name = _impl.process_tenant_name(tenant_name)
|
||||
|
||||
# Only perform the existence check when run using a database
|
||||
# Callers using a transaction are expected to check existence themselves if required
|
||||
existence_check_marker = [] if not isinstance(db_or_tr, _impl.TransactionRead) else [None]
|
||||
_create_tenant_impl(db_or_tr, tenant_name, existence_check_marker)
|
||||
|
||||
def delete_tenant(db_or_tr, tenant_name):
|
||||
tenant_name = _impl.process_tenant_name(tenant_name)
|
||||
|
||||
# Only perform the existence check when run using a database
|
||||
# Callers using a transaction are expected to check existence themselves if required
|
||||
existence_check_marker = [] if not isinstance(db_or_tr, _impl.TransactionRead) else [None]
|
||||
_delete_tenant_impl(db_or_tr, tenant_name, existence_check_marker)
|
||||
|
|
@ -26,9 +26,22 @@ from fdb.tuple import pack
|
|||
if __name__ == '__main__':
|
||||
fdb.api_version(710)
|
||||
|
||||
def cleanup_tenant(db, tenant_name):
|
||||
try:
|
||||
tenant = db.open_tenant(tenant_name)
|
||||
del tenant[:]
|
||||
fdb.tenant_management.delete_tenant(db, tenant_name)
|
||||
except fdb.FDBError as e:
|
||||
if e.code == 2131: # tenant not found
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
def test_tenant_tuple_name(db):
|
||||
tuplename=(b'test', b'level', b'hierarchy', 3, 1.24, 'str')
|
||||
db.allocate_tenant(tuplename)
|
||||
cleanup_tenant(db, tuplename)
|
||||
|
||||
fdb.tenant_management.create_tenant(db, tuplename)
|
||||
|
||||
tenant=db.open_tenant(tuplename)
|
||||
tenant[b'foo'] = b'bar'
|
||||
|
|
@ -36,25 +49,15 @@ def test_tenant_tuple_name(db):
|
|||
assert tenant[b'foo'] == b'bar'
|
||||
|
||||
del tenant[b'foo']
|
||||
db.delete_tenant(tuplename)
|
||||
fdb.tenant_management.delete_tenant(db, tuplename)
|
||||
|
||||
def cleanup_tenant(db, tenant_name):
|
||||
try:
|
||||
tenant = db.open_tenant(tenant_name)
|
||||
del tenant[:]
|
||||
db.delete_tenant(tenant_name)
|
||||
except fdb.FDBError as e:
|
||||
if e.code == 2131: # tenant not found
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
def test_tenant_operations(db):
|
||||
cleanup_tenant(db, b'tenant1')
|
||||
cleanup_tenant(db, b'tenant2')
|
||||
|
||||
db.allocate_tenant(b'tenant1')
|
||||
db.allocate_tenant(b'tenant2')
|
||||
fdb.tenant_management.create_tenant(db, b'tenant1')
|
||||
fdb.tenant_management.create_tenant(db, b'tenant2')
|
||||
|
||||
tenant1 = db.open_tenant(b'tenant1')
|
||||
tenant2 = db.open_tenant(b'tenant2')
|
||||
|
|
@ -90,7 +93,7 @@ def test_tenant_operations(db):
|
|||
assert db[prefix2 + b'tenant_test_key'] == b'tenant2'
|
||||
assert db[b'tenant_test_key'] == b'no_tenant'
|
||||
|
||||
db.delete_tenant(b'tenant1')
|
||||
fdb.tenant_management.delete_tenant(db, b'tenant1')
|
||||
try:
|
||||
tenant1[b'tenant_test_key']
|
||||
assert False
|
||||
|
|
@ -98,7 +101,7 @@ def test_tenant_operations(db):
|
|||
assert e.code == 2131 # tenant not found
|
||||
|
||||
del tenant2[:]
|
||||
db.delete_tenant(b'tenant2')
|
||||
fdb.tenant_management.delete_tenant(db, b'tenant2')
|
||||
|
||||
assert db[prefix1 + b'tenant_test_key'] == None
|
||||
assert db[prefix2 + b'tenant_test_key'] == None
|
||||
|
|
@ -108,9 +111,70 @@ def test_tenant_operations(db):
|
|||
|
||||
assert db[b'tenant_test_key'] == None
|
||||
|
||||
def test_tenant_operation_retries(db):
|
||||
cleanup_tenant(db, b'tenant1')
|
||||
cleanup_tenant(db, b'tenant2')
|
||||
|
||||
# Test that the tenant creation only performs the existence check once
|
||||
fdb.tenant_management._create_tenant_impl(db, b'tenant1', [], force_existence_check_maybe_committed=True)
|
||||
|
||||
# An attempt to create the tenant again should fail
|
||||
try:
|
||||
fdb.tenant_management.create_tenant(db, b'tenant1')
|
||||
assert False
|
||||
except fdb.FDBError as e:
|
||||
assert e.code == 2132 # tenant already exists
|
||||
|
||||
# Using a transaction skips the existence check
|
||||
tr = db.create_transaction()
|
||||
fdb.tenant_management.create_tenant(tr, b'tenant1')
|
||||
|
||||
# Test that a concurrent tenant creation doesn't interfere with the existence check logic
|
||||
tr = db.create_transaction()
|
||||
existence_check_marker = []
|
||||
fdb.tenant_management._create_tenant_impl(tr, b'tenant2', existence_check_marker)
|
||||
|
||||
fdb.tenant_management.create_tenant(db, b'tenant2')
|
||||
|
||||
tr = db.create_transaction()
|
||||
try:
|
||||
fdb.tenant_management._create_tenant_impl(tr, b'tenant2', existence_check_marker)
|
||||
tr.commit().wait()
|
||||
except fdb.FDBError as e:
|
||||
tr.on_error(e).wait()
|
||||
|
||||
# Test that tenant deletion only performs the existence check once
|
||||
fdb.tenant_management._delete_tenant_impl(db, b'tenant1', [], force_existence_check_maybe_committed=True)
|
||||
|
||||
# An attempt to delete the tenant again should fail
|
||||
try:
|
||||
fdb.tenant_management.delete_tenant(db, b'tenant1')
|
||||
assert False
|
||||
except fdb.FDBError as e:
|
||||
assert e.code == 2131 # tenant not found
|
||||
|
||||
# Using a transaction skips the existence check
|
||||
tr = db.create_transaction()
|
||||
fdb.tenant_management.delete_tenant(tr, b'tenant1')
|
||||
|
||||
# Test that a concurrent tenant deletion doesn't interfere with the existence check logic
|
||||
tr = db.create_transaction()
|
||||
existence_check_marker = []
|
||||
fdb.tenant_management._delete_tenant_impl(tr, b'tenant2', existence_check_marker)
|
||||
|
||||
fdb.tenant_management.delete_tenant(db, b'tenant2')
|
||||
|
||||
tr = db.create_transaction()
|
||||
try:
|
||||
fdb.tenant_management._delete_tenant_impl(tr, b'tenant2', existence_check_marker)
|
||||
tr.commit().wait()
|
||||
except fdb.FDBError as e:
|
||||
tr.on_error(e).wait()
|
||||
|
||||
def test_tenants(db):
|
||||
test_tenant_tuple_name(db)
|
||||
test_tenant_operations(db)
|
||||
test_tenant_operation_retries(db)
|
||||
|
||||
# Expect a cluster file as input. This test will write to the FDB cluster, so
|
||||
# be aware of potential side effects.
|
||||
|
|
|
|||
|
|
@ -593,11 +593,11 @@ class Tester:
|
|||
inst.push(b"WAITED_FOR_EMPTY")
|
||||
elif inst.op == six.u("TENANT_CREATE"):
|
||||
name = inst.pop()
|
||||
self.db.allocate_tenant(name)
|
||||
fdb.tenant_management.create_tenant(self.db, name)
|
||||
inst.push(b"RESULT_NOT_PRESENT")
|
||||
elif inst.op == six.u("TENANT_DELETE"):
|
||||
name = inst.pop()
|
||||
self.db.delete_tenant(name)
|
||||
fdb.tenant_management.delete_tenant(self.db, name)
|
||||
inst.push(b"RESULT_NOT_PRESENT")
|
||||
elif inst.op == six.u("TENANT_SET_ACTIVE"):
|
||||
name = inst.pop()
|
||||
|
|
@ -621,7 +621,8 @@ class Tester:
|
|||
test_size_limit_option(db)
|
||||
test_get_approximate_size(db)
|
||||
|
||||
test_tenants(db)
|
||||
if fdb.get_api_version() >= 710:
|
||||
test_tenants(db)
|
||||
|
||||
except fdb.FDBError as e:
|
||||
print("Unit tests failed: %s" % e.description)
|
||||
|
|
|
|||
|
|
@ -325,20 +325,6 @@ A |database-blurb1| |database-blurb2|
|
|||
.. |sync-read| replace:: This read is fully synchronous.
|
||||
.. |sync-write| replace:: This change will be committed immediately, and is fully synchronous.
|
||||
|
||||
.. method:: Database.allocate_tenant(tenant_name):
|
||||
|
||||
Creates a new tenant in the cluster. |sync-write|
|
||||
|
||||
The tenant name can be either a byte string or a tuple and cannot start with the ``\xff`` byte. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name.
|
||||
|
||||
.. method:: Database.delete_tenant(tenant_name):
|
||||
|
||||
Delete a tenant from the cluster. |sync-write|
|
||||
|
||||
The tenant name can be either a byte string or a tuple. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name.
|
||||
|
||||
It is an error to delete a tenant that still has data. To delete a non-empty tenant, first clear all of the keys in the tenant.
|
||||
|
||||
.. method:: Database.get(key)
|
||||
|
||||
Returns the value associated with the specified key in the database (or ``None`` if the key does not exist). |sync-read|
|
||||
|
|
@ -1590,3 +1576,32 @@ Locality information
|
|||
.. method:: fdb.locality.get_addresses_for_key(tr, key)
|
||||
|
||||
Returns a :class:`fdb.FutureStringArray`. You must call the :meth:`fdb.Future.wait()` method on this object to retrieve a list of public network addresses as strings, one for each of the storage servers responsible for storing ``key`` and its associated value.
|
||||
|
||||
Tenant management
|
||||
=================
|
||||
|
||||
.. module:: fdb.tenant_management
|
||||
|
||||
The FoundationDB API includes functions to manage the set of tenants in a cluster.
|
||||
|
||||
.. method:: fdb.tenant_management.create_tenant(db_or_tr, tenant_name)
|
||||
|
||||
Creates a new tenant in the cluster.
|
||||
|
||||
The tenant name can be either a byte string or a tuple and cannot start with the ``\xff`` byte. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name.
|
||||
|
||||
If a database is provided to this function for the ``db_or_tr`` parameter, then this function will first check if the tenant already exists. If it does, it will fail with a ``tenant_already_exists`` error. Otherwise, it will create a transaction and attempt to create the tenant in a retry loop. If the tenant is created concurrently by another transaction, this function may still return successfully.
|
||||
|
||||
If a transaction is provided to this function for the ``db_or_tr`` parameter, then this function will not check if the tenant already exists. It is up to the user to perform that check if required. The user must also successfully commit the transaction in order for the creation to take effect.
|
||||
|
||||
.. method:: fdb.tenant_management.delete_tenant(db_or_tr, tenant_name)
|
||||
|
||||
Delete a tenant from the cluster.
|
||||
|
||||
The tenant name can be either a byte string or a tuple. If a tuple is provided, the tuple will be packed using the tuple layer to generate the byte string tenant name.
|
||||
|
||||
It is an error to delete a tenant that still has data. To delete a non-empty tenant, first clear all of the keys in the tenant.
|
||||
|
||||
If a database is provided to this function for the ``db_or_tr`` parameter, then this function will first check if the tenant already exists. If it does not, it will fail with a ``tenant_not_found`` error. Otherwise, it will create a transaction and attempt to delete the tenant in a retry loop. If the tenant is deleted concurrently by another transaction, this function may still return successfully.
|
||||
|
||||
If a transaction is provided to this function for the ``db_or_tr`` parameter, then this function will not check if the tenant already exists. It is up to the user to perform that check if required. The user must also successfully commit the transaction in order for the deletion to take effect.
|
||||
|
|
|
|||
Loading…
Reference in New Issue