780 lines
26 KiB
Python
Executable File
780 lines
26 KiB
Python
Executable File
#!/usr/bin/python
|
|
#
|
|
# python_correctness.py
|
|
#
|
|
# This source file is part of the FoundationDB open source project
|
|
#
|
|
# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
import string
|
|
import random
|
|
import traceback
|
|
|
|
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
|
|
from python_tests import PythonTest
|
|
|
|
import fdb
|
|
import fdb.tuple
|
|
fdb.api_version(400)
|
|
|
|
|
|
# A class that mimics some of the operations of the FoundationDB key-value store
|
|
class KeyValueStore():
|
|
|
|
# Uses a simple dictionary to store key-value pairs
|
|
# Any operations that depend on the order of keys first sort the data
|
|
store = dict()
|
|
|
|
def get(self, key):
|
|
if key is fdb.KeySelector:
|
|
key = get_key(key)
|
|
|
|
if key in self.store:
|
|
return self.store[key]
|
|
else:
|
|
return None
|
|
|
|
def get_key(self, keySelector):
|
|
sortedKeys = list(sorted(self.store.keys()))
|
|
for index, key in enumerate(sortedKeys):
|
|
if key >= keySelector.key:
|
|
index += keySelector.offset
|
|
if (key == keySelector.key and not keySelector.or_equal) or key != keySelector.key:
|
|
index -= 1
|
|
|
|
if index < 0 or index >= len(sortedKeys):
|
|
return ''
|
|
else:
|
|
return sortedKeys[index]
|
|
|
|
index = len(sortedKeys) + keySelector.offset - 1
|
|
if index < 0 or index >= len(sortedKeys):
|
|
return ''
|
|
else:
|
|
return sortedKeys[index]
|
|
|
|
def get_range(self, begin, end, limit=None):
|
|
values = []
|
|
count = 0
|
|
|
|
if begin is fdb.KeySelector:
|
|
begin = get_key(begin)
|
|
if end is fdb.KeySelector:
|
|
end = get_key(end)
|
|
|
|
for key, value in sorted(self.store.iteritems()):
|
|
if limit is not None and count >= limit:
|
|
break
|
|
|
|
if key >= end:
|
|
break
|
|
|
|
if key >= begin:
|
|
values.append([key, value])
|
|
count += 1
|
|
|
|
return values
|
|
|
|
def get_range_startswith(self, prefix, limit=None):
|
|
values = []
|
|
count = 0
|
|
for key, value in sorted(self.store.iteritems()):
|
|
if limit is not None and count >= limit:
|
|
break
|
|
|
|
if key > prefix and not key.startswith(prefix):
|
|
break
|
|
|
|
if key.startswith(prefix):
|
|
values.append([key, value])
|
|
count += 1
|
|
|
|
return values
|
|
|
|
def set(self, key, value):
|
|
self.store[key] = value
|
|
|
|
def clear(self, key):
|
|
if key is fdb.KeySelector:
|
|
key = get_key(key)
|
|
|
|
if key in self.store:
|
|
del self.store[key]
|
|
|
|
def clear_range(self, begin, end):
|
|
for key, value in self.get_range(begin, end):
|
|
del self.store[key]
|
|
|
|
def clear_range_startswith(self, prefix):
|
|
for key, value in self.get_range_startswith(prefix):
|
|
del self.store[key]
|
|
|
|
|
|
class PythonCorrectness(PythonTest):
|
|
callback = False
|
|
callbackError = ''
|
|
|
|
# Python correctness tests (checks if functions run and yield correct results)
|
|
def run_test(self):
|
|
try:
|
|
db = fdb.open(None, 'DB')
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('fdb.open failed'))
|
|
return
|
|
|
|
try:
|
|
print('Testing functions...')
|
|
self.testFunctions(db)
|
|
|
|
print('Testing correctness...')
|
|
del db[:]
|
|
self.testCorrectness(db)
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Failed to complete all tests'))
|
|
|
|
# Generates a random set of keys and values
|
|
def generateData(self, numKeys, minKeyLength, maxKeyLength, minValueLength, maxValueLength, prefix='', allowDuplicates=True):
|
|
data = list()
|
|
keys = set()
|
|
while len(data) < numKeys:
|
|
# key = prefix + ''.join(random.choice(string.ascii_lowercase)
|
|
# for i in range(0, random.randint(minKeyLength - len(prefix), maxKeyLength - len(prefix))))
|
|
key = prefix + ''.join(chr(random.randint(0, 254))
|
|
for i in range(0, random.randint(minKeyLength - len(prefix), maxKeyLength - len(prefix))))
|
|
if not allowDuplicates:
|
|
if key in keys:
|
|
continue
|
|
else:
|
|
keys.add(key)
|
|
|
|
value = ''.join('x' for i in range(0, random.randint(maxKeyLength, maxValueLength)))
|
|
data.append([key, value])
|
|
|
|
return data
|
|
|
|
# Function to test the callback feature of Future objects
|
|
def testCallback(self, future):
|
|
try:
|
|
future.wait()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.callbackError = getError('Callback future get failed')
|
|
|
|
self.callback = True
|
|
|
|
# Tests that all of the functions in the python API can be called without failing
|
|
def testFunctions(self, db):
|
|
self.callback = False
|
|
self.callbackError = ''
|
|
|
|
try:
|
|
tr = db.create_transaction()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('db.create_transaction failed'))
|
|
return
|
|
|
|
try:
|
|
tr['testkey'] = 'testvalue'
|
|
value = tr['testkey']
|
|
value.is_ready()
|
|
value.block_until_ready()
|
|
value.wait()
|
|
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Set/Get value failed (block until ready)'))
|
|
|
|
try:
|
|
value = tr['testkey']
|
|
value.wait()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Get value failed'))
|
|
|
|
try:
|
|
tr['testkey'] = 'newtestvalue'
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Replace value failed'))
|
|
|
|
try:
|
|
value = tr['fakekey']
|
|
# The following line would generate a segfault
|
|
# value.capi.fdb_future_block_until_ready(0)
|
|
value.wait()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Get non-existent key failed'))
|
|
|
|
try:
|
|
tr.commit().wait()
|
|
tr.get_committed_version()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Commit failed'))
|
|
|
|
try:
|
|
tr.reset()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Reset failed'))
|
|
|
|
try:
|
|
version = tr.get_read_version()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('tr.get_read_version failed'))
|
|
|
|
try:
|
|
value = tr['testkey']
|
|
value.wait()
|
|
tr.reset()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Get and reset failed'))
|
|
|
|
try:
|
|
tr.set_read_version(version.wait())
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Set read version failed'))
|
|
|
|
try:
|
|
value = tr['testkey']
|
|
callbackTime = time.time()
|
|
value.on_ready(self.testCallback)
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Get future and set callback failed'))
|
|
|
|
try:
|
|
del tr['testkey']
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Delete key failed'))
|
|
|
|
try:
|
|
del tr['fakekey']
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Delete non-existent key failed'))
|
|
|
|
try:
|
|
tr.set('testkey', 'testvalue')
|
|
value = tr.get('testkey')
|
|
value.wait()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Future.get failed'))
|
|
|
|
try:
|
|
tr.clear('testkey')
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Clear key failed'))
|
|
|
|
try:
|
|
tr['testkey1'] = 'testvalue1'
|
|
tr['testkey2'] = 'testvalue2'
|
|
tr['testkey3'] = 'testvalue3'
|
|
|
|
for k, v in tr.get_range('testkey1', 'testkey3'):
|
|
v += ''
|
|
|
|
for k, v in tr.get_range('testkey1', 'testkey2', 2):
|
|
v += ''
|
|
|
|
for k, v in tr['testkey1':'testkey3']:
|
|
v += ''
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Get range failed'))
|
|
|
|
try:
|
|
tr['otherkey1'] = 'othervalue1'
|
|
tr['otherkey2'] = 'othervalue2'
|
|
|
|
for k, v in tr.get_range_startswith('testkey'):
|
|
v += ''
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Get range starts with failed'))
|
|
|
|
try:
|
|
tr.clear_range_startswith('otherkey')
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Clear range starts with failed'))
|
|
|
|
try:
|
|
tr.clear_range('testkey1', 'testkey3')
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Clear range failed'))
|
|
|
|
try:
|
|
tr['testkey1'] = 'testvalue1'
|
|
tr['testkey2'] = 'testvalue2'
|
|
tr['testkey3'] = 'testvalue3'
|
|
|
|
begin = fdb.KeySelector('testkey2', 0, 0)
|
|
end = fdb.KeySelector('testkey2', 0, 1)
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Create key selector failed'))
|
|
|
|
try:
|
|
for k, v in tr.get_range(begin, end):
|
|
v += ''
|
|
|
|
for k, v in tr.get_range(begin, end, 2):
|
|
v += ''
|
|
|
|
for k, v in tr[begin:end]:
|
|
v += ''
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Get range (key selectors) failed'))
|
|
|
|
try:
|
|
tr.clear_range(begin, end)
|
|
|
|
tr['testkey1'] = 'testvalue1'
|
|
tr['testkey2'] = 'testvalue2'
|
|
tr['testkey3'] = 'testvalue3'
|
|
|
|
del tr[begin:end]
|
|
|
|
tr['testkey1'] = 'testvalue1'
|
|
tr['testkey2'] = 'testvalue2'
|
|
tr['testkey3'] = 'testvalue3'
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Clear range (key selectors) failed'))
|
|
|
|
try:
|
|
begin = fdb.KeySelector.last_less_than('testkey2')
|
|
end = fdb.KeySelector.first_greater_or_equal('testkey2')
|
|
|
|
for k, v in tr.get_range(begin, end):
|
|
v += ''
|
|
|
|
begin = fdb.KeySelector.last_less_or_equal('testkey2')
|
|
end = fdb.KeySelector.first_greater_than('testkey2')
|
|
|
|
for k, v in tr.get_range(begin, end):
|
|
v += ''
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Builtin key selectors failed'))
|
|
|
|
try:
|
|
del tr['testkey1':'testkey3']
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Delete key range failed'))
|
|
|
|
try:
|
|
tr.commit().wait()
|
|
tr.get_committed_version()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Commit failed'))
|
|
|
|
try:
|
|
key = fdb.tuple.pack(('k1', 'k2', 'k3'))
|
|
kTuple = fdb.tuple.unpack(key)
|
|
if kTuple[0] != 'k1' and kTuple[1] != 'k2' and kTuple[2] != 'k3':
|
|
self.result.add_error('Tuple <-> key conversion yielded incorrect results')
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Tuple <-> key conversion failed'))
|
|
|
|
try:
|
|
tr[fdb.tuple.pack(('k1', 'k2'))] = 'v'
|
|
tr[fdb.tuple.pack(('k1', 'k2', 'k3'))] = 'v1'
|
|
tr[fdb.tuple.pack(('k1', 'k2', 'k3', 'k4'))] = 'v2'
|
|
|
|
for k, v in tr[fdb.tuple.range(('k1', 'k2'))]:
|
|
v += ''
|
|
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Tuple get range failed'))
|
|
|
|
try:
|
|
tr['testint'] = '10'
|
|
y = int(tr['testint']) + 1
|
|
if y != 11:
|
|
self.result.add_error('Value retrieval yielded incorrect results')
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(getError('Future value retrieval failed'))
|
|
|
|
if not self.callback:
|
|
time.sleep(5)
|
|
if not self.callback:
|
|
self.result.add_error('Warning: Future callback not called after %f seconds' % (time.time() - callbackTime))
|
|
if len(self.callbackError) > 0:
|
|
self.result.add_error(self.callbackError)
|
|
|
|
# Compares a FoundationDB database with an in-memory key-value store
|
|
def compareDatabaseToMemory(self, db, store):
|
|
dbResult = self.correctnessGetRangeTransactional(db, '\x00', '\xff')
|
|
storeResult = store.get_range('\x00', '\xff')
|
|
|
|
return self.compareResults(dbResult, storeResult)
|
|
|
|
# Compares result sets coming from a FoundationDB database and an in-memory key-value store
|
|
def compareResults(self, dbResults, storeResults):
|
|
if len(dbResults) != len(storeResults):
|
|
# print 'mismatched lengths: ' + str(len(dbResults)) + ' - ' + str(len(storeResults))
|
|
return False
|
|
|
|
for i in range(0, len(dbResults)):
|
|
# if i >= len(storeResults):
|
|
# print 'mismatched key: ' + dbResults[i].key
|
|
# return False
|
|
if dbResults[i].key != storeResults[i][0] or dbResults[i].value != storeResults[i][1]:
|
|
# print 'mismatched key: ' + dbResults[i].key + ' - ' + storeResults[i][0]
|
|
return False
|
|
|
|
return True
|
|
|
|
# Performs the same operations on a FoundationDB database and an in-memory key-value store and compares the results
|
|
def testCorrectness(self, db):
|
|
numKeys = 5000
|
|
ratioShortKeys = 0.5
|
|
minShortKeyLength = 1
|
|
maxShortKeyLength = 3
|
|
minLongKeyLength = 1
|
|
maxLongKeyLength = 128
|
|
minValueLength = 1
|
|
maxValueLength = 10000
|
|
maxTransactionBytes = 5000000
|
|
|
|
numReads = 100
|
|
numRangeReads = 100
|
|
numPrefixReads = 100
|
|
numGetKeys = 100
|
|
|
|
numClears = 100
|
|
numRangeClears = 10
|
|
numPrefixClears = 10
|
|
|
|
maxKeysPerTransaction = max(1, maxTransactionBytes / (maxValueLength + maxLongKeyLength))
|
|
|
|
try:
|
|
store = KeyValueStore()
|
|
|
|
# Generate some random data
|
|
data = self.generateData(numKeys * ratioShortKeys, minShortKeyLength, maxShortKeyLength, minValueLength, maxValueLength)
|
|
data.extend(self.generateData(numKeys * (1 - ratioShortKeys), minLongKeyLength, maxLongKeyLength, minValueLength, maxValueLength))
|
|
|
|
# Insert the data
|
|
self.correctnessSet(db, store, data, maxKeysPerTransaction)
|
|
if not self.compareDatabaseToMemory(db, store):
|
|
self.result.add_error('transaction.set resulted in incorrect database')
|
|
|
|
# Compare the results of single key reads
|
|
if not self.correctnessGet(db, store, data, numReads, maxKeysPerTransaction):
|
|
self.result.add_error('transaction.get returned incorrect result')
|
|
|
|
# Compare the results of range reads
|
|
for i in range(0, numRangeReads):
|
|
if not self.correctnessGetRange(db, store, data):
|
|
self.result.add_error('transaction.get_range returned incorrect results')
|
|
break
|
|
|
|
# Compare the results of prefix reads
|
|
for i in range(0, numPrefixReads):
|
|
if not self.correctnessGetPrefix(db, store, data):
|
|
self.result.add_error('transaction.get_range_startswith returned incorrect results')
|
|
break
|
|
|
|
# Compare the results of get key
|
|
if not self.correctnessGetKey(db, store, data, numGetKeys, maxKeysPerTransaction):
|
|
self.result.add_error('transaction.get_key returned incorrect results')
|
|
|
|
# Compare the results of clear
|
|
clearedKeys = self.correctnessClear(db, store, data, numClears, maxKeysPerTransaction)
|
|
if not self.compareDatabaseToMemory(db, store):
|
|
self.result.add_error('transaction.clear resulted in incorrect database')
|
|
|
|
# for key in clearedKeys:
|
|
# print 'clearing key ' + key
|
|
# else:
|
|
# print 'successful compare'
|
|
|
|
# Fill the database back up with data
|
|
self.correctnessSet(db, store, data, maxKeysPerTransaction)
|
|
if not self.compareDatabaseToMemory(db, store):
|
|
self.result.add_error('transaction.set resulted in incorrect database')
|
|
|
|
# Compare the results of clear_range
|
|
for i in range(0, numRangeClears):
|
|
self.correctnessClearRange(db, store, data)
|
|
|
|
success = self.compareDatabaseToMemory(db, store)
|
|
if not success:
|
|
self.result.add_error('transaction.clear_range resulted in incorrect database')
|
|
|
|
# Fill the database back up with data
|
|
self.correctnessSet(db, store, data, maxKeysPerTransaction)
|
|
if not self.compareDatabaseToMemory(db, store):
|
|
self.result.add_error('transaction.set resulted in incorrect database')
|
|
break
|
|
|
|
if not success:
|
|
break
|
|
|
|
# Compare the results of clear_range_startswith
|
|
self.correctnessClearPrefix(db, store, data, numPrefixClears)
|
|
if not self.compareDatabaseToMemory(db, store):
|
|
self.result.add_error('transaction.clear_range_startswith resulted in incorrect database')
|
|
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
self.result.add_error(self.getError('Database error in correctness test'))
|
|
|
|
# Stores data in the database and a memory key-value store
|
|
def correctnessSet(self, db, store, data, maxKeysPerTransaction):
|
|
for [key, value] in data:
|
|
store.set(key, value)
|
|
|
|
keysCommitted = 0
|
|
while keysCommitted < len(data):
|
|
self.correctnessSetTransactional(db, data[keysCommitted: keysCommitted + maxKeysPerTransaction])
|
|
keysCommitted += maxKeysPerTransaction
|
|
|
|
# Stores data in the database
|
|
@fdb.transactional
|
|
def correctnessSetTransactional(self, tr, data):
|
|
for [key, value] in data:
|
|
tr.set(key, value)
|
|
|
|
# Compares the results of the get operation from the database and a memory key-value store
|
|
def correctnessGet(self, db, store, data, numReads, maxKeysPerTransaction):
|
|
keys = []
|
|
for i in range(0, numReads):
|
|
index = random.randint(0, len(data) - 1)
|
|
keys.append(data[index][0])
|
|
|
|
keysRetrieved = 0
|
|
while keysRetrieved < len(keys):
|
|
subKeys = keys[keysRetrieved: keysRetrieved + maxKeysPerTransaction]
|
|
|
|
values = self.correctnessGetTransactional(db, subKeys)
|
|
for i in range(0, numReads):
|
|
if values[i] != store.get(subKeys[i]):
|
|
print('mismatch: %s', subKeys[i])
|
|
return False
|
|
keysRetrieved += maxKeysPerTransaction
|
|
|
|
return True
|
|
|
|
# Gets the values for the specified list of keys from the database
|
|
@fdb.transactional
|
|
def correctnessGetTransactional(self, tr, keys):
|
|
futures = []
|
|
for key in keys:
|
|
futures.append(tr.get(key))
|
|
|
|
values = []
|
|
for future in futures:
|
|
values.append(future.wait())
|
|
|
|
return values
|
|
|
|
# Compares the results of the get_range operation from the database and a memory key-value store
|
|
def correctnessGetRange(self, db, store, data):
|
|
index = random.randint(0, len(data) - 1)
|
|
index2 = random.randint(0, len(data) - 1)
|
|
|
|
key1 = min(data[index][0], data[index2][0])
|
|
key2 = max(data[index][0], data[index2][0])
|
|
|
|
dbResults = self.correctnessGetRangeTransactional(db, key1, key2, data)
|
|
storeResults = store.get_range(key1, key2)
|
|
|
|
return self.compareResults(dbResults, storeResults)
|
|
|
|
# Gets the entries in the range [key1,key2) from the database
|
|
@fdb.transactional
|
|
def correctnessGetRangeTransactional(self, tr, key1, key2, data=None):
|
|
if data is not None:
|
|
return list(tr.get_range(key1, key2, len(data)))
|
|
else:
|
|
return list(tr.get_range(key1, key2))
|
|
|
|
# Compares the results of the get_range_startswith operation from the database and a memory key-value store
|
|
def correctnessGetPrefix(self, db, store, data):
|
|
prefix = ''.join(chr(random.randint(0, 254)) for i in range(0, random.randint(1, 3)))
|
|
dbResults = self.correctnessGetPrefixTransactional(db, prefix)
|
|
storeResults = store.get_range_startswith(prefix)
|
|
|
|
return self.compareResults(dbResults, storeResults)
|
|
|
|
# Gets the entries with a given prefix from the database
|
|
@fdb.transactional
|
|
def correctnessGetPrefixTransactional(self, tr, prefix):
|
|
return list(tr.get_range_startswith(prefix))
|
|
|
|
# Compares the results of the get_key operation from the database and a memory key-value store
|
|
def correctnessGetKey(self, db, store, data, numGetKeys, maxKeysPerTransaction):
|
|
selectors = []
|
|
for i in range(0, numGetKeys):
|
|
index = random.randint(0, len(data) - 1)
|
|
or_equal = random.randint(0, 1)
|
|
offset = random.randint(max(-index + (1 - or_equal), -10), min(len(data) - index - 1 + or_equal, 10))
|
|
|
|
key = sorted(data)[index][0]
|
|
selector = fdb.KeySelector(key, or_equal, offset)
|
|
selectors.append(selector)
|
|
|
|
keysRetrieved = 0
|
|
while keysRetrieved < len(selectors):
|
|
subSelectors = selectors[keysRetrieved: keysRetrieved + maxKeysPerTransaction]
|
|
dbKeys = self.correctnessGetKeyTransactional(db, subSelectors)
|
|
for i in range(0, numGetKeys):
|
|
if dbKeys[i] != store.get_key(subSelectors[i]):
|
|
return False
|
|
keysRetrieved += maxKeysPerTransaction
|
|
|
|
return True
|
|
|
|
# Gets the keys specified by the list of key selectors
|
|
@fdb.transactional
|
|
def correctnessGetKeyTransactional(self, tr, keySelectors):
|
|
futures = []
|
|
count = 0
|
|
for selector in keySelectors:
|
|
futures.append(tr.get_key(selector))
|
|
count += 1
|
|
|
|
keys = []
|
|
count = 0
|
|
for future in futures:
|
|
keys.append(future.wait())
|
|
count += 1
|
|
|
|
return keys
|
|
|
|
# Clears data from a database and a memory key-value store
|
|
def correctnessClear(self, db, store, data, numClears, maxKeysPerTransaction):
|
|
clearedKeys = []
|
|
for i in range(0, numClears):
|
|
index = random.randint(0, len(data) - 1)
|
|
clearedKeys.append(data[index][0])
|
|
store.clear(data[index][0])
|
|
|
|
keysCleared = 0
|
|
while keysCleared < len(clearedKeys):
|
|
self.correctnessClearTransactional(db, clearedKeys[keysCleared: keysCleared + maxKeysPerTransaction])
|
|
keysCleared += maxKeysPerTransaction
|
|
|
|
return clearedKeys
|
|
|
|
# Clears a list of keys from the database
|
|
@fdb.transactional
|
|
def correctnessClearTransactional(self, tr, clearedKeys):
|
|
for key in clearedKeys:
|
|
tr.clear(key)
|
|
|
|
# Clears a range of data from a database and a memory key-value store
|
|
def correctnessClearRange(self, db, store, data):
|
|
index = random.randint(0, len(data) - 1)
|
|
index2 = random.randint(0, len(data) - 1)
|
|
|
|
key1 = min(data[index][0], data[index2][0])
|
|
key2 = max(data[index][0], data[index2][0])
|
|
|
|
self.correctnessClearRangeTransactional(db, key1, key2)
|
|
store.clear_range(key1, key2)
|
|
|
|
# Clears a range of memory from a database
|
|
@fdb.transactional
|
|
def correctnessClearRangeTransactional(self, tr, key1, key2):
|
|
tr.clear_range(key1, key2)
|
|
|
|
# Clears data with random prefixes from a database and a memory key-value store
|
|
def correctnessClearPrefix(self, db, store, data, numPrefixClears):
|
|
prefixes = []
|
|
for i in range(0, numPrefixClears):
|
|
prefix = ''.join(chr(random.randint(0, 254)) for i in range(0, random.randint(1, 3)))
|
|
prefixes.append(prefix)
|
|
store.clear_range_startswith(prefix)
|
|
|
|
self.correctnessClearPrefixTransactional(db, prefixes)
|
|
|
|
# Clears keys from a database that have a prefix in the prefixes list
|
|
@fdb.transactional
|
|
def correctnessClearPrefixTransactional(self, tr, prefixes):
|
|
for prefix in prefixes:
|
|
tr.clear_range_startswith(prefix)
|
|
|
|
# Adds the stack trace to an error message
|
|
def getError(self, message):
|
|
errorMessage = message + "\n" + traceback.format_exc()
|
|
print('%s', errorMessage)
|
|
return errorMessage
|
|
|
|
|
|
if __name__ == '__main__':
|
|
print("Running PythonCorrectness test on Python version %d.%d.%d%s%d" %
|
|
(sys.version_info[0], sys.version_info[1], sys.version_info[2], sys.version_info[3][0], sys.version_info[4]))
|
|
|
|
PythonCorrectness().run()
|