foundationdb/tests/python_tests/python_correctness.py

865 lines
28 KiB
Python
Executable File

#!/usr/bin/python
#
# python_correctness.py
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import sys
import time
import random
import traceback
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
from python_tests import PythonTest
import fdb
import fdb.tuple
fdb.api_version(400)
# A class that mimics some of the operations of the FoundationDB key-value store
class KeyValueStore:
# Uses a simple dictionary to store key-value pairs
# Any operations that depend on the order of keys first sort the data
store = dict()
def get(self, key):
if key is fdb.KeySelector:
key = self.get_key(key)
if key in self.store:
return self.store[key]
else:
return None
def get_key(self, key_selector):
sorted_keys = list(sorted(self.store.keys()))
for index, key in enumerate(sorted_keys):
if key >= key_selector.key:
index += key_selector.offset
if (
key == key_selector.key and not key_selector.or_equal
) or key != key_selector.key:
index -= 1
if index < 0 or index >= len(sorted_keys):
return ""
else:
return sorted_keys[index]
index = len(sorted_keys) + key_selector.offset - 1
if index < 0 or index >= len(sorted_keys):
return ""
else:
return sorted_keys[index]
def get_range(self, begin, end, limit=None):
values = []
count = 0
if begin is fdb.KeySelector:
begin = self.get_key(begin)
if end is fdb.KeySelector:
end = self.get_key(end)
for key, value in sorted(self.store.items()):
if limit is not None and count >= limit:
break
if key >= end:
break
if key >= begin:
values.append([key, value])
count += 1
return values
def get_range_startswith(self, prefix, limit=None):
values = []
count = 0
for key, value in sorted(self.store.items()):
if limit is not None and count >= limit:
break
if key > prefix and not key.startswith(prefix):
break
if key.startswith(prefix):
values.append([key, value])
count += 1
return values
def set(self, key, value):
self.store[key] = value
def clear(self, key):
if key is fdb.KeySelector:
key = self.get_key(key)
if key in self.store:
del self.store[key]
def clear_range(self, begin, end):
for key, value in self.get_range(begin, end):
del self.store[key]
def clear_range_startswith(self, prefix):
for key, value in self.get_range_startswith(prefix):
del self.store[key]
class PythonCorrectness(PythonTest):
callback = False
callbackError = ""
# Python correctness tests (checks if functions run and yield correct results)
def run_test(self):
try:
db = fdb.open(None, "DB")
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("fdb.open failed"))
return
try:
print("Testing functions...")
self.test_functions(db)
print("Testing correctness...")
del db[:]
self.test_correctness(db)
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Failed to complete all tests"))
# Generates a random set of keys and values
def generate_data(
self,
num_keys,
min_key_length,
max_key_length,
min_value_length,
max_value_length,
prefix="",
allow_duplicates=True,
):
data = list()
keys = set()
while len(data) < num_keys:
# key = prefix + ''.join(random.choice(string.ascii_lowercase)
# for i in range(0, random.randint(minKeyLength - len(prefix), maxKeyLength - len(prefix))))
key = prefix + "".join(
chr(random.randint(0, 254))
for _ in range(
0,
random.randint(
min_key_length - len(prefix), max_key_length - len(prefix)
),
)
)
if not allow_duplicates:
if key in keys:
continue
else:
keys.add(key)
value = "".join(
"x" for _ in range(0, random.randint(max_key_length, max_value_length))
)
data.append([key, value])
return data
# Function to test the callback feature of Future objects
def test_callback(self, future):
try:
future.wait()
except KeyboardInterrupt:
raise
except Exception:
self.callbackError = self.get_error("Callback future get failed")
self.callback = True
# Tests that all of the functions in the python API can be called without failing
def test_functions(self, db):
self.callback = False
self.callbackError = ""
try:
tr = db.create_transaction()
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("db.create_transaction failed"))
return
try:
tr["testkey"] = "testvalue"
value = tr["testkey"]
value.is_ready()
value.block_until_ready()
value.wait()
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(
self.get_error("Set/Get value failed (block until ready)")
)
try:
value = tr["testkey"]
value.wait()
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Get value failed"))
try:
tr["testkey"] = "newtestvalue"
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Replace value failed"))
try:
value = tr["fakekey"]
# The following line would generate a segfault
# value.capi.fdb_future_block_until_ready(0)
value.wait()
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Get non-existent key failed"))
try:
tr.commit().wait()
tr.get_committed_version()
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Commit failed"))
try:
tr.reset()
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Reset failed"))
try:
version = tr.get_read_version()
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("tr.get_read_version failed"))
try:
value = tr["testkey"]
value.wait()
tr.reset()
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Get and reset failed"))
try:
tr.set_read_version(version.wait())
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Set read version failed"))
try:
value = tr["testkey"]
callback_time = time.time()
value.on_ready(self.test_callback)
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Get future and set callback failed"))
try:
del tr["testkey"]
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Delete key failed"))
try:
del tr["fakekey"]
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Delete non-existent key failed"))
try:
tr.set("testkey", "testvalue")
value = tr.get("testkey")
value.wait()
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Future.get failed"))
try:
tr.clear("testkey")
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Clear key failed"))
try:
tr["testkey1"] = "testvalue1"
tr["testkey2"] = "testvalue2"
tr["testkey3"] = "testvalue3"
for k, v in tr.get_range("testkey1", "testkey3"):
v += ""
for k, v in tr.get_range("testkey1", "testkey2", 2):
v += ""
for k, v in tr["testkey1":"testkey3"]:
v += ""
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Get range failed"))
try:
tr["otherkey1"] = "othervalue1"
tr["otherkey2"] = "othervalue2"
for k, v in tr.get_range_startswith("testkey"):
v += ""
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Get range starts with failed"))
try:
tr.clear_range_startswith("otherkey")
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Clear range starts with failed"))
try:
tr.clear_range("testkey1", "testkey3")
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Clear range failed"))
try:
tr["testkey1"] = "testvalue1"
tr["testkey2"] = "testvalue2"
tr["testkey3"] = "testvalue3"
begin = fdb.KeySelector("testkey2", 0, 0)
end = fdb.KeySelector("testkey2", 0, 1)
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Create key selector failed"))
try:
for k, v in tr.get_range(begin, end):
v += ""
for k, v in tr.get_range(begin, end, 2):
v += ""
for k, v in tr[begin:end]:
v += ""
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Get range (key selectors) failed"))
try:
tr.clear_range(begin, end)
tr["testkey1"] = "testvalue1"
tr["testkey2"] = "testvalue2"
tr["testkey3"] = "testvalue3"
del tr[begin:end]
tr["testkey1"] = "testvalue1"
tr["testkey2"] = "testvalue2"
tr["testkey3"] = "testvalue3"
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Clear range (key selectors) failed"))
try:
begin = fdb.KeySelector.last_less_than("testkey2")
end = fdb.KeySelector.first_greater_or_equal("testkey2")
for k, v in tr.get_range(begin, end):
v += ""
begin = fdb.KeySelector.last_less_or_equal("testkey2")
end = fdb.KeySelector.first_greater_than("testkey2")
for k, v in tr.get_range(begin, end):
v += ""
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Builtin key selectors failed"))
try:
del tr["testkey1":"testkey3"]
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Delete key range failed"))
try:
tr.commit().wait()
tr.get_committed_version()
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Commit failed"))
try:
key = fdb.tuple.pack(("k1", "k2", "k3"))
k_tuple = fdb.tuple.unpack(key)
if k_tuple[0] != "k1" and k_tuple[1] != "k2" and k_tuple[2] != "k3":
self.result.add_error(
"Tuple <-> key conversion yielded incorrect results"
)
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Tuple <-> key conversion failed"))
try:
tr[fdb.tuple.pack(("k1", "k2"))] = "v"
tr[fdb.tuple.pack(("k1", "k2", "k3"))] = "v1"
tr[fdb.tuple.pack(("k1", "k2", "k3", "k4"))] = "v2"
for k, v in tr[fdb.tuple.range(("k1", "k2"))]:
v += ""
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Tuple get range failed"))
try:
tr["testint"] = "10"
y = int(tr["testint"]) + 1
if y != 11:
self.result.add_error("Value retrieval yielded incorrect results")
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Future value retrieval failed"))
if not self.callback:
time.sleep(5)
if not self.callback:
self.result.add_error(
"Warning: Future callback not called after %f seconds"
% (time.time() - callback_time)
)
if len(self.callbackError) > 0:
self.result.add_error(self.callbackError)
# Compares a FoundationDB database with an in-memory key-value store
def compare_database_to_memory(self, db, store):
db_result = self.correctness_get_range_transactional(db, "\x00", "\xff")
store_result = store.get_range("\x00", "\xff")
return self.compare_results(db_result, store_result)
# Compares result sets coming from a FoundationDB database and an in-memory key-value store
def compare_results(self, db_results, store_results):
if len(db_results) != len(store_results):
# print 'mismatched lengths: ' + str(len(dbResults)) + ' - ' + str(len(storeResults))
return False
for i in range(0, len(db_results)):
# if i >= len(storeResults):
# print 'mismatched key: ' + dbResults[i].key
# return False
if (
db_results[i].key != store_results[i][0]
or db_results[i].value != store_results[i][1]
):
# print 'mismatched key: ' + dbResults[i].key + ' - ' + storeResults[i][0]
return False
return True
# Performs the same operations on a FoundationDB database and an in-memory key-value store and compares the results
def test_correctness(self, db):
num_keys = 5000
ratio_short_keys = 0.5
min_short_key_length = 1
max_short_key_length = 3
min_long_key_length = 1
max_long_key_length = 128
min_value_length = 1
max_value_length = 10000
max_transaction_bytes = 5000000
num_reads = 100
num_range_reads = 100
num_prefix_reads = 100
num_get_keys = 100
# num_clears = 100
num_range_clears = 10
num_prefix_clears = 10
max_keys_per_transaction = max(
1, int(max_transaction_bytes / (max_value_length + max_long_key_length))
)
try:
store = KeyValueStore()
# Generate some random data
data = self.generate_data(
num_keys * ratio_short_keys,
min_short_key_length,
max_short_key_length,
min_value_length,
max_value_length,
)
data.extend(
self.generate_data(
num_keys * (1 - ratio_short_keys),
min_long_key_length,
max_long_key_length,
min_value_length,
max_value_length,
)
)
# Insert the data
self.correctness_set(db, store, data, max_keys_per_transaction)
if not self.compare_database_to_memory(db, store):
self.result.add_error("transaction.set resulted in incorrect database")
# Compare the results of single key reads
if not self.correctness_get(
db, store, data, num_reads, max_keys_per_transaction
):
self.result.add_error("transaction.get returned incorrect result")
# Compare the results of range reads
for i in range(0, num_range_reads):
if not self.correctness_get_range(db, store, data):
self.result.add_error(
"transaction.get_range returned incorrect results"
)
break
# Compare the results of prefix reads
for i in range(0, num_prefix_reads):
if not self.correctness_get_prefix(db, store, data):
self.result.add_error(
"transaction.get_range_startswith returned incorrect results"
)
break
# Compare the results of get key
if not self.correctness_get_key(
db, store, data, num_get_keys, max_keys_per_transaction
):
self.result.add_error("transaction.get_key returned incorrect results")
# Compare the results of clear
# clearedKeys = self.correctnessClear(db, store, data, num_clears, max_keys_per_transaction)
if not self.compare_database_to_memory(db, store):
self.result.add_error(
"transaction.clear resulted in incorrect database"
)
# for key in clearedKeys:
# print 'clearing key ' + key
# else:
# print 'successful compare'
# Fill the database back up with data
self.correctness_set(db, store, data, max_keys_per_transaction)
if not self.compare_database_to_memory(db, store):
self.result.add_error("transaction.set resulted in incorrect database")
# Compare the results of clear_range
for i in range(0, num_range_clears):
self.correctness_clear_range(db, store, data)
success = self.compare_database_to_memory(db, store)
if not success:
self.result.add_error(
"transaction.clear_range resulted in incorrect database"
)
# Fill the database back up with data
self.correctness_set(db, store, data, max_keys_per_transaction)
if not self.compare_database_to_memory(db, store):
self.result.add_error(
"transaction.set resulted in incorrect database"
)
break
if not success:
break
# Compare the results of clear_range_startswith
self.correctness_clear_prefix(db, store, data, num_prefix_clears)
if not self.compare_database_to_memory(db, store):
self.result.add_error(
"transaction.clear_range_startswith resulted in incorrect database"
)
except KeyboardInterrupt:
raise
except Exception:
self.result.add_error(self.get_error("Database error in correctness test"))
# Stores data in the database and a memory key-value store
def correctness_set(self, db, store, data, max_keys_per_transaction):
for [key, value] in data:
store.set(key, value)
keys_committed = 0
while keys_committed < len(data):
self.correctness_set_transactional(
db, data[keys_committed : keys_committed + max_keys_per_transaction]
)
keys_committed += max_keys_per_transaction
# Stores data in the database
@fdb.transactional
def correctness_set_transactional(self, tr, data):
for [key, value] in data:
tr.set(key, value)
# Compares the results of the get operation from the database and a memory key-value store
def correctness_get(self, db, store, data, num_reads, max_keys_per_transaction):
keys = []
for i in range(0, num_reads):
index = random.randint(0, len(data) - 1)
keys.append(data[index][0])
keys_retrieved = 0
while keys_retrieved < len(keys):
sub_keys = keys[keys_retrieved : keys_retrieved + max_keys_per_transaction]
values = self.correctness_get_transactional(db, sub_keys)
for i in range(0, num_reads):
if values[i] != store.get(sub_keys[i]):
print("mismatch: %s", sub_keys[i])
return False
keys_retrieved += max_keys_per_transaction
return True
# Gets the values for the specified list of keys from the database
@fdb.transactional
def correctness_get_transactional(self, tr, keys):
futures = []
for key in keys:
futures.append(tr.get(key))
values = []
for future in futures:
values.append(future.wait())
return values
# Compares the results of the get_range operation from the database and a memory key-value store
def correctness_get_range(self, db, store, data):
index = random.randint(0, len(data) - 1)
index2 = random.randint(0, len(data) - 1)
key1 = min(data[index][0], data[index2][0])
key2 = max(data[index][0], data[index2][0])
db_results = self.correctness_get_range_transactional(db, key1, key2, data)
store_results = store.get_range(key1, key2)
return self.compare_results(db_results, store_results)
# Gets the entries in the range [key1,key2) from the database
@fdb.transactional
def correctness_get_range_transactional(self, tr, key1, key2, data=None):
if data is not None:
return list(tr.get_range(key1, key2, len(data)))
else:
return list(tr.get_range(key1, key2))
# Compares the results of the get_range_startswith operation from the database and a memory key-value store
def correctness_get_prefix(self, db, store, data):
prefix = "".join(
chr(random.randint(0, 254)) for _ in range(0, random.randint(1, 3))
)
db_results = self.correctness_get_prefix_transactional(db, prefix)
store_results = store.get_range_startswith(prefix)
return self.compare_results(db_results, store_results)
# Gets the entries with a given prefix from the database
@fdb.transactional
def correctness_get_prefix_transactional(self, tr, prefix):
return list(tr.get_range_startswith(prefix))
# Compares the results of the get_key operation from the database and a memory key-value store
def correctness_get_key(
self, db, store, data, num_get_keys, max_keys_per_transaction
):
selectors = []
for i in range(0, num_get_keys):
index = random.randint(0, len(data) - 1)
or_equal = random.randint(0, 1)
offset = random.randint(
max(-index + (1 - or_equal), -10),
min(len(data) - index - 1 + or_equal, 10),
)
key = sorted(data)[index][0]
selector = fdb.KeySelector(key, or_equal, offset)
selectors.append(selector)
keys_retrieved = 0
while keys_retrieved < len(selectors):
sub_selectors = selectors[
keys_retrieved : keys_retrieved + max_keys_per_transaction
]
db_keys = self.correctness_get_key_transactional(db, sub_selectors)
for i in range(0, num_get_keys):
if db_keys[i] != store.get_key(sub_selectors[i]):
return False
keys_retrieved += max_keys_per_transaction
return True
# Gets the keys specified by the list of key selectors
@fdb.transactional
def correctness_get_key_transactional(self, tr, key_selectors):
futures = []
count = 0
for selector in key_selectors:
futures.append(tr.get_key(selector))
count += 1
keys = []
count = 0
for future in futures:
keys.append(future.wait())
count += 1
return keys
# Clears data from a database and a memory key-value store
def correctness_clear(self, db, store, data, num_clears, max_keys_per_transaction):
cleared_keys = []
for i in range(0, num_clears):
index = random.randint(0, len(data) - 1)
cleared_keys.append(data[index][0])
store.clear(data[index][0])
keys_cleared = 0
while keys_cleared < len(cleared_keys):
self.correctness_clear_transactional(
db, cleared_keys[keys_cleared : keys_cleared + max_keys_per_transaction]
)
keys_cleared += max_keys_per_transaction
return cleared_keys
# Clears a list of keys from the database
@fdb.transactional
def correctness_clear_transactional(self, tr, cleared_keys):
for key in cleared_keys:
tr.clear(key)
# Clears a range of data from a database and a memory key-value store
def correctness_clear_range(self, db, store, data):
index = random.randint(0, len(data) - 1)
index2 = random.randint(0, len(data) - 1)
key1 = min(data[index][0], data[index2][0])
key2 = max(data[index][0], data[index2][0])
self.correctness_clear_range_transactional(db, key1, key2)
store.clear_range(key1, key2)
# Clears a range of memory from a database
@fdb.transactional
def correctness_clear_range_transactional(self, tr, key1, key2):
tr.clear_range(key1, key2)
# Clears data with random prefixes from a database and a memory key-value store
def correctness_clear_prefix(self, db, store, data, num_prefix_clears):
prefixes = []
for i in range(0, num_prefix_clears):
prefix = "".join(
chr(random.randint(0, 254)) for _ in range(0, random.randint(1, 3))
)
prefixes.append(prefix)
store.clear_range_startswith(prefix)
self.correctness_clear_prefix_transactional(db, prefixes)
# Clears keys from a database that have a prefix in the prefixes list
@fdb.transactional
def correctness_clear_prefix_transactional(self, tr, prefixes):
for prefix in prefixes:
tr.clear_range_startswith(prefix)
# Adds the stack trace to an error message
def get_error(self, message):
error_message = message + "\n" + traceback.format_exc()
print("%s", error_message)
return error_message
if __name__ == "__main__":
print(
"Running PythonCorrectness test on Python version %d.%d.%d%s%d"
% (
sys.version_info[0],
sys.version_info[1],
sys.version_info[2],
sys.version_info[3][0],
sys.version_info[4],
)
)
PythonCorrectness().run()