foundationdb/fdbserver/workloads/ApiCorrectness.actor.cpp

/*
 * ApiCorrectness.actor.cpp
 *
 * This source file is part of the FoundationDB open source project
 *
 * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "fdbserver/QuietDatabase.h"

#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/workloads/ApiWorkload.h"
#include "fdbserver/workloads/MemoryKeyValueStore.h"
#include "flow/actorcompiler.h"  // This must be the last #include.

//An enum of API operation types used in the random test
enum OperationType {
	SET,
	GET,
	GET_RANGE,
	GET_RANGE_SELECTOR,
	GET_KEY,
	CLEAR,
	CLEAR_RANGE
};

//A workload that executes the NativeAPIs functions and verifies that their outcomes are correct
struct ApiCorrectnessWorkload : ApiWorkload {

private:

//Enable to track the activity on a particular key
#if CENABLED(0, NOT_IN_CLEAN)
#define targetKey LiteralStringRef( ??? )

	void debugKey(KeyRef key, std::string context) {
		if(key == targetKey)
			TraceEvent("ApiCorrectnessDebugKey").detail("Context", context).detail("Key", printable(key));
	}

	void debugKey(KeyRangeRef keyRange, std::string context) {
		if(keyRange.contains(targetKey))
			TraceEvent("ApiCorrectnessDebugKey").detail("Context", context).detail("Key", printable(targetKey))
			.detail("RangeBegin", printable(keyRange.begin)).detail("RangeEnd", printable(keyRange.end));
	}

#else
	void debugKey(KeyRef key, std::string context) { }
	void debugKey(KeyRangeRef keyRange, std::string context) { }

#endif

public:
	//The number of gets that should be performed
	int numGets;

	//The number of getRanges that should be performed
	int numGetRanges;

	//The number of getRanges using key selectors that should be performed
	int numGetRangeSelectors;

	//The number of getKeys that should be performed
	int numGetKeys;

	//The number of clears that should be performed
	int numClears;

	//The number of clears using key ranges that should be performed
	int numClearRanges;

	//The smallest legal size of the database after a clear.  A smaller size will trigger a database reset
	int minSizeAfterClear;

	//The maximum number of keys that can be in this client's key space when performing the random test
	int maxRandomTestKeys;

	//The amount of time to run the random tests
	double randomTestDuration;

	//The maximum number of keys operated on in a transaction; used to prevent transaction_too_old errors
	int maxKeysPerTransaction;

	//The number of API calls made by the random test
	PerfIntCounter numRandomOperations;

	//The API being used by this client
	TransactionType transactionType;

	ApiCorrectnessWorkload(WorkloadContext const& wcx) : ApiWorkload(wcx), numRandomOperations("Num Random Operations") {
		numGets = getOption(options, LiteralStringRef("numGets"), 1000);
		numGetRanges = getOption(options, LiteralStringRef("numGetRanges"), 100);
		numGetRangeSelectors = getOption(options, LiteralStringRef("numGetRangeSelectors"), 100);
		numGetKeys = getOption(options, LiteralStringRef("numGetKeys"), 100);
		numClears = getOption(options, LiteralStringRef("numClears"), 100);
		numClearRanges = getOption(options, LiteralStringRef("numClearRanges"), 100);
		minSizeAfterClear = getOption(options, LiteralStringRef("minSizeAfterClear"), (int)(0.1 * numKeys));

		maxRandomTestKeys = getOption(options, LiteralStringRef("maxRandomTestKeys"), numKeys);
		randomTestDuration = getOption(options, LiteralStringRef("randomTestDuration"), 60.0);

		int maxTransactionBytes = getOption(options, LiteralStringRef("maxTransactionBytes"), 500000);
		maxKeysPerTransaction = std::max(1, maxTransactionBytes / (maxValueLength + maxLongKeyLength));

		if(maxTransactionBytes > 500000) {
			TraceEvent("RemapEventSeverity").detail("TargetEvent", "Net2_LargePacket").detail("OriginalSeverity", SevWarnAlways).detail("NewSeverity", SevInfo);
			TraceEvent("RemapEventSeverity").detail("TargetEvent", "LargeTransaction").detail("OriginalSeverity", SevWarnAlways).detail("NewSeverity", SevInfo);
			TraceEvent("RemapEventSeverity").detail("TargetEvent", "DiskQueueMemoryWarning").detail("OriginalSeverity", SevWarnAlways).detail("NewSeverity", SevInfo);
		}
	}

	virtual ~ApiCorrectnessWorkload(){ }

	std::string description() {
		return "ApiCorrectness";
	}

	void getMetrics(vector<PerfMetric>& m) {
		m.push_back(PerfMetric("Number of Random Operations Performed", numRandomOperations.getValue(), false));
	}

	ACTOR Future<Void> performSetup(Database cx, ApiCorrectnessWorkload *self) {
		//Choose a random transaction type (NativeAPI, ReadYourWrites, ThreadSafe, MultiVersion)
		std::vector<TransactionType> types;
		types.push_back(NATIVE);
		types.push_back(READ_YOUR_WRITES);
		types.push_back(THREAD_SAFE);
		types.push_back(MULTI_VERSION);

		wait(self->chooseTransactionFactory(cx, types));

		return Void();
	}

	Future<Void> performSetup(Database const& cx) {
		return performSetup(cx, this);
	}

	ACTOR Future<Void> performTest(Database cx, Standalone<VectorRef<KeyValueRef>> data, ApiCorrectnessWorkload *self) {
		//Run the scripted test for a maximum of 10 minutes
		wait(timeout(self->runScriptedTest(self, data), 600, Void()));

		if(!self->hasFailed()) {
			//Return database to original state (for a maximum of 1800 seconds)
			try {
				wait(timeoutError(::success(self->runSet(data, self)), 1800));
			}
			catch(Error &e) {
				if(e.code() == error_code_timed_out) {
					if(!self->hasFailed())
						self->testFailure("Timeout during database reset");

					return Void();
				}

				throw;
			}

			//Run the random test for the user-specified duration
			wait(timeout(self->runRandomTest(self, data), self->randomTestDuration, Void()));
		}

		return Void();
	}

	Future<Void> performTest(Database const& cx, Standalone<VectorRef<KeyValueRef>> const& data) {
		return performTest(cx, data, this);
	}

	//Run a scripted set of API operations
	ACTOR Future<Void> runScriptedTest(ApiCorrectnessWorkload *self, VectorRef<KeyValueRef> data) {
		//Test the set function
		bool setResult = wait(self->runSet(data, self));
		if(!setResult)
			return Void();

		//Test the get function
		wait(::success(self->runGet(data, self->numGets, self)));

		//Test the getRange function
		state int i;
		for (i = 0; i < self->numGetRanges; i++) wait(::success(self->runGetRange(data, self)));

		//Test the getRange function using key selectors
		for (i = 0; i < self->numGetRangeSelectors; i++) wait(::success(self->runGetRangeSelector(data, self)));

		//Test the getKey function
		wait(::success(self->runGetKey(data, self->numGetKeys, self)));

		//Test the clear function
		bool clearResult = wait(self->runClear(data, self->numClears, self));
		if(!clearResult)
			return Void();

		//Test the clear function using keyRanges
		for(i = 0; i < self->numClearRanges; i++) {
			//Alternate restoring the database to its original state and clearing a single range
			if(self->store.size() < self->minSizeAfterClear) {
				bool resetResult = wait(self->runSet(data, self));
				if(!resetResult)
					return Void();
			}

			bool clearRangeResults = wait(self->runClearRange(data, self));
			if(!clearRangeResults)
				return Void();
		}

		return Void();
	}

	//Generate and execute a sequence of random operations
	ACTOR Future<Void> runRandomTest(ApiCorrectnessWorkload *self, Standalone<VectorRef<KeyValueRef>> data) {
		loop {
			double setProbability = 1 - ((double)self->store.size()) / self->maxRandomTestKeys;
			int pdfArray[] = { 0, (int)(100 * setProbability), 100, 50, 50, 20, (int)(100 * (1 - setProbability)), (int)(10 * (1 - setProbability)) };
			vector<int> pdf = vector<int>(pdfArray, pdfArray + 8);

			OperationType operation;

			//Choose a random operation type (SET, GET, GET_RANGE, GET_RANGE_SELECTOR, GET_KEY, CLEAR, CLEAR_RANGE).
			int totalDensity = 0;
			for(int i = 0; i < pdf.size(); i++)
				totalDensity += pdf[i];

			int cumulativeDensity = 0;
			int random = deterministicRandom()->randomInt(0, totalDensity);
			for(int i = 0; i < pdf.size() - 1; i++) {
				if(cumulativeDensity + pdf[i] <= random && random < cumulativeDensity + pdf[i] + pdf[i + 1]) {
					operation = (OperationType)i;
					break;
				}

				cumulativeDensity += pdf[i];
			}

			++self->numRandomOperations;

			//Test the set operation
			if(operation == SET) {
				bool useShortKeys = deterministicRandom()->randomInt(0, 2) == 1;
				int minKeyLength = useShortKeys ? self->minShortKeyLength : self->minLongKeyLength;
				int maxKeyLength = useShortKeys ? self->maxShortKeyLength : self->maxLongKeyLength;

				state Standalone<VectorRef<KeyValueRef>> newData = self->generateData(std::min((uint64_t)100, self->maxRandomTestKeys - self->store.size()), minKeyLength, maxKeyLength,
																		  self->minValueLength, self->maxValueLength, self->clientPrefix, true);

				data.append_deep(data.arena(), newData.begin(), newData.size());

				bool result = wait(self->runSet(newData, self));
				if(!result)
					return Void();
			}

			//Test the get operation
			else if(operation == GET) {
				bool result = wait(self->runGet(data, 10, self));
				if(!result)
					return Void();
			}

			//Test the getRange operation
			else if(operation == GET_RANGE) {
				bool result = wait(self->runGetRange(data, self));
				if(!result)
					return Void();
			}

			//Test the getRange operation with key selectors
			else if(operation == GET_RANGE_SELECTOR) {
				bool result = wait(self->runGetRangeSelector(data, self));
				if(!result)
					return Void();
			}

			//Test the getKey operation
			else if(operation == GET_KEY) {
				bool result = wait(self->runGetKey(data, 10, self));
				if(!result)
					return Void();
			}

			//Test the clear operation
			else if(operation == CLEAR) {
				bool result = wait(self->runClear(data, 10, self));
				if(!result)
					return Void();
			}

			//Test the clear operation (using key range)
			else if(operation == CLEAR_RANGE) {
				bool result = wait(self->runClearRange(data, self));
				if(!result)
					return Void();
			}
		}
	}

	//Adds the key-value pairs in data to the database and memory store
	ACTOR Future<bool> runSet(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload *self) {
		state int currentIndex = 0;
		while(currentIndex < data.size()) {
			state Reference<TransactionWrapper> transaction = self->createTransaction();

			//Set keys in the database
			loop {
				try {
					//For now, make this transaction self-conflicting to avoid commit errors
					Optional<Value> value = wait(transaction->get(data[currentIndex].key));

					for(int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, data.size()); i++) {
						transaction->addReadConflictRange(singleKeyRange(data[i].key));
						transaction->set(data[i].key, data[i].value);
					}

					wait(transaction->commit());
					for(int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, data.size()); i++)
						debugMutation("ApiCorrectnessSet", transaction->getCommittedVersion(), MutationRef(MutationRef::DebugKey, data[i].key, data[i].value));

					currentIndex += self->maxKeysPerTransaction;
					break;
				}
				catch(Error &e) {
					wait(transaction->onError(e));
				}
			}
		}

		//Set keys in memory
		for(int i = 0; i < data.size(); i++) {
			self->store.set(data[i].key, data[i].value);
			self->debugKey(data[i].key, "Set");
		}

		//Check that the database and memory store are the same
		bool result = wait(self->compareDatabaseToMemory());
		if(!result)
			self->testFailure("Set resulted in incorrect database");

		return result;
	}

	//Gets a specified number of values from the database and memory store and compares them, returning true if all
	//results were the same
	ACTOR Future<bool> runGet(VectorRef<KeyValueRef> data, int numReads, ApiCorrectnessWorkload *self) {
		//Generate a set of random keys to get
		state Standalone<VectorRef<Key>> keys;
		for(int i = 0; i < numReads; i++)
			keys.push_back(keys.arena(), self->selectRandomKey(data, 0.9));

		state vector<Optional<Value>> values;

		state int currentIndex = 0;
		while(currentIndex < keys.size()) {
			state Reference<TransactionWrapper> transaction = self->createTransaction();

			//Get the values from the database
			loop {
				try {
					state vector<Future<Optional<Value>>> dbValueFutures;
					for(int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, keys.size()); i++)
						dbValueFutures.push_back(transaction->get(keys[i]));

					wait(waitForAll(dbValueFutures));

					for(int i = 0; i < dbValueFutures.size(); i++)
						values.push_back(dbValueFutures[i].get());

					currentIndex += self->maxKeysPerTransaction;

					break;
				}
				catch(Error &e) {
					wait(transaction->onError(e));
				}
			}
		}

		bool result = true;

		//Get the values from the memory store and compare them
		for(int i = 0; i < keys.size(); i++) {
			if(values[i] != self->store.get(keys[i])) {
				result = false;
				break;
			}
		}

		if(!result)
			self->testFailure("Get returned incorrect results");

		return result;
	}

	//Gets a single range of values from the database and memory stores and compares them, returning true if the
	//results were the same
	ACTOR Future<bool> runGetRange(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload *self) {
		//Generate a random range
		Key key = self->selectRandomKey(data, 0.5);
		Key key2 = self->selectRandomKey(data, 0.5);

		state Key start = std::min(key, key2);
		state Key end = std::max(key, key2);

		//Generate a random maximum number of results
		state int limit = deterministicRandom()->randomInt(0, 101);
		state bool reverse = deterministicRandom()->random01() > 0.5 ? false : true;

		//Get the range from memory
		state Standalone<RangeResultRef> storeResults = self->store.getRange(KeyRangeRef(start, end), limit, reverse);

		//Get the range from the database
		state Standalone<RangeResultRef> dbResults;
		state Version readVersion;

		state Reference<TransactionWrapper> transaction = self->createTransaction();

		loop {
			try {
				Version version = wait(transaction->getReadVersion());
				readVersion = version;

				KeyRangeRef range(start, end);
				Standalone<RangeResultRef> rangeResults = wait(transaction->getRange(range, limit, reverse));
				dbResults = rangeResults;
				break;
			}
			catch(Error &e) {
				wait(transaction->onError(e));
			}
		}

		//Compare the ranges
		bool result = self->compareResults(dbResults, storeResults, readVersion);
		if(!result)
			self->testFailure("GetRange returned incorrect results");

		return result;
	}

	//Gets a single range of values using key selectors from the database and memory store and compares them, returning
	//true if the results were the same
	ACTOR Future<bool> runGetRangeSelector(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload *self) {
		KeySelector selectors[2];
		Key keys[2];

		int maxSelectorAttempts = 100;
		int currentSelectorAttempts = 0;

		//Generate a random pair of key selectors and determine the keys they point to
		//Don't use key selectors which would return results outside the key-space of this client unless this client
		//is the first or last client
		for(int i = 0; i < 2; i++) {
			loop {
				//Gradually decrease the maximum offset to increase the likelihood of finding a valid key selector in a small store
				selectors[i] = self->generateKeySelector(data, std::min(100, maxSelectorAttempts - currentSelectorAttempts));
				keys[i] = self->store.getKey(selectors[i]);

				if(keys[i].startsWith(StringRef(self->clientPrefix)) || (keys[i].size() == 0 && self->clientPrefixInt == 0) ||
					(keys[i].startsWith(LiteralStringRef("\xff")) && self->clientPrefixInt == self->clientCount - 1))
				{
					break;
				}

				//Don't loop forever trying to generate valid key selectors if there are no keys in the store
				if(++currentSelectorAttempts == maxSelectorAttempts)
					return true;
			}
		}

		state KeySelector startSelector;
		state KeySelector endSelector;

		state Key startKey;
		state Key endKey;

		//Make sure startKey is less than endKey
		if(keys[0] < keys[1]) {
			startSelector = selectors[0];
			startKey = keys[0];
			endSelector = selectors[1];
			endKey = keys[1];
		}
		else {
			startSelector = selectors[1];
			startKey = keys[1];
			endSelector = selectors[0];
			endKey = keys[0];
		}

		//Choose a random maximum number of results
		state int limit = deterministicRandom()->randomInt(0, 101);
		state bool reverse = deterministicRandom()->random01() < 0.5 ? false : true;

		//Get the range from the memory store
		state Standalone<RangeResultRef> storeResults = self->store.getRange(KeyRangeRef(startKey, endKey), limit, reverse);

		//Get the range from the database
		state Standalone<RangeResultRef> dbResults;

		state Reference<TransactionWrapper> transaction = self->createTransaction();
		state Version readVersion;

		loop {
			try {
				Version version = wait(transaction->getReadVersion());
				readVersion = version;

				Standalone<RangeResultRef> range = wait(transaction->getRange(startSelector, endSelector, limit, reverse));

				if(endKey == self->store.endKey()) {
					for(int i = 0; i < range.size(); i++) {
						//Don't include results in the 0xFF key-space
						if(!range[i].key.startsWith(LiteralStringRef("\xff")))
							dbResults.push_back_deep(dbResults.arena(), range[i]);
					}
					if( reverse && dbResults.size() < storeResults.size() ) {
						storeResults.resize( storeResults.arena(), dbResults.size() );
					}
				}
				else
					dbResults = range;

				break;
			}
			catch(Error &e) {
				wait(transaction->onError(e));
			}
		}

		//Compare the results
		bool result = self->compareResults(dbResults, storeResults, readVersion);

		if(!result)
			self->testFailure("GetRange (KeySelector) returned incorrect results");

		return result;
	}

	//Gets a specified number of keys from the database and memory store and compares them, returning true if all
	//results were the same
	ACTOR Future<bool> runGetKey(VectorRef<KeyValueRef> data, int numGetKeys, ApiCorrectnessWorkload *self) {
		//Generate a set of random key selectors
		state Standalone<VectorRef<KeySelector>> selectors;
		for(int i = 0; i < numGetKeys; i++)
			selectors.push_back(selectors.arena(), self->generateKeySelector(data, 100));

		state Standalone<VectorRef<KeyRef>> keys;

		state int currentIndex = 0;
		while(currentIndex < selectors.size()) {
			//Get the keys from the database
			state Reference<TransactionWrapper> transaction = self->createTransaction();

			loop {
				try {
					state vector<Future<Standalone<KeyRef>>> dbKeyFutures;
					for(int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, selectors.size()); i++)
						dbKeyFutures.push_back(transaction->getKey(selectors[i]));

					wait(waitForAll(dbKeyFutures));

					for(int i = 0; i < dbKeyFutures.size(); i++)
						keys.push_back_deep(keys.arena(), dbKeyFutures[i].get());

					currentIndex += self->maxKeysPerTransaction;

					break;
				}
				catch(Error &e) {
					wait(transaction->onError(e));
				}
			}
		}

		state bool result = true;

		//Get the keys from the memory store and compare them
		state int i;
		for(i = 0; i < selectors.size(); i++) {
			KeyRef key = self->store.getKey(selectors[i]);
			if(keys[i].startsWith(StringRef(self->clientPrefix)) && keys[i] != key)
				result = false;
			else if(keys[i] < StringRef(self->clientPrefix) && key != self->store.startKey())
				result = false;
			else if(keys[i] > StringRef(self->clientPrefix + "\xff") && key != self->store.endKey())
				result = false;

			//If there was a failure, print some debugging info about the failed key
			if(!result) {
				printf("Bad result for key selector %s: db=%s, mem=%s\n", selectors[i].toString().c_str(), printable(keys[i]).c_str(), printable(key).c_str());
				state int dir = selectors[i].offset > 0 ? 1 : -1;
				state int j;
				for(j = 0; j <= abs(selectors[i].offset); j++) {
					state KeySelector sel = KeySelectorRef(selectors[i].getKey(), selectors[i].orEqual, j * dir);
					state Key storeKey = self->store.getKey(sel);

					state Reference<TransactionWrapper> tr = self->createTransaction();

					state Key dbKey;
					loop {
						try {
							Key key = wait(tr->getKey(sel));
							dbKey = key;
							break;
						}
						catch(Error &e) {
							wait(tr->onError(e));
						}
					}

					if(!(storeKey == self->store.startKey() && dbKey < StringRef(self->clientPrefix)) && !(storeKey == self->store.endKey() && dbKey > StringRef(self->clientPrefix + "\xff")))
						printf("Offset %d: db=%s, mem=%s\n", j * dir, printable(dbKey).c_str(), printable(storeKey).c_str());
				}

				break;
			}
		}

		if(!result)
			self->testFailure("GetKey returned incorrect results");

		return result;
	}

	//Clears a specified number of keys from the database and memory store
	ACTOR Future<bool> runClear(VectorRef<KeyValueRef> data, int numClears, ApiCorrectnessWorkload *self)
	{
		//Generate a random set of keys to clear
		state Standalone<VectorRef<Key>> keys;
		for(int i = 0; i < numClears; i++)
			keys.push_back(keys.arena(), self->selectRandomKey(data, 0.9));

		state int currentIndex = 0;
		while(currentIndex < keys.size()) {
			//Clear the keys from the database
			state Reference<TransactionWrapper> transaction = self->createTransaction();
			loop {
				try {
					//For now, make this transaction self-conflicting to avoid commit errors
					Optional<Value> value = wait(transaction->get(keys[0]));

					for(int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, keys.size()); i++) {
						transaction->addReadConflictRange(singleKeyRange(keys[i]));
						transaction->clear(keys[i]);
					}

					wait(transaction->commit());
					for(int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, keys.size()); i++)
						debugMutation("ApiCorrectnessClear", transaction->getCommittedVersion(), MutationRef(MutationRef::DebugKey, keys[i], StringRef()));

					currentIndex += self->maxKeysPerTransaction;
					break;
				}
				catch(Error &e) {
					wait(transaction->onError(e));
				}
			}
		}

		//Clear the keys from the memory store
		for(int i = 0; i < keys.size(); i++) {
			self->store.clear(keys[i]);
			self->debugKey(keys[i], "Clear");
		}

		//Check that the database and memory store are the same
		bool result = wait(self->compareDatabaseToMemory());
		if(!result)
			self->testFailure("Clear resulted in incorrect database");

		return result;
	}

	//Clears a single range of keys from the database and memory store
	ACTOR Future<bool> runClearRange(VectorRef<KeyValueRef> data, ApiCorrectnessWorkload *self) {
		//Generate a random range to clear
		Key key = self->selectRandomKey(data, 0.5);
		Key key2 = self->selectRandomKey(data, 0.5);

		state Key start = std::min(key, key2);
		state Key end = std::max(key, key2);

		//Clear the range in memory
		self->store.clear(KeyRangeRef(start, end));

		//Clear the range in the database
		state Reference<TransactionWrapper> transaction = self->createTransaction();

		loop {
			try {
				//For now, make this transaction self-conflicting to avoid commit errors
				Optional<Value> value = wait(transaction->get(start));

				state KeyRangeRef range(start, end);
				if(!range.empty()) {
					transaction->addReadConflictRange(range);
				}
				transaction->clear(range);
				wait(transaction->commit());
				debugKeyRange("ApiCorrectnessClear", transaction->getCommittedVersion(), range);
				break;
			}
			catch(Error &e) {
				wait(transaction->onError(e));
			}
		}

		self->debugKey(KeyRangeRef(start, end), "ClearRange");

		//Check that the database and memory store are the same
		bool result = wait(self->compareDatabaseToMemory());
		if(!result)
			self->testFailure("Clear (range) resulted in incorrect database");

		return result;
	}

};

WorkloadFactory<ApiCorrectnessWorkload> ApiCorrectnessWorkloadFactory("ApiCorrectness");