Update various workloads to specify that they are expecting raw access. A couple other raw access related fixes. Disable tenant tests in backup and DR tests for now.

This commit is contained in:
A.J. Beamon 2022-03-01 08:21:43 -08:00
parent 592f31755e
commit 4b521b38cb
7 changed files with 99 additions and 15 deletions

View File

@ -2295,6 +2295,14 @@ ACTOR void setupAndRun(std::string dataFolder,
testConfig.storageEngineExcludeTypes.push_back(4);
}
// Disable the default tenant in backup and DR tests for now. This is because backup does not currently duplicate
// the tenant map and related state.
// TODO: reenable when backup/DR supports tenants.
if (std::string_view(testFile).find("Backup") != std::string_view::npos ||
std::string_view(testFile).find("Switchover") != std::string_view::npos) {
allowDefaultTenant = false;
}
// The RocksDB engine is not always built with the rest of fdbserver. Don't try to use it if it is not included
// in the build.
if (!rocksDBEnabled) {

View File

@ -37,6 +37,8 @@
ACTOR Future<std::pair<Standalone<VectorRef<KeyValueRef>>, Version>> readDatabase(Database cx) {
state Transaction tr(cx);
loop {
// Change feeds do not currently support tenant based access
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
state Standalone<VectorRef<KeyValueRef>> output;
state Version readVersion;
try {

View File

@ -20,6 +20,7 @@
#include <cstdint>
#include <limits>
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/ManagementAPI.actor.h"
#include "fdbserver/MoveKeys.actor.h"
@ -105,6 +106,7 @@ struct DataLossRecoveryWorkload : TestWorkload {
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
state Optional<Value> res = wait(timeoutError(tr.get(key), 30.0));
const bool equal = !expectedValue.isError() && res == expectedValue.get();
if (!equal) {
@ -126,6 +128,7 @@ struct DataLossRecoveryWorkload : TestWorkload {
state Transaction tr(cx);
loop {
try {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
if (value.present()) {
tr.set(key, value.get());
} else {
@ -193,6 +196,7 @@ struct DataLossRecoveryWorkload : TestWorkload {
loop {
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
BinaryWriter wrMyOwner(Unversioned());
wrMyOwner << owner;
tr.set(moveKeysLockOwnerKey, wrMyOwner.toValue());
@ -228,6 +232,7 @@ struct DataLossRecoveryWorkload : TestWorkload {
state Transaction validateTr(cx);
loop {
try {
validateTr.setOption(FDBTransactionOptions::RAW_ACCESS);
Standalone<VectorRef<const char*>> addresses = wait(validateTr.getAddressesForKey(keys.begin));
// The move function is not what we are testing here, crash the test if the move fails.
ASSERT(addresses.size() == 1);

View File

@ -262,6 +262,25 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
// m.push_back( retries.getMetric() );
}
// Prevent a write only transaction whose commit was previously cancelled from being reordered after this
// transaction
ACTOR Future<Void> writeBarrier(Reference<IDatabase> db) {
state Reference<ITransaction> tr = db->createTransaction();
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
// Write-only transactions have a self-conflict in the system keys
tr->addWriteConflictRange(allKeys);
tr->clear(normalKeys);
wait(unsafeThreadFutureToFuture(tr->commit()));
return Void();
} catch (Error& e) {
wait(unsafeThreadFutureToFuture(tr->onError(e)));
}
}
}
ACTOR Future<Void> loadAndRun(FuzzApiCorrectnessWorkload* self) {
state double startTime = now();
state int nodesPerTenant = std::max<int>(1, self->nodes / (self->numTenants + 1));
@ -274,6 +293,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
state int tenantNum = -1;
for (; tenantNum < self->numTenants; ++tenantNum) {
state int i = 0;
wait(self->writeBarrier(self->db));
for (; i < nodesPerTenant; i += keysPerBatch) {
state Reference<ITransaction> tr = tenantNum < 0
? self->db->createTransaction()
@ -283,12 +303,6 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
return Void();
try {
if (i == 0) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
// To prevent a write only transaction whose commit was previously
// cancelled from being reordered after this transaction
tr->addWriteConflictRange(allKeys);
tr->clear(normalKeys);
}
if (self->useSystemKeys)

View File

@ -18,6 +18,7 @@
* limitations under the License.
*/
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbserver/TesterInterface.actor.h"
#include "fdbserver/workloads/workloads.actor.h"
@ -54,6 +55,7 @@ struct LockDatabaseWorkload : TestWorkload {
state Transaction tr(cx);
loop {
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
wait(lockDatabase(&tr, lockID));
state RangeResult data = wait(tr.getRange(normalKeys, 50000));
ASSERT(!data.more);
@ -70,6 +72,7 @@ struct LockDatabaseWorkload : TestWorkload {
loop {
try {
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
Optional<Value> val = wait(tr.get(databaseLockedKey));
if (!val.present())
return Void();

View File

@ -70,6 +70,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
Future<Void> _setup(Database cx, SpecialKeySpaceCorrectnessWorkload* self) {
cx->specialKeySpace = std::make_unique<SpecialKeySpace>();
self->ryw = makeReference<ReadYourWritesTransaction>(cx);
self->ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
self->ryw->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED);
self->ryw->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
self->ryw->setVersion(100);
@ -291,6 +292,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
// begin key outside module range
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
wait(success(tx->getRange(
KeyRangeRef(LiteralStringRef("\xff\xff/transactio"), LiteralStringRef("\xff\xff/transaction0")),
CLIENT_KNOBS->TOO_MANY)));
@ -303,6 +305,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// end key outside module range
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
wait(success(tx->getRange(
KeyRangeRef(LiteralStringRef("\xff\xff/transaction/"), LiteralStringRef("\xff\xff/transaction1")),
CLIENT_KNOBS->TOO_MANY)));
@ -315,6 +318,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// both begin and end outside module range
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
wait(success(tx->getRange(
KeyRangeRef(LiteralStringRef("\xff\xff/transaction"), LiteralStringRef("\xff\xff/transaction1")),
CLIENT_KNOBS->TOO_MANY)));
@ -327,6 +331,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// legal range read using the module range
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
wait(success(tx->getRange(
KeyRangeRef(LiteralStringRef("\xff\xff/transaction/"), LiteralStringRef("\xff\xff/transaction0")),
CLIENT_KNOBS->TOO_MANY)));
@ -337,6 +342,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// cross module read with option turned on
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED);
const KeyRef startKey = LiteralStringRef("\xff\xff/transactio");
const KeyRef endKey = LiteralStringRef("\xff\xff/transaction1");
@ -350,6 +356,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// end keySelector inside module range, *** a tricky corner case ***
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->addReadConflictRange(singleKeyRange(LiteralStringRef("testKey")));
KeySelector begin = KeySelectorRef(readConflictRangeKeysRange.begin, false, 1);
KeySelector end = KeySelectorRef(LiteralStringRef("\xff\xff/transaction0"), false, 0);
@ -361,6 +368,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// No module found error case with keys
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
wait(success(tx->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/A_no_module_related_prefix"),
LiteralStringRef("\xff\xff/I_am_also_not_in_any_module")),
CLIENT_KNOBS->TOO_MANY)));
@ -373,6 +381,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// No module found error with KeySelectors, *** a tricky corner case ***
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
KeySelector begin = KeySelectorRef(LiteralStringRef("\xff\xff/zzz_i_am_not_a_module"), false, 1);
KeySelector end = KeySelectorRef(LiteralStringRef("\xff\xff/zzz_to_be_the_final_one"), false, 2);
wait(success(tx->getRange(begin, end, CLIENT_KNOBS->TOO_MANY)));
@ -385,6 +394,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// begin and end keySelectors clamp up to the boundary of the module
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
const KeyRef key = LiteralStringRef("\xff\xff/cluster_file_path");
KeySelector begin = KeySelectorRef(key, false, 0);
KeySelector end = KeySelectorRef(keyAfter(key), false, 2);
@ -395,6 +405,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
throw;
}
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->addReadConflictRange(singleKeyRange(LiteralStringRef("readKey")));
const KeyRef key = LiteralStringRef("\xff\xff/transaction/a_to_be_the_first");
KeySelector begin = KeySelectorRef(key, false, 0);
@ -408,6 +419,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// Errors introduced by SpecialKeyRangeRWImpl
// Writes are disabled by default
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->set(LiteralStringRef("\xff\xff/I_am_not_a_range_can_be_written"), ValueRef());
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled)
@ -417,6 +429,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// The special key is not in a range that can be called with set
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(LiteralStringRef("\xff\xff/I_am_not_a_range_can_be_written"), ValueRef());
ASSERT(false);
@ -428,6 +441,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// A clear cross two ranges are forbidden
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->clear(KeyRangeRef(SpecialKeySpace::getManagementApiCommandRange("exclude").begin,
SpecialKeySpace::getManagementApiCommandRange("failed").end));
@ -440,6 +454,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// base key of the end key selector not in (\xff\xff, \xff\xff\xff), throw key_outside_legal_range()
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
const KeySelector startKeySelector = KeySelectorRef(LiteralStringRef("\xff\xff/test"), true, -200);
const KeySelector endKeySelector = KeySelectorRef(LiteralStringRef("test"), true, -10);
RangeResult result =
@ -453,6 +468,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// test case when registered range is the same as the underlying module
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
state RangeResult result = wait(tx->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces/"),
LiteralStringRef("\xff\xff/worker_interfaces0")),
CLIENT_KNOBS->TOO_MANY));
@ -480,6 +496,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
state Reference<ReadYourWritesTransaction> referenceTx = makeReference<ReadYourWritesTransaction>(cx);
state bool ryw = deterministicRandom()->coinflip();
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
if (!ryw) {
tx->setOption(FDBTransactionOptions::READ_YOUR_WRITES_DISABLE);
}
@ -630,6 +647,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
state Reference<ReadYourWritesTransaction> tx = makeReference<ReadYourWritesTransaction>(cx);
// test ordered option keys
{
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
for (const std::string& option : SpecialKeySpace::getManagementApiOptionsSet()) {
tx->set(
@ -648,6 +666,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
}
// "exclude" error message shema check
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(LiteralStringRef("Invalid_Network_Address")
.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("exclude")),
@ -676,6 +695,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// "setclass"
{
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
// test getRange
state RangeResult result = wait(tx->getRange(
@ -747,6 +767,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
{
try {
// test getRange
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
state RangeResult class_source_result = wait(tx->getRange(
KeyRangeRef(LiteralStringRef("process/class_source/"), LiteralStringRef("process/class_source0"))
.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::CONFIGURATION).begin),
@ -807,6 +828,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// maske sure we lock the database
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
// lock the database
UID uid = deterministicRandom()->randomUniqueID();
@ -853,6 +875,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
loop {
try {
tx->reset();
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
// unlock the database
tx->clear(SpecialKeySpace::getManagementApiCommandPrefix("lock"));
@ -904,6 +927,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
{
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->clear(SpecialKeySpace::getManagementApiCommandPrefix("consistencycheck"));
wait(tx->commit());
@ -1001,6 +1025,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
loop {
try {
std::string new_processes_key(new_coordinator_process);
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
for (const auto& address : old_coordinators_processes) {
new_processes_key += "," + address;
@ -1071,6 +1096,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
loop {
try {
std::string new_processes_key;
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
for (const auto& address : old_coordinators_processes) {
new_processes_key += new_processes_key.size() ? "," : "";
@ -1127,6 +1153,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
TraceEvent(SevDebug, "AdvanceVersionSuccess").detail("Version", v3);
break;
}
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
// force the cluster to recover at v2
tx->set(SpecialKeySpace::getManagementApiCommandPrefix("advanceversion"), std::to_string(v2));
@ -1192,6 +1219,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// update the sample rate and size limit
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(LiteralStringRef("client_txn_sample_rate")
.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("profile")),
@ -1225,6 +1253,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// Change back to default
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(LiteralStringRef("client_txn_sample_rate")
.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("profile")),
@ -1242,6 +1271,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// Test invalid values
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set((deterministicRandom()->coinflip() ? LiteralStringRef("client_txn_sample_rate")
: LiteralStringRef("client_txn_size_limit"))
@ -1297,6 +1327,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// Make sure setting more than one zone as maintenance will fail
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(Key(deterministicRandom()->randomAlphaNumeric(8))
.withPrefix(SpecialKeySpace::getManagementApiCommandPrefix("maintenance")),
@ -1333,6 +1364,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
state int ignoreSSFailuresRetry = 0;
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->set(ignoreSSFailuresZoneString.withPrefix(
SpecialKeySpace::getManagementApiCommandPrefix("maintenance")),
@ -1371,6 +1403,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// set dd mode to 0 and disable DD for rebalance
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
KeyRef ddPrefix = SpecialKeySpace::getManagementApiCommandPrefix("datadistribution");
tx->set(LiteralStringRef("mode").withPrefix(ddPrefix), LiteralStringRef("0"));
@ -1410,6 +1443,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
// then, clear all changes
loop {
try {
tx->setOption(FDBTransactionOptions::RAW_ACCESS);
tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tx->clear(ignoreSSFailuresZoneString.withPrefix(
SpecialKeySpace::getManagementApiCommandPrefix("maintenance")));
@ -1452,6 +1486,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
try {
Version readVersion = wait(tr1->getReadVersion());
tr2->setVersion(readVersion);
tr1->setOption(FDBTransactionOptions::RAW_ACCESS);
tr1->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
tr2->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
KeyRef ddPrefix = SpecialKeySpace::getManagementApiCommandPrefix("datadistribution");

View File

@ -583,9 +583,29 @@ struct WriteDuringReadWorkload : TestWorkload {
std::string(deterministicRandom()->randomInt(valueSizeRange.first, valueSizeRange.second + 1), 'x'));
}
// Prevent a write only transaction whose commit was previously cancelled from being reordered after this
// transaction
ACTOR Future<Void> writeBarrier(Database cx) {
state Transaction tr(cx);
loop {
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
// Write-only transactions have a self-conflict in the system keys
tr.addWriteConflictRange(allKeys);
wait(tr.commit());
return Void();
} catch (Error& e) {
wait(tr.onError(e));
}
}
}
ACTOR Future<Void> loadAndRun(Database cx, WriteDuringReadWorkload* self) {
state double startTime = now();
loop {
wait(self->writeBarrier(cx));
state int i = 0;
state int keysPerBatch =
std::min<int64_t>(1000,
@ -595,19 +615,16 @@ struct WriteDuringReadWorkload : TestWorkload {
for (; i < self->nodes; i += keysPerBatch) {
state Transaction tr(cx);
loop {
if (now() - startTime > self->testDuration)
return Void();
try {
if (i == 0) {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.addWriteConflictRange(
allKeys); // To prevent a write only transaction whose commit was previously cancelled
// from being reordered after this transaction
tr.clear(normalKeys);
}
if (now() - startTime > self->testDuration)
return Void();
if (self->useSystemKeys)
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (i == 0) {
tr.clear(normalKeys);
}
int end = std::min(self->nodes, i + keysPerBatch);
tr.clear(KeyRangeRef(self->getKeyForIndex(i), self->getKeyForIndex(end)));
self->memoryDatabase.erase(self->memoryDatabase.lower_bound(self->getKeyForIndex(i)),