wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Renamed members of UnitTestParameters to look cleaner. Added getDouble(). Updated more Redwood unit test parameters to be initialized from params.

This commit is contained in:
Steve Atherton 2021-04-07 18:14:44 -07:00
parent b4e42476b7
commit 434f41a093
6 changed files with 105 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
fdbserver/VersionedBTree.actor.cpp
View File

@ -7029,7 +7029,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") {
bytes += deltaTest(a, b);
}
double elapsed = timer() - start;
printf("DeltaTest() on random large records %g M/s %g MB/s\n", count / elapsed / 1e6, bytes / elapsed / 1e6);
printf("DeltaTest() on random large records %f M/s %f MB/s\n", count / elapsed / 1e6, bytes / elapsed / 1e6);
keyBuffer.resize(30);
valueBuffer.resize(100);
@ -7041,7 +7041,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") {
RedwoodRecordRef b = randomRedwoodRecordRef(keyBuffer, valueBuffer);
bytes += deltaTest(a, b);
}
printf("DeltaTest() on random small records %g M/s %g MB/s\n", count / elapsed / 1e6, bytes / elapsed / 1e6);
printf("DeltaTest() on random small records %f M/s %f MB/s\n", count / elapsed / 1e6, bytes / elapsed / 1e6);
RedwoodRecordRef rec1;
RedwoodRecordRef rec2;
@ -7058,7 +7058,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") {
for (i = 0; i < count; ++i) {
total += rec1.getCommonPrefixLen(rec2, 50);
}
printf("%" PRId64 " getCommonPrefixLen(skip=50) %g M/s\n", total, count / (timer() - start) / 1e6);
printf("%" PRId64 " getCommonPrefixLen(skip=50) %f M/s\n", total, count / (timer() - start) / 1e6);
start = timer();
total = 0;
@ -7066,7 +7066,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") {
for (i = 0; i < count; ++i) {
total += rec1.getCommonPrefixLen(rec2, 0);
}
printf("%" PRId64 " getCommonPrefixLen(skip=0) %g M/s\n", total, count / (timer() - start) / 1e6);
printf("%" PRId64 " getCommonPrefixLen(skip=0) %f M/s\n", total, count / (timer() - start) / 1e6);
char buf[1000];
RedwoodRecordRef::Delta& d = *(RedwoodRecordRef::Delta*)buf;
@ -7079,7 +7079,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") {
for (i = 0; i < count; ++i) {
total += rec1.writeDelta(d, rec2, commonPrefix);
}
printf("%" PRId64 " writeDelta(commonPrefix=%d) %g M/s\n", total, commonPrefix, count / (timer() - start) / 1e6);
printf("%" PRId64 " writeDelta(commonPrefix=%d) %f M/s\n", total, commonPrefix, count / (timer() - start) / 1e6);
start = timer();
total = 0;
@ -7087,7 +7087,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") {
for (i = 0; i < count; ++i) {
total += rec1.writeDelta(d, rec2);
}
printf("%" PRId64 " writeDelta() %g M/s\n", total, count / (timer() - start) / 1e6);
printf("%" PRId64 " writeDelta() %f M/s\n", total, count / (timer() - start) / 1e6);
return Void();
}
@ -7647,30 +7647,43 @@ TEST_CASE("/redwood/correctness/btree") {
g_redwoodMetricsActor = Void(); // Prevent trace event metrics from starting
g_redwoodMetrics.clear();
state std::string pagerFile = "unittest_pageFile.redwood";
state std::string fileName = params.get("fileName").orDefault("unittest_pageFile.redwood");
IPager2* pager;
state bool serialTest = deterministicRandom()->coinflip();
state bool shortTest = deterministicRandom()->coinflip();
state bool serialTest = params.getInt("serialTest").orDefault(deterministicRandom()->coinflip());
state bool shortTest = params.getInt("shortTest").orDefault(deterministicRandom()->coinflip());
state int pageSize =
shortTest ? 200 : (deterministicRandom()->coinflip() ? 4096 : deterministicRandom()->randomInt(200, 400));
state int64_t targetPageOps = shortTest ? 50000 : 1000000;
state bool pagerMemoryOnly = shortTest && (deterministicRandom()->random01() < .001);
state int maxKeySize = deterministicRandom()->randomInt(1, pageSize * 2);
state int maxValueSize = randomSize(pageSize * 25);
state int maxCommitSize = shortTest ? 1000 : randomSize(std::min<int>((maxKeySize + maxValueSize) * 20000, 10e6));
state double clearProbability = deterministicRandom()->random01() * .1;
state double clearSingleKeyProbability = deterministicRandom()->random01();
state double clearPostSetProbability = deterministicRandom()->random01() * .1;
state double coldStartProbability = pagerMemoryOnly ? 0 : (deterministicRandom()->random01() * 0.3);
state double advanceOldVersionProbability = deterministicRandom()->random01();
state int64_t targetPageOps = params.getInt("targetPageOps").orDefault(shortTest ? 50000 : 1000000);
state bool pagerMemoryOnly =
params.getInt("pagerMemoryOnly").orDefault(shortTest && (deterministicRandom()->random01() < .001));
state int maxKeySize = params.getInt("maxKeySize").orDefault(deterministicRandom()->randomInt(1, pageSize * 2));
state int maxValueSize = params.getInt("maxValueSize").orDefault(randomSize(pageSize * 25));
state int maxCommitSize =
params.getInt("maxCommitSize")
.orDefault(shortTest ? 1000 : randomSize(std::min<int>((maxKeySize + maxValueSize) * 20000, 10e6)));
state double clearProbability =
params.getDouble("clearProbability").orDefault(deterministicRandom()->random01() * .1);
state double clearSingleKeyProbability =
params.getDouble("clearSingleKeyProbability").orDefault(deterministicRandom()->random01());
state double clearPostSetProbability =
params.getDouble("clearPostSetProbability").orDefault(deterministicRandom()->random01() * .1);
state double coldStartProbability = params.getDouble("coldStartProbability")
.orDefault(pagerMemoryOnly ? 0 : (deterministicRandom()->random01() * 0.3));
state double advanceOldVersionProbability =
params.getDouble("advanceOldVersionProbability").orDefault(deterministicRandom()->random01());
state int64_t cacheSizeBytes =
pagerMemoryOnly ? 2e9 : (pageSize * deterministicRandom()->randomInt(1, (BUGGIFY ? 2 : 10000) + 1));
state Version versionIncrement = deterministicRandom()->randomInt64(1, 1e8);
state Version remapCleanupWindow = BUGGIFY ? 0 : deterministicRandom()->randomInt64(1, versionIncrement * 50);
state int maxVerificationMapEntries = 300e3;
params.getInt("cacheSizeBytes")
.orDefault(pagerMemoryOnly ? 2e9
: (pageSize * deterministicRandom()->randomInt(1, (BUGGIFY ? 2 : 10000) + 1)));
state Version versionIncrement =
params.getInt("versionIncrement").orDefault(deterministicRandom()->randomInt64(1, 1e8));
state Version remapCleanupWindow =
params.getInt("remapCleanupWindow")
.orDefault(BUGGIFY ? 0 : deterministicRandom()->randomInt64(1, versionIncrement * 50));
state int maxVerificationMapEntries = params.getInt("maxVerificationMapEntries").orDefault(300e3);
printf("\n");
printf("targetPageOps: %" PRId64 "\n", targetPageOps);
@ -7693,11 +7706,11 @@ TEST_CASE("/redwood/correctness/btree") {
printf("\n");
printf("Deleting existing test data...\n");
deleteFile(pagerFile);
deleteFile(fileName);
printf("Initializing...\n");
pager = new DWALPager(pageSize, pagerFile, cacheSizeBytes, remapCleanupWindow, pagerMemoryOnly);
state VersionedBTree* btree = new VersionedBTree(pager, pagerFile);
pager = new DWALPager(pageSize, fileName, cacheSizeBytes, remapCleanupWindow, pagerMemoryOnly);
state VersionedBTree* btree = new VersionedBTree(pager, fileName);
wait(btree->init());
state std::map<std::pair<std::string, Version>, Optional<std::string>> written;
@ -7900,8 +7913,8 @@ TEST_CASE("/redwood/correctness/btree") {
wait(closedFuture);
printf("Reopening btree from disk.\n");
IPager2* pager = new DWALPager(pageSize, pagerFile, cacheSizeBytes, remapCleanupWindow);
btree = new VersionedBTree(pager, pagerFile);
IPager2* pager = new DWALPager(pageSize, fileName, cacheSizeBytes, remapCleanupWindow);
btree = new VersionedBTree(pager, fileName);
wait(btree->init());
Version v = btree->getLatestVersion();
@ -7937,7 +7950,7 @@ TEST_CASE("/redwood/correctness/btree") {
state Future<Void> closedFuture = btree->onClosed();
btree->close();
wait(closedFuture);
btree = new VersionedBTree(new DWALPager(pageSize, pagerFile, cacheSizeBytes, 0), pagerFile);
btree = new VersionedBTree(new DWALPager(pageSize, fileName, cacheSizeBytes, 0), fileName);
wait(btree->init());
wait(btree->clearAllAndCheckSanity());
@ -8045,22 +8058,22 @@ TEST_CASE(":/redwood/performance/set") {
deleteFile(pagerFile);
}
state int pageSize = params.getIntParam("pageSize").orDefault(SERVER_KNOBS->REDWOOD_DEFAULT_PAGE_SIZE);
state int64_t pageCacheBytes = params.getIntParam("pageCacheBytes").orDefault(FLOW_KNOBS->PAGE_CACHE_4K);
state int nodeCount = params.getIntParam("nodeCount").orDefault(1e9);
state int maxRecordsPerCommit = params.getIntParam("maxRecordsPerCommit").orDefault(20000);
state int maxKVBytesPerCommit = params.getIntParam("maxKVBytesPerCommit").orDefault(20e6);
state int64_t kvBytesTarget = params.getIntParam("kvBytesTarget").orDefault(4e9);
state int minKeyPrefixBytes = params.getIntParam("minKeyPrefixBytes").orDefault(25);
state int maxKeyPrefixBytes = params.getIntParam("maxKeyPrefixBytes").orDefault(25);
state int minValueSize = params.getIntParam("minValueSize").orDefault(100);
state int maxValueSize = params.getIntParam("maxValueSize").orDefault(500);
state int minConsecutiveRun = params.getIntParam("minConsecutiveRun").orDefault(1);
state int maxConsecutiveRun = params.getIntParam("maxConsecutiveRun").orDefault(100);
state char firstKeyChar = params.getParam("firstKeyChar").orDefault("a")[0];
state char lastKeyChar = params.getParam("lastKeyChar").orDefault("m")[0];
state int pageSize = params.getInt("pageSize").orDefault(SERVER_KNOBS->REDWOOD_DEFAULT_PAGE_SIZE);
state int64_t pageCacheBytes = params.getInt("pageCacheBytes").orDefault(FLOW_KNOBS->PAGE_CACHE_4K);
state int nodeCount = params.getInt("nodeCount").orDefault(1e9);
state int maxRecordsPerCommit = params.getInt("maxRecordsPerCommit").orDefault(20000);
state int maxKVBytesPerCommit = params.getInt("maxKVBytesPerCommit").orDefault(20e6);
state int64_t kvBytesTarget = params.getInt("kvBytesTarget").orDefault(4e9);
state int minKeyPrefixBytes = params.getInt("minKeyPrefixBytes").orDefault(25);
state int maxKeyPrefixBytes = params.getInt("maxKeyPrefixBytes").orDefault(25);
state int minValueSize = params.getInt("minValueSize").orDefault(100);
state int maxValueSize = params.getInt("maxValueSize").orDefault(500);
state int minConsecutiveRun = params.getInt("minConsecutiveRun").orDefault(1);
state int maxConsecutiveRun = params.getInt("maxConsecutiveRun").orDefault(100);
state char firstKeyChar = params.get("firstKeyChar").orDefault("a")[0];
state char lastKeyChar = params.get("lastKeyChar").orDefault("m")[0];
state Version remapCleanupWindow =
params.getIntParam("remapCleanupWindow").orDefault(SERVER_KNOBS->REDWOOD_REMAP_CLEANUP_WINDOW);
params.getInt("remapCleanupWindow").orDefault(SERVER_KNOBS->REDWOOD_REMAP_CLEANUP_WINDOW);
printf("pageSize: %d\n", pageSize);
printf("pageCacheBytes: %" PRId64 "\n", pageCacheBytes);
@ -8543,10 +8556,10 @@ ACTOR Future<Void> doPrefixInsertComparison(int suffixSize,
}
TEST_CASE(":/redwood/performance/prefixSizeComparison") {
state int suffixSize = 12;
state int valueSize = 100;
state int recordCountTarget = 100e6;
state int usePrefixesInOrder = false;
state int suffixSize = params.getInt("suffixSize").orDefault(12);
state int valueSize = params.getInt("valueSize").orDefault(100);
state int recordCountTarget = params.getInt("recordCountTarget").orDefault(100e6);
state bool usePrefixesInOrder = params.getInt("usePrefixesInOrder").orDefault(0);
wait(doPrefixInsertComparison(
suffixSize, valueSize, recordCountTarget, usePrefixesInOrder, KVSource({ { 10, 100000 } })));
@ -8564,9 +8577,9 @@ TEST_CASE(":/redwood/performance/prefixSizeComparison") {
}
TEST_CASE(":/redwood/performance/sequentialInsert") {
state int prefixLen = 30;
state int valueSize = 100;
state int recordCountTarget = 100e6;
state int prefixLen = params.getInt("prefixLen").orDefault(30);
state int valueSize = params.getInt("valueSize").orDefault(100);
state int recordCountTarget = params.getInt("recordCountTarget").orDefault(100e6);
deleteFile("test.redwood");
wait(delay(5));

2
fdbserver/fdbserver.actor.cpp
View File

@ -1059,7 +1059,7 @@ private:
fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
testParams.setParam(syn.substr(7), args.OptionArg());
testParams.set(syn.substr(7), args.OptionArg());
break;
}
case OPT_LOCALITY: {

18
fdbserver/networktest.actor.cpp
View File

@ -570,15 +570,15 @@ struct P2PNetworkTest {
// The client will close the connection after a random idleMilliseconds.
// Reads and writes can optionally preceded by random delays, waitReadMilliseconds and waitWriteMilliseconds.
TEST_CASE(":/network/p2ptest") {
state P2PNetworkTest p2p(params.getParam("listenerAddresses").orDefault(""),
params.getParam("remoteAddresses").orDefault(""),
params.getIntParam("connectionsOut").orDefault(1),
params.getParam("requestBytes").orDefault("50:100"),
params.getParam("replyBytes").orDefault("500:1000"),
params.getParam("requests").orDefault("10:10000"),
params.getParam("idleMilliseconds").orDefault("0"),
params.getParam("waitReadMilliseconds").orDefault("0"),
params.getParam("waitWriteMilliseconds").orDefault("0"));
state P2PNetworkTest p2p(params.get("listenerAddresses").orDefault(""),
params.get("remoteAddresses").orDefault(""),
params.getInt("connectionsOut").orDefault(1),
params.get("requestBytes").orDefault("50:100"),
params.get("replyBytes").orDefault("500:1000"),
params.get("requests").orDefault("10:10000"),
params.get("idleMilliseconds").orDefault("0"),
params.get("waitReadMilliseconds").orDefault("0"),
params.get("waitWriteMilliseconds").orDefault("0"));
wait(p2p.run());
return Void();

6
fdbserver/workloads/UnitTests.actor.cpp
View File

@ -48,9 +48,9 @@ struct UnitTestWorkload : TestWorkload {
testRunLimit = getOption(options, LiteralStringRef("maxTestCases"), -1);
// Consume all remaining options as testParams which the unit test can access
for(auto &kv : options) {
if(kv.value.size() != 0) {
testParams.setParam(kv.key.toString(), getOption(options, kv.key, StringRef()).toString());
for (auto& kv : options) {
if (kv.value.size() != 0) {
testParams.set(kv.key.toString(), getOption(options, kv.key, StringRef()).toString());
}
}

24
flow/UnitTest.cpp
View File

@ -27,12 +27,12 @@ UnitTest::UnitTest(const char* name, const char* file, int line, TestFunction fu
g_unittests.tests = this;
}
void UnitTestParameters::setParam(const std::string& name, const std::string& value) {
void UnitTestParameters::set(const std::string& name, const std::string& value) {
printf("setting %s = %s\n", name.c_str(), value.c_str());
params[name] = value;
}
Optional<std::string> UnitTestParameters::getParam(const std::string& name) const {
Optional<std::string> UnitTestParameters::get(const std::string& name) const {
auto it = params.find(name);
if (it != params.end()) {
return it->second;
@ -40,14 +40,26 @@ Optional<std::string> UnitTestParameters::getParam(const std::string& name) cons
return {};
}
void UnitTestParameters::setParam(const std::string& name, int64_t value) {
setParam(name, format("%" PRId64, value));
void UnitTestParameters::set(const std::string& name, int64_t value) {
set(name, format("%" PRId64, value));
};
Optional<int64_t> UnitTestParameters::getIntParam(const std::string& name) const {
auto opt = getParam(name);
void UnitTestParameters::set(const std::string& name, double value) {
set(name, format("%g", value));
};
Optional<int64_t> UnitTestParameters::getInt(const std::string& name) const {
auto opt = get(name);
if (opt.present()) {
return atoll(opt.get().c_str());
}
return {};
}
Optional<double> UnitTestParameters::getDouble(const std::string& name) const {
auto opt = get(name);
if (opt.present()) {
return atof(opt.get().c_str());
}
return {};
}

14
flow/UnitTest.h
View File

@ -52,16 +52,22 @@ struct UnitTestParameters {
std::map<std::string, std::string> params;
// Set a named parameter to a string value, replacing any existing value
void setParam(const std::string& name, const std::string& value);
void set(const std::string& name, const std::string& value);
// Set a named parameter to an integer converted to a string value, replacing any existing value
void setParam(const std::string& name, int64_t value);
void set(const std::string& name, int64_t value);
// Set a named parameter to a double converted to a string value, replacing any existing value
void set(const std::string& name, double value);
// Get a parameter's value, will return !present() if parameter was not set
Optional<std::string> getParam(const std::string& name) const;
Optional<std::string> get(const std::string& name) const;
// Get a parameter's value as an integer, will return !present() if parameter was not set
Optional<int64_t> getIntParam(const std::string& name) const;
Optional<int64_t> getInt(const std::string& name) const;
// Get a parameter's value parsed as a double, will return !present() if parameter was not set
Optional<double> getDouble(const std::string& name) const;
};
// Unit test definition structured as a linked list item