From eec119e0d0548a71c59be3c41028bfa5e2fd98fb Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Sun, 4 Apr 2021 21:36:05 -0700 Subject: [PATCH 01/60] Added an fdbserver role to run unit tests directly without a cluster or test spec file, and added a unit test parameters concept for passing options into unit tests. Updated p2p network test to use unit test parameters instead of the environment. --- fdbserver/TesterInterface.actor.h | 2 +- fdbserver/fdbserver.actor.cpp | 40 ++++++++++++++++---- fdbserver/networktest.actor.cpp | 63 ++++++++++++++++++------------- fdbserver/tester.actor.cpp | 20 ++++++++-- flow/UnitTest.cpp | 30 +++++++++++++++ flow/UnitTest.h | 20 ++++++++++ 6 files changed, 137 insertions(+), 38 deletions(-) diff --git a/fdbserver/TesterInterface.actor.h b/fdbserver/TesterInterface.actor.h index f5e84a2a58..a59e2244ea 100644 --- a/fdbserver/TesterInterface.actor.h +++ b/fdbserver/TesterInterface.actor.h @@ -135,7 +135,7 @@ ACTOR Future testerServerCore(TesterInterface interf, LocalityData locality); enum test_location_t { TEST_HERE, TEST_ON_SERVERS, TEST_ON_TESTERS }; -enum test_type_t { TEST_TYPE_FROM_FILE, TEST_TYPE_CONSISTENCY_CHECK }; +enum test_type_t { TEST_TYPE_FROM_FILE, TEST_TYPE_CONSISTENCY_CHECK, TEST_TYPE_UNIT_TESTS }; ACTOR Future runTests(Reference connFile, test_type_t whatToRun, diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 545d407953..f59fdcd7c0 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -66,6 +66,7 @@ #include "flow/SystemMonitor.h" #include "flow/TLSConfig.actor.h" #include "flow/Tracing.h" +#include "flow/UnitTest.h" #if defined(__linux__) || defined(__FreeBSD__) #include @@ -88,7 +89,7 @@ enum { OPT_CONNFILE, OPT_SEEDCONNFILE, OPT_SEEDCONNSTRING, OPT_ROLE, OPT_LISTEN, OPT_PUBLICADDR, OPT_DATAFOLDER, OPT_LOGFOLDER, OPT_PARENTPID, OPT_TRACER, OPT_NEWCONSOLE, OPT_NOBOX, OPT_TESTFILE, OPT_RESTARTING, OPT_RESTORING, OPT_RANDOMSEED, OPT_KEY, OPT_MEMLIMIT, OPT_STORAGEMEMLIMIT, OPT_CACHEMEMLIMIT, OPT_MACHINEID, OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_BUILD_FLAGS, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, - OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE, + OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_UNITTESTPARAM, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE, OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE, OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE }; @@ -162,6 +163,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_HELP, "--help", SO_NONE }, { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_UNITTESTPARAM, "--test_", SO_REQ_SEP }, { OPT_LOCALITY, "--locality_", SO_REQ_SEP }, { OPT_TESTSERVERS, "--testservers", SO_REQ_SEP }, { OPT_TEST_ON_SERVERS, "--testonservers", SO_NONE }, @@ -622,16 +624,19 @@ static void printUsage(const char* name, bool devhelp) { printOptionUsage("-h, -?, --help", "Display this help and exit."); if (devhelp) { printf(" --build_flags Print build information and exit.\n"); - printOptionUsage("-r ROLE, --role ROLE", - " Server role (valid options are fdbd, test, multitest," - " simulation, networktestclient, networktestserver, restore" - " consistencycheck, kvfileintegritycheck, kvfilegeneratesums). The default is `fdbd'."); + printOptionUsage( + "-r ROLE, --role ROLE", + " Server role (valid options are fdbd, test, multitest," + " simulation, networktestclient, networktestserver, restore" + " consistencycheck, kvfileintegritycheck, kvfilegeneratesums, unittests). The default is `fdbd'."); #ifdef _WIN32 printOptionUsage("-n, --newconsole", " Create a new console."); printOptionUsage("-q, --no_dialog", " Disable error dialog on crash."); printOptionUsage("--parentpid PID", " Specify a process after whose termination to exit."); #endif - printOptionUsage("-f TESTFILE, --testfile", " Testfile to run, defaults to `tests/default.txt'."); + printOptionUsage("-f TESTFILE, --testfile", + " Testfile to run, defaults to `tests/default.txt'. If role is `unittests', specifies which " + "unit tests to run as a search prefix."); printOptionUsage("-R, --restarting", " Restart a previous simulation that was cleanly shut down."); printOptionUsage("-s SEED, --seed SEED", " Random seed."); printOptionUsage("-k KEY, --key KEY", "Target key for search role."); @@ -651,6 +656,8 @@ static void printUsage(const char* name, bool devhelp) { printOptionUsage("--num_testers NUM", " A multitester will wait for NUM testers before starting" " (defaults to 1)."); + printOptionUsage("--test_PARAMNAME PARAMVALUE", + " Set a UnitTest named parameter to the given value. Names are case sensitive."); #ifdef __linux__ printOptionUsage("--rsssize SIZE", " Turns on automatic heap profiling when RSS memory size exceeds" @@ -922,6 +929,7 @@ enum class ServerRole { SkipListTest, Test, VersionedMapTest, + UnitTests }; struct CLIOptions { std::string commandLine; @@ -1044,6 +1052,15 @@ private: knobs.push_back(std::make_pair(syn, args.OptionArg())); break; } + case OPT_UNITTESTPARAM: { + std::string syn = args.OptionSyntax(); + if (!StringRef(syn).startsWith(LiteralStringRef("--test_"))) { + fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str()); + flushAndExit(FDB_EXIT_ERROR); + } + UnitTestCollection::setParam(syn.substr(7), args.OptionArg()); + break; + } case OPT_LOCALITY: { std::string syn = args.OptionSyntax(); if (!StringRef(syn).startsWith(LiteralStringRef("--locality_"))) { @@ -1102,6 +1119,8 @@ private: role = ServerRole::KVFileGenerateIOLogChecksums; else if (!strcmp(sRole, "consistencycheck")) role = ServerRole::ConsistencyCheck; + else if (!strcmp(sRole, "unittests")) + role = ServerRole::UnitTests; else { fprintf(stderr, "ERROR: Unknown role `%s'\n", sRole); printHelpTeaser(argv[0]); @@ -1461,7 +1480,8 @@ private: return StringRef(addr).startsWith(LiteralStringRef("auto:")); }); if ((role != ServerRole::Simulation && role != ServerRole::CreateTemplateDatabase && - role != ServerRole::KVFileIntegrityCheck && role != ServerRole::KVFileGenerateIOLogChecksums) || + role != ServerRole::KVFileIntegrityCheck && role != ServerRole::KVFileGenerateIOLogChecksums && + role != ServerRole::UnitTests) || autoPublicAddress) { if (seedSpecified && !fileExists(connFile)) { @@ -1994,6 +2014,12 @@ int main(int argc, char* argv[]) { StringRef(), opts.localities)); g_network->run(); + } else if (role == ServerRole::UnitTests) { + setupRunLoopProfiler(); + auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId); + f = stopAfter(runTests( + opts.connectionFile, TEST_TYPE_UNIT_TESTS, TEST_HERE, 1, opts.testFile, StringRef(), opts.localities)); + g_network->run(); } else if (role == ServerRole::CreateTemplateDatabase) { createTemplateDatabase(); } else if (role == ServerRole::NetworkTestClient) { diff --git a/fdbserver/networktest.actor.cpp b/fdbserver/networktest.actor.cpp index 4acb46a2f0..540720e948 100644 --- a/fdbserver/networktest.actor.cpp +++ b/fdbserver/networktest.actor.cpp @@ -517,13 +517,6 @@ struct P2PNetworkTest { self->listeners.size(), self->remotes.size(), self->connectionsOut); - printf("Request size: %s\n", self->requestBytes.toString().c_str()); - printf("Response size: %s\n", self->replyBytes.toString().c_str()); - printf("Requests per outgoing session: %d\n", self->requests.toString().c_str()); - printf("Delay before socket read: %s\n", self->waitReadMilliseconds.toString().c_str()); - printf("Delay before socket write: %s\n", self->waitWriteMilliseconds.toString().c_str()); - printf("Delay before session close: %s\n", self->idleMilliseconds.toString().c_str()); - printf("Send/Recv size %d bytes\n", FLOW_KNOBS->MAX_PACKET_SEND_BYTES); for (auto n : self->remotes) { printf("Remote: %s\n", n.toString().c_str()); @@ -534,6 +527,19 @@ struct P2PNetworkTest { actors.add(incoming(self, el)); } + printf("Request size: %s\n", self->requestBytes.toString().c_str()); + printf("Response size: %s\n", self->replyBytes.toString().c_str()); + printf("Requests per outgoing session: %s\n", self->requests.toString().c_str()); + printf("Delay before socket read: %s\n", self->waitReadMilliseconds.toString().c_str()); + printf("Delay before socket write: %s\n", self->waitWriteMilliseconds.toString().c_str()); + printf("Delay before session close: %s\n", self->idleMilliseconds.toString().c_str()); + printf("Send/Recv size %d bytes\n", FLOW_KNOBS->MAX_PACKET_SEND_BYTES); + + if ((self->remotes.empty() || self->connectionsOut == 0) && self->listeners.empty()) { + printf("No listeners and no remotes or connectionsOut, so there is nothing to do!\n"); + ASSERT((!self->remotes.empty() && (self->connectionsOut > 0)) || !self->listeners.empty()); + } + if (!self->remotes.empty()) { for (int i = 0; i < self->connectionsOut; ++i) { actors.add(outgoing(self)); @@ -549,27 +555,30 @@ struct P2PNetworkTest { Future run() { return run_impl(this); } }; -int getEnvInt(const char* name, int defaultValue = 0) { - const char* val = getenv(name); - return val != nullptr ? atol(val) : defaultValue; -} - -std::string getEnvStr(const char* name, std::string defaultValue = "") { - const char* val = getenv(name); - return val != nullptr ? val : defaultValue; -} - -// TODO: Remove this hacky thing and make a "networkp2ptest" role in fdbserver +// Peer-to-Peer network test. +// One or more instances can be run and set to talk to each other. +// Each instance +// - listens on 0 or more listenerAddresses +// - maintains 0 or more connectionsOut at a time, each to a random choice from remoteAddresses +// Address lists are a string of comma-separated IP:port[:tls] strings. +// +// The other arguments can be specified as "fixedValue" or "minValue:maxValue". +// Each outgoing connection will live for a random requests count. +// Each request will +// - send a random requestBytes sized message +// - wait for a random replyBytes sized response. +// The client will close the connection after a random idleMilliseconds. +// Reads and writes can optionally preceded by random delays, waitReadMilliseconds and waitWriteMilliseconds. TEST_CASE("!p2ptest") { - state P2PNetworkTest p2p(getEnvStr("listenerAddresses", ""), - getEnvStr("remoteAddresses", ""), - getEnvInt("connectionsOut", 0), - getEnvStr("requestBytes", "0"), - getEnvStr("replyBytes", "0"), - getEnvStr("requests", "0"), - getEnvStr("idleMilliseconds", "0"), - getEnvStr("waitReadMilliseconds", "0"), - getEnvStr("waitWriteMilliseconds", "0")); + state P2PNetworkTest p2p(UnitTestCollection::getParam("listenerAddresses").orDefault(""), + UnitTestCollection::getParam("remoteAddresses").orDefault(""), + UnitTestCollection::getIntParam("connectionsOut").orDefault(1), + UnitTestCollection::getParam("requestBytes").orDefault("50:100"), + UnitTestCollection::getParam("replyBytes").orDefault("500:1000"), + UnitTestCollection::getParam("requests").orDefault("10:10000"), + UnitTestCollection::getParam("idleMilliseconds").orDefault("0"), + UnitTestCollection::getParam("waitReadMilliseconds").orDefault("0"), + UnitTestCollection::getParam("waitWriteMilliseconds").orDefault("0")); wait(p2p.run()); return Void(); diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index 839df40999..4307329840 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -763,7 +763,7 @@ ACTOR Future runWorkload(Database cx, std::vector runTests(Reference connFile, auto cc = makeReference>>(); auto ci = makeReference>>(); vector> actors; - actors.push_back(reportErrors(monitorLeader(connFile, cc), "MonitorLeader")); - actors.push_back(reportErrors(extractClusterInterface(cc, ci), "ExtractClusterInterface")); + if (connFile) { + actors.push_back(reportErrors(monitorLeader(connFile, cc), "MonitorLeader")); + actors.push_back(reportErrors(extractClusterInterface(cc, ci), "ExtractClusterInterface")); + } if (whatToRun == TEST_TYPE_CONSISTENCY_CHECK) { TestSpec spec; @@ -1603,6 +1605,18 @@ ACTOR Future runTests(Reference connFile, KeyValueRef(LiteralStringRef("shuffleShards"), LiteralStringRef("true"))); spec.options.push_back_deep(spec.options.arena(), options); testSpecs.push_back(spec); + } else if (whatToRun == TEST_TYPE_UNIT_TESTS) { + TestSpec spec; + Standalone> options; + spec.title = LiteralStringRef("UnitTests"); + spec.startDelay = 0; + spec.useDB = false; + spec.timeout = 0; + options.push_back_deep(options.arena(), + KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("UnitTests"))); + options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("testsMatching"), fileName)); + spec.options.push_back_deep(spec.options.arena(), options); + testSpecs.push_back(spec); } else { ifstream ifs; ifs.open(fileName.c_str(), ifstream::in); diff --git a/flow/UnitTest.cpp b/flow/UnitTest.cpp index 1d383ac00e..83eec8c62a 100644 --- a/flow/UnitTest.cpp +++ b/flow/UnitTest.cpp @@ -26,3 +26,33 @@ UnitTest::UnitTest(const char* name, const char* file, int line, TestFunction fu : name(name), file(file), line(line), func(func), next(g_unittests.tests) { g_unittests.tests = this; } + +UnitTestParameters& UnitTestCollection::params() { + static UnitTestParameters p; + return p; +} + +void UnitTestCollection::setParam(const std::string& name, const std::string& value) { + printf("setting %s = %s\n", name.c_str(), value.c_str()); + params()[name] = value; +} + +Optional UnitTestCollection::getParam(const std::string& name) { + auto it = params().find(name); + if (it != params().end()) { + return it->second; + } + return {}; +} + +void UnitTestCollection::setParam(const std::string& name, int64_t value) { + setParam(name, format("%" PRId64, value)); +}; + +Optional UnitTestCollection::getIntParam(const std::string& name) { + auto opt = getParam(name); + if (opt.present()) { + return atoll(opt.get().c_str()); + } + return {}; +} diff --git a/flow/UnitTest.h b/flow/UnitTest.h index c76344e4bb..92f44084ba 100644 --- a/flow/UnitTest.h +++ b/flow/UnitTest.h @@ -45,6 +45,9 @@ #include "flow/flow.h" +#include + +// Unit test definition structured as a linked list item struct UnitTest { typedef Future (*TestFunction)(); @@ -57,8 +60,25 @@ struct UnitTest { UnitTest(const char* name, const char* file, int line, TestFunction func); }; +// Collection of unit tests in the form of a linked list +typedef std::map UnitTestParameters; struct UnitTestCollection { UnitTest* tests; + + // Map of named case-sensitive parameters available for all unit tests + static UnitTestParameters& params(); + + // Set a named parameter to a string value, replacing any existing value + static void setParam(const std::string& name, const std::string& value); + + // Set a named parameter to an integer converted to a string value, replacing any existing value + static void setParam(const std::string& name, int64_t value); + + // Get a parameter's value, will return !present() if parameter was not set + static Optional getParam(const std::string& name); + + // Get a parameter's value as an integer, will return !present() if parameter was not set + static Optional getIntParam(const std::string& name); }; extern UnitTestCollection g_unittests; From ffeb94ada43edce94b0ec5da0a10b1252c140408 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Sun, 4 Apr 2021 23:33:04 -0700 Subject: [PATCH 02/60] Updated Redwood set unit test to use unit test parameters. --- fdbserver/VersionedBTree.actor.cpp | 32 ++++++++++++++++-------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 8051d956b0..72fbb5eada 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -8045,21 +8045,23 @@ TEST_CASE("!/redwood/performance/set") { deleteFile(pagerFile); } - state int pageSize = SERVER_KNOBS->REDWOOD_DEFAULT_PAGE_SIZE; - state int64_t pageCacheBytes = FLOW_KNOBS->PAGE_CACHE_4K; - state int nodeCount = 1e9; - state int maxRecordsPerCommit = 20000; - state int maxKVBytesPerCommit = 20e6; - state int64_t kvBytesTarget = 4e9; - state int minKeyPrefixBytes = 25; - state int maxKeyPrefixBytes = 25; - state int minValueSize = 100; - state int maxValueSize = 500; - state int minConsecutiveRun = 1; - state int maxConsecutiveRun = 100000; - state char firstKeyChar = 'a'; - state char lastKeyChar = 'm'; - state Version remapCleanupWindow = SERVER_KNOBS->REDWOOD_REMAP_CLEANUP_WINDOW; + state int pageSize = UnitTestCollection::getIntParam("pageSize").orDefault(SERVER_KNOBS->REDWOOD_DEFAULT_PAGE_SIZE); + state int64_t pageCacheBytes = + UnitTestCollection::getIntParam("pageCacheBytes").orDefault(FLOW_KNOBS->PAGE_CACHE_4K); + state int nodeCount = UnitTestCollection::getIntParam("nodeCount").orDefault(1e9); + state int maxRecordsPerCommit = UnitTestCollection::getIntParam("maxRecordsPerCommit").orDefault(20000); + state int maxKVBytesPerCommit = UnitTestCollection::getIntParam("maxKVBytesPerCommit").orDefault(20e6); + state int64_t kvBytesTarget = UnitTestCollection::getIntParam("kvBytesTarget").orDefault(4e9); + state int minKeyPrefixBytes = UnitTestCollection::getIntParam("minKeyPrefixBytes").orDefault(25); + state int maxKeyPrefixBytes = UnitTestCollection::getIntParam("maxKeyPrefixBytes").orDefault(25); + state int minValueSize = UnitTestCollection::getIntParam("minValueSize").orDefault(100); + state int maxValueSize = UnitTestCollection::getIntParam("maxValueSize").orDefault(500); + state int minConsecutiveRun = UnitTestCollection::getIntParam("minConsecutiveRun").orDefault(1); + state int maxConsecutiveRun = UnitTestCollection::getIntParam("maxConsecutiveRun").orDefault(100); + state char firstKeyChar = UnitTestCollection::getParam("firstKeyChar").orDefault("a")[0]; + state char lastKeyChar = UnitTestCollection::getParam("lastKeyChar").orDefault("m")[0]; + state Version remapCleanupWindow = + UnitTestCollection::getIntParam("remapCleanupWindow").orDefault(SERVER_KNOBS->REDWOOD_REMAP_CLEANUP_WINDOW); printf("pageSize: %d\n", pageSize); printf("pageCacheBytes: %" PRId64 "\n", pageCacheBytes); From e7573d546f094bd96e1b2fca0914c6e8d61c77e4 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Mon, 5 Apr 2021 00:03:15 -0700 Subject: [PATCH 03/60] Some unit tests names had a prefixed "!" in order to be excluded from random selection, this has been changed to a ":" as it is less problematic on the command line. Some Redwood unit tests have been enabled for random selection. --- fdbserver/VersionedBTree.actor.cpp | 18 +++++++++--------- fdbserver/networktest.actor.cpp | 2 +- tests/RedwoodCorrectness.txt | 2 +- tests/RedwoodCorrectnessBTree.txt | 2 +- tests/RedwoodCorrectnessPager.txt | 2 +- tests/RedwoodCorrectnessUnits.txt | 2 +- tests/RedwoodPerfPrefixCompression.txt | 2 +- tests/RedwoodPerfSequentialInsert.txt | 2 +- tests/RedwoodPerfSet.txt | 2 +- tests/RedwoodPerfTests.txt | 2 +- tests/rare/RedwoodCorrectnessBTree.toml | 2 +- 11 files changed, 19 insertions(+), 19 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 72fbb5eada..773d3a1cce 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -6956,7 +6956,7 @@ RedwoodRecordRef randomRedwoodRecordRef(const std::string& keyBuffer, const std: return rec; } -TEST_CASE("!/redwood/correctness/unit/RedwoodRecordRef") { +TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") { ASSERT(RedwoodRecordRef::Delta::LengthFormatSizes[0] == 3); ASSERT(RedwoodRecordRef::Delta::LengthFormatSizes[1] == 4); ASSERT(RedwoodRecordRef::Delta::LengthFormatSizes[2] == 6); @@ -7092,7 +7092,7 @@ TEST_CASE("!/redwood/correctness/unit/RedwoodRecordRef") { return Void(); } -TEST_CASE("!/redwood/correctness/unit/deltaTree/RedwoodRecordRef") { +TEST_CASE("/redwood/correctness/unit/deltaTree/RedwoodRecordRef") { // Sanity check on delta tree node format ASSERT(DeltaTree::Node::headerSize(false) == 4); ASSERT(DeltaTree::Node::headerSize(true) == 8); @@ -7271,7 +7271,7 @@ TEST_CASE("!/redwood/correctness/unit/deltaTree/RedwoodRecordRef") { return Void(); } -TEST_CASE("!/redwood/correctness/unit/deltaTree/IntIntPair") { +TEST_CASE("/redwood/correctness/unit/deltaTree/IntIntPair") { const int N = 200; IntIntPair prev = { 1, 0 }; IntIntPair next = { 10000, 10000 }; @@ -7615,7 +7615,7 @@ struct SimpleCounter { std::string toString() { return format("%" PRId64 "/%.2f/%.2f", x, rate() / 1e6, avgRate() / 1e6); } }; -TEST_CASE("!/redwood/performance/mutationBuffer") { +TEST_CASE(":/redwood/performance/mutationBuffer") { // This test uses pregenerated short random keys int count = 10e6; @@ -7643,7 +7643,7 @@ TEST_CASE("!/redwood/performance/mutationBuffer") { return Void(); } -TEST_CASE("!/redwood/correctness/btree") { +TEST_CASE("/redwood/correctness/btree") { g_redwoodMetricsActor = Void(); // Prevent trace event metrics from starting g_redwoodMetrics.clear(); @@ -8003,7 +8003,7 @@ ACTOR Future randomScans(VersionedBTree* btree, return Void(); } -TEST_CASE("!/redwood/correctness/pager/cow") { +TEST_CASE(":/redwood/correctness/pager/cow") { state std::string pagerFile = "unittest_pageFile.redwood"; printf("Deleting old test data\n"); deleteFile(pagerFile); @@ -8030,7 +8030,7 @@ TEST_CASE("!/redwood/correctness/pager/cow") { return Void(); } -TEST_CASE("!/redwood/performance/set") { +TEST_CASE(":/redwood/performance/set") { state SignalableActorCollection actors; g_redwoodMetricsActor = Void(); // Prevent trace event metrics from starting @@ -8543,7 +8543,7 @@ ACTOR Future doPrefixInsertComparison(int suffixSize, return Void(); } -TEST_CASE("!/redwood/performance/prefixSizeComparison") { +TEST_CASE(":/redwood/performance/prefixSizeComparison") { state int suffixSize = 12; state int valueSize = 100; state int recordCountTarget = 100e6; @@ -8564,7 +8564,7 @@ TEST_CASE("!/redwood/performance/prefixSizeComparison") { return Void(); } -TEST_CASE("!/redwood/performance/sequentialInsert") { +TEST_CASE(":/redwood/performance/sequentialInsert") { state int prefixLen = 30; state int valueSize = 100; state int recordCountTarget = 100e6; diff --git a/fdbserver/networktest.actor.cpp b/fdbserver/networktest.actor.cpp index 540720e948..9603fc25cb 100644 --- a/fdbserver/networktest.actor.cpp +++ b/fdbserver/networktest.actor.cpp @@ -569,7 +569,7 @@ struct P2PNetworkTest { // - wait for a random replyBytes sized response. // The client will close the connection after a random idleMilliseconds. // Reads and writes can optionally preceded by random delays, waitReadMilliseconds and waitWriteMilliseconds. -TEST_CASE("!p2ptest") { +TEST_CASE(":/network/p2ptest") { state P2PNetworkTest p2p(UnitTestCollection::getParam("listenerAddresses").orDefault(""), UnitTestCollection::getParam("remoteAddresses").orDefault(""), UnitTestCollection::getIntParam("connectionsOut").orDefault(1), diff --git a/tests/RedwoodCorrectness.txt b/tests/RedwoodCorrectness.txt index fbda6b04f4..6f190f2131 100644 --- a/tests/RedwoodCorrectness.txt +++ b/tests/RedwoodCorrectness.txt @@ -4,4 +4,4 @@ useDB=false testName=UnitTests maxTestCases=0 - testsMatching=!/redwood/correctness/ + testsMatching=/redwood/correctness/ diff --git a/tests/RedwoodCorrectnessBTree.txt b/tests/RedwoodCorrectnessBTree.txt index a2495adb7a..92bb3de164 100644 --- a/tests/RedwoodCorrectnessBTree.txt +++ b/tests/RedwoodCorrectnessBTree.txt @@ -4,4 +4,4 @@ useDB=false testName=UnitTests maxTestCases=0 - testsMatching=!/redwood/correctness/btree + testsMatching=/redwood/correctness/btree diff --git a/tests/RedwoodCorrectnessPager.txt b/tests/RedwoodCorrectnessPager.txt index 13f9ef1961..4b94c21cfc 100644 --- a/tests/RedwoodCorrectnessPager.txt +++ b/tests/RedwoodCorrectnessPager.txt @@ -4,4 +4,4 @@ useDB=false testName=UnitTests maxTestCases=0 - testsMatching=!/redwood/correctness/pager + testsMatching=:/redwood/correctness/pager diff --git a/tests/RedwoodCorrectnessUnits.txt b/tests/RedwoodCorrectnessUnits.txt index d32242f3df..ac56735455 100644 --- a/tests/RedwoodCorrectnessUnits.txt +++ b/tests/RedwoodCorrectnessUnits.txt @@ -4,4 +4,4 @@ useDB=false testName=UnitTests maxTestCases=0 - testsMatching=!/redwood/correctness/unit/ + testsMatching=/redwood/correctness/unit/ diff --git a/tests/RedwoodPerfPrefixCompression.txt b/tests/RedwoodPerfPrefixCompression.txt index 09bb6a30cc..3383a74c2b 100644 --- a/tests/RedwoodPerfPrefixCompression.txt +++ b/tests/RedwoodPerfPrefixCompression.txt @@ -4,4 +4,4 @@ useDB=false testName=UnitTests maxTestCases=0 - testsMatching=!/redwood/performance/prefixSizeComparison + testsMatching=:/redwood/performance/prefixSizeComparison diff --git a/tests/RedwoodPerfSequentialInsert.txt b/tests/RedwoodPerfSequentialInsert.txt index 2e61df3b53..21c7005951 100644 --- a/tests/RedwoodPerfSequentialInsert.txt +++ b/tests/RedwoodPerfSequentialInsert.txt @@ -4,4 +4,4 @@ useDB=false testName=UnitTests maxTestCases=0 - testsMatching=!/redwood/performance/sequentialInsert + testsMatching=:/redwood/performance/sequentialInsert diff --git a/tests/RedwoodPerfSet.txt b/tests/RedwoodPerfSet.txt index 0694fccdce..f720479ac2 100644 --- a/tests/RedwoodPerfSet.txt +++ b/tests/RedwoodPerfSet.txt @@ -4,4 +4,4 @@ useDB=false testName=UnitTests maxTestCases=0 - testsMatching=!/redwood/performance/set + testsMatching=:/redwood/performance/set diff --git a/tests/RedwoodPerfTests.txt b/tests/RedwoodPerfTests.txt index 91675d4b64..8d56ebc823 100644 --- a/tests/RedwoodPerfTests.txt +++ b/tests/RedwoodPerfTests.txt @@ -4,4 +4,4 @@ useDB=false testName=UnitTests maxTestCases=0 - testsMatching=!/redwood/performance/ + testsMatching=:/redwood/performance/ diff --git a/tests/rare/RedwoodCorrectnessBTree.toml b/tests/rare/RedwoodCorrectnessBTree.toml index fea0577ee7..1a7320e416 100644 --- a/tests/rare/RedwoodCorrectnessBTree.toml +++ b/tests/rare/RedwoodCorrectnessBTree.toml @@ -6,4 +6,4 @@ startDelay = 0 [[test.workload]] testName = 'UnitTests' maxTestCases = 0 - testsMatching = '!/redwood/correctness/btree' + testsMatching = ':/redwood/correctness/btree' From b4e42476b726a18b196a31db338db960f2e99c81 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 6 Apr 2021 02:36:10 -0700 Subject: [PATCH 04/60] Unit test parameters are no longer global, they are accessible via a parameter to the unit test and initialized from otherwise unconsumed test options for the UnitTests workload in the test spec or from the fdbserver command line when using the unittests role. --- fdbserver/TesterInterface.actor.h | 4 ++- fdbserver/VersionedBTree.actor.cpp | 31 +++++++++---------- fdbserver/fdbserver.actor.cpp | 13 ++++++-- fdbserver/networktest.actor.cpp | 18 +++++------ fdbserver/tester.actor.cpp | 7 ++++- fdbserver/workloads/UnitTests.actor.cpp | 11 ++++++- flow/UnitTest.cpp | 19 +++++------- flow/UnitTest.h | 41 +++++++++++++------------ flow/actorcompiler/ActorParser.cs | 8 ++++- 9 files changed, 88 insertions(+), 64 deletions(-) diff --git a/fdbserver/TesterInterface.actor.h b/fdbserver/TesterInterface.actor.h index a59e2244ea..c3dea2d445 100644 --- a/fdbserver/TesterInterface.actor.h +++ b/fdbserver/TesterInterface.actor.h @@ -29,6 +29,7 @@ #include "fdbrpc/fdbrpc.h" #include "fdbrpc/PerfMetric.h" #include "fdbclient/NativeAPI.actor.h" +#include "flow/UnitTest.h" #include "flow/actorcompiler.h" // has to be last include struct CheckReply { constexpr static FileIdentifier file_identifier = 11; @@ -143,7 +144,8 @@ ACTOR Future runTests(Reference connFile, int minTestersExpected, std::string fileName = std::string(), StringRef startingConfiguration = StringRef(), - LocalityData locality = LocalityData()); + LocalityData locality = LocalityData(), + UnitTestParameters testOptions = UnitTestParameters()); #include "flow/unactorcompiler.h" #endif diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 773d3a1cce..c12015ade8 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -8045,23 +8045,22 @@ TEST_CASE(":/redwood/performance/set") { deleteFile(pagerFile); } - state int pageSize = UnitTestCollection::getIntParam("pageSize").orDefault(SERVER_KNOBS->REDWOOD_DEFAULT_PAGE_SIZE); - state int64_t pageCacheBytes = - UnitTestCollection::getIntParam("pageCacheBytes").orDefault(FLOW_KNOBS->PAGE_CACHE_4K); - state int nodeCount = UnitTestCollection::getIntParam("nodeCount").orDefault(1e9); - state int maxRecordsPerCommit = UnitTestCollection::getIntParam("maxRecordsPerCommit").orDefault(20000); - state int maxKVBytesPerCommit = UnitTestCollection::getIntParam("maxKVBytesPerCommit").orDefault(20e6); - state int64_t kvBytesTarget = UnitTestCollection::getIntParam("kvBytesTarget").orDefault(4e9); - state int minKeyPrefixBytes = UnitTestCollection::getIntParam("minKeyPrefixBytes").orDefault(25); - state int maxKeyPrefixBytes = UnitTestCollection::getIntParam("maxKeyPrefixBytes").orDefault(25); - state int minValueSize = UnitTestCollection::getIntParam("minValueSize").orDefault(100); - state int maxValueSize = UnitTestCollection::getIntParam("maxValueSize").orDefault(500); - state int minConsecutiveRun = UnitTestCollection::getIntParam("minConsecutiveRun").orDefault(1); - state int maxConsecutiveRun = UnitTestCollection::getIntParam("maxConsecutiveRun").orDefault(100); - state char firstKeyChar = UnitTestCollection::getParam("firstKeyChar").orDefault("a")[0]; - state char lastKeyChar = UnitTestCollection::getParam("lastKeyChar").orDefault("m")[0]; + state int pageSize = params.getIntParam("pageSize").orDefault(SERVER_KNOBS->REDWOOD_DEFAULT_PAGE_SIZE); + state int64_t pageCacheBytes = params.getIntParam("pageCacheBytes").orDefault(FLOW_KNOBS->PAGE_CACHE_4K); + state int nodeCount = params.getIntParam("nodeCount").orDefault(1e9); + state int maxRecordsPerCommit = params.getIntParam("maxRecordsPerCommit").orDefault(20000); + state int maxKVBytesPerCommit = params.getIntParam("maxKVBytesPerCommit").orDefault(20e6); + state int64_t kvBytesTarget = params.getIntParam("kvBytesTarget").orDefault(4e9); + state int minKeyPrefixBytes = params.getIntParam("minKeyPrefixBytes").orDefault(25); + state int maxKeyPrefixBytes = params.getIntParam("maxKeyPrefixBytes").orDefault(25); + state int minValueSize = params.getIntParam("minValueSize").orDefault(100); + state int maxValueSize = params.getIntParam("maxValueSize").orDefault(500); + state int minConsecutiveRun = params.getIntParam("minConsecutiveRun").orDefault(1); + state int maxConsecutiveRun = params.getIntParam("maxConsecutiveRun").orDefault(100); + state char firstKeyChar = params.getParam("firstKeyChar").orDefault("a")[0]; + state char lastKeyChar = params.getParam("lastKeyChar").orDefault("m")[0]; state Version remapCleanupWindow = - UnitTestCollection::getIntParam("remapCleanupWindow").orDefault(SERVER_KNOBS->REDWOOD_REMAP_CLEANUP_WINDOW); + params.getIntParam("remapCleanupWindow").orDefault(SERVER_KNOBS->REDWOOD_REMAP_CLEANUP_WINDOW); printf("pageSize: %d\n", pageSize); printf("pageCacheBytes: %" PRId64 "\n", pageCacheBytes); diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index f59fdcd7c0..325caa10c5 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -978,6 +978,7 @@ struct CLIOptions { Reference connectionFile; Standalone machineId; + UnitTestParameters testParams; static CLIOptions parseArgs(int argc, char* argv[]) { CLIOptions opts; @@ -1058,7 +1059,7 @@ private: fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str()); flushAndExit(FDB_EXIT_ERROR); } - UnitTestCollection::setParam(syn.substr(7), args.OptionArg()); + testParams.setParam(syn.substr(7), args.OptionArg()); break; } case OPT_LOCALITY: { @@ -2017,8 +2018,14 @@ int main(int argc, char* argv[]) { } else if (role == ServerRole::UnitTests) { setupRunLoopProfiler(); auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId); - f = stopAfter(runTests( - opts.connectionFile, TEST_TYPE_UNIT_TESTS, TEST_HERE, 1, opts.testFile, StringRef(), opts.localities)); + f = stopAfter(runTests(opts.connectionFile, + TEST_TYPE_UNIT_TESTS, + TEST_HERE, + 1, + opts.testFile, + StringRef(), + opts.localities, + opts.testParams)); g_network->run(); } else if (role == ServerRole::CreateTemplateDatabase) { createTemplateDatabase(); diff --git a/fdbserver/networktest.actor.cpp b/fdbserver/networktest.actor.cpp index 9603fc25cb..cea0c2fca8 100644 --- a/fdbserver/networktest.actor.cpp +++ b/fdbserver/networktest.actor.cpp @@ -570,15 +570,15 @@ struct P2PNetworkTest { // The client will close the connection after a random idleMilliseconds. // Reads and writes can optionally preceded by random delays, waitReadMilliseconds and waitWriteMilliseconds. TEST_CASE(":/network/p2ptest") { - state P2PNetworkTest p2p(UnitTestCollection::getParam("listenerAddresses").orDefault(""), - UnitTestCollection::getParam("remoteAddresses").orDefault(""), - UnitTestCollection::getIntParam("connectionsOut").orDefault(1), - UnitTestCollection::getParam("requestBytes").orDefault("50:100"), - UnitTestCollection::getParam("replyBytes").orDefault("500:1000"), - UnitTestCollection::getParam("requests").orDefault("10:10000"), - UnitTestCollection::getParam("idleMilliseconds").orDefault("0"), - UnitTestCollection::getParam("waitReadMilliseconds").orDefault("0"), - UnitTestCollection::getParam("waitWriteMilliseconds").orDefault("0")); + state P2PNetworkTest p2p(params.getParam("listenerAddresses").orDefault(""), + params.getParam("remoteAddresses").orDefault(""), + params.getIntParam("connectionsOut").orDefault(1), + params.getParam("requestBytes").orDefault("50:100"), + params.getParam("replyBytes").orDefault("500:1000"), + params.getParam("requests").orDefault("10:10000"), + params.getParam("idleMilliseconds").orDefault("0"), + params.getParam("waitReadMilliseconds").orDefault("0"), + params.getParam("waitWriteMilliseconds").orDefault("0")); wait(p2p.run()); return Void(); diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index 4307329840..5e7ce2b550 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -1572,7 +1572,8 @@ ACTOR Future runTests(Reference connFile, int minTestersExpected, std::string fileName, StringRef startingConfiguration, - LocalityData locality) { + LocalityData locality, + UnitTestParameters testOptions) { state vector testSpecs; auto cc = makeReference>>(); auto ci = makeReference>>(); @@ -1615,6 +1616,10 @@ ACTOR Future runTests(Reference connFile, options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("UnitTests"))); options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("testsMatching"), fileName)); + // Add unit test options as test spec options + for (auto& kv : testOptions.params) { + options.push_back_deep(options.arena(), KeyValueRef(kv.first, kv.second)); + } spec.options.push_back_deep(spec.options.arena(), options); testSpecs.push_back(spec); } else { diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp index 2cb4210d0f..024cfc9973 100644 --- a/fdbserver/workloads/UnitTests.actor.cpp +++ b/fdbserver/workloads/UnitTests.actor.cpp @@ -34,6 +34,7 @@ struct UnitTestWorkload : TestWorkload { bool enabled; std::string testPattern; int testRunLimit; + UnitTestParameters testParams; PerfIntCounter testsAvailable, testsExecuted, testsFailed; PerfDoubleCounter totalWallTime, totalSimTime; @@ -45,6 +46,14 @@ struct UnitTestWorkload : TestWorkload { enabled = !clientId; // only do this on the "first" client testPattern = getOption(options, LiteralStringRef("testsMatching"), Value()).toString(); testRunLimit = getOption(options, LiteralStringRef("maxTestCases"), -1); + + // Consume all remaining options as testParams which the unit test can access + for(auto &kv : options) { + if(kv.value.size() != 0) { + testParams.setParam(kv.key.toString(), getOption(options, kv.key, StringRef()).toString()); + } + } + forceLinkIndexedSetTests(); forceLinkDequeTests(); forceLinkFlowTests(); @@ -94,7 +103,7 @@ struct UnitTestWorkload : TestWorkload { state double start_timer = timer(); try { - wait(test->func()); + wait(test->func(self->testParams)); } catch (Error& e) { ++self->testsFailed; result = e; diff --git a/flow/UnitTest.cpp b/flow/UnitTest.cpp index 83eec8c62a..7303cd33c7 100644 --- a/flow/UnitTest.cpp +++ b/flow/UnitTest.cpp @@ -27,29 +27,24 @@ UnitTest::UnitTest(const char* name, const char* file, int line, TestFunction fu g_unittests.tests = this; } -UnitTestParameters& UnitTestCollection::params() { - static UnitTestParameters p; - return p; -} - -void UnitTestCollection::setParam(const std::string& name, const std::string& value) { +void UnitTestParameters::setParam(const std::string& name, const std::string& value) { printf("setting %s = %s\n", name.c_str(), value.c_str()); - params()[name] = value; + params[name] = value; } -Optional UnitTestCollection::getParam(const std::string& name) { - auto it = params().find(name); - if (it != params().end()) { +Optional UnitTestParameters::getParam(const std::string& name) const { + auto it = params.find(name); + if (it != params.end()) { return it->second; } return {}; } -void UnitTestCollection::setParam(const std::string& name, int64_t value) { +void UnitTestParameters::setParam(const std::string& name, int64_t value) { setParam(name, format("%" PRId64, value)); }; -Optional UnitTestCollection::getIntParam(const std::string& name) { +Optional UnitTestParameters::getIntParam(const std::string& name) const { auto opt = getParam(name); if (opt.present()) { return atoll(opt.get().c_str()); diff --git a/flow/UnitTest.h b/flow/UnitTest.h index 92f44084ba..21d51a158f 100644 --- a/flow/UnitTest.h +++ b/flow/UnitTest.h @@ -47,9 +47,26 @@ #include +struct UnitTestParameters { + // Map of named case-sensitive parameters + std::map params; + + // Set a named parameter to a string value, replacing any existing value + void setParam(const std::string& name, const std::string& value); + + // Set a named parameter to an integer converted to a string value, replacing any existing value + void setParam(const std::string& name, int64_t value); + + // Get a parameter's value, will return !present() if parameter was not set + Optional getParam(const std::string& name) const; + + // Get a parameter's value as an integer, will return !present() if parameter was not set + Optional getIntParam(const std::string& name) const; +}; + // Unit test definition structured as a linked list item struct UnitTest { - typedef Future (*TestFunction)(); + typedef Future (*TestFunction)(const UnitTestParameters& params); const char* name; const char* file; @@ -61,24 +78,8 @@ struct UnitTest { }; // Collection of unit tests in the form of a linked list -typedef std::map UnitTestParameters; struct UnitTestCollection { UnitTest* tests; - - // Map of named case-sensitive parameters available for all unit tests - static UnitTestParameters& params(); - - // Set a named parameter to a string value, replacing any existing value - static void setParam(const std::string& name, const std::string& value); - - // Set a named parameter to an integer converted to a string value, replacing any existing value - static void setParam(const std::string& name, int64_t value); - - // Get a parameter's value, will return !present() if parameter was not set - static Optional getParam(const std::string& name); - - // Get a parameter's value as an integer, will return !present() if parameter was not set - static Optional getIntParam(const std::string& name); }; extern UnitTestCollection g_unittests; @@ -91,17 +92,17 @@ extern UnitTestCollection g_unittests; #ifdef FLOW_DISABLE_UNIT_TESTS -#define TEST_CASE(name) static Future FILE_UNIQUE_NAME(disabled_testcase_func)() +#define TEST_CASE(name) static Future FILE_UNIQUE_NAME(disabled_testcase_func)(const UnitTestParameters& params) #define ACTOR_TEST_CASE(actorname, name) #else #define TEST_CASE(name) \ - static Future FILE_UNIQUE_NAME(testcase_func)(); \ + static Future FILE_UNIQUE_NAME(testcase_func)(const UnitTestParameters& params); \ namespace { \ static UnitTest FILE_UNIQUE_NAME(testcase)(name, __FILE__, __LINE__, &FILE_UNIQUE_NAME(testcase_func)); \ } \ - static Future FILE_UNIQUE_NAME(testcase_func)() + static Future FILE_UNIQUE_NAME(testcase_func)(const UnitTestParameters& params) // ACTOR_TEST_CASE generated by actorcompiler; don't use directly #define ACTOR_TEST_CASE(actorname, name) \ diff --git a/flow/actorcompiler/ActorParser.cs b/flow/actorcompiler/ActorParser.cs index d92bba9d53..f44b4e433f 100644 --- a/flow/actorcompiler/ActorParser.cs +++ b/flow/actorcompiler/ActorParser.cs @@ -535,7 +535,13 @@ namespace actorcompiler actor.testCaseParameters = str(paramRange); actor.name = "flowTestCase" + toks.First().SourceLine; - actor.parameters = new VarDeclaration[] { }; + actor.parameters = new VarDeclaration[] { new VarDeclaration { + name = "params", + type = "UnitTestParameters", + initializer = "", + initializerConstructorSyntax = false + } + }; actor.returnType = "Void"; } From 2a64c227fb60599b2596cfa2512e24debbe2d6f3 Mon Sep 17 00:00:00 2001 From: Jon Fu Date: Wed, 7 Apr 2021 15:59:51 -0400 Subject: [PATCH 05/60] Added options to test config that specify maxtlogversion and array of excluded storage engine types --- fdbserver/SimulatedCluster.actor.cpp | 26 ++++++++++++++----- fdbserver/TesterInterface.actor.h | 7 +++-- fdbserver/tester.actor.cpp | 6 +++-- .../to_6.3.10/CycleTestRestart-1.txt | 3 ++- .../to_6.3.10/CycleTestRestart-2.txt | 3 ++- 5 files changed, 33 insertions(+), 12 deletions(-) diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index e5ce23da2f..f8d9610f32 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "fdbrpc/Locality.h" #include "fdbrpc/simulator.h" #include "fdbclient/DatabaseContext.h" @@ -874,7 +875,9 @@ void SimulationConfig::set_config(std::string config) { StringRef StringRefOf(const char* s) { return StringRef((uint8_t*)s, strlen(s)); } - +// Generates and sets an appropriate configuration for the database according to +// the provided testConfig. Some attributes are randomly generated for more coverage +// of different combinations void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) { set_config("new"); const bool simple = false; // Set true to simplify simulation configs for easier debugging @@ -897,7 +900,9 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) { db.resolverCount = deterministicRandom()->randomInt(1, 7); int storage_engine_type = deterministicRandom()->randomInt(0, 4); // Continuously re-pick the storage engine type if it's the one we want to exclude - while (storage_engine_type == testConfig.storageEngineExcludeType) { + while (std::find(testConfig.storageEngineExcludeTypes.begin(), + testConfig.storageEngineExcludeTypes.end(), + storage_engine_type) != testConfig.storageEngineExcludeTypes.end()) { storage_engine_type = deterministicRandom()->randomInt(0, 4); } switch (storage_engine_type) { @@ -989,11 +994,11 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) { if (deterministicRandom()->random01() < 0.5) { int logSpill = deterministicRandom()->randomInt(TLogSpillType::VALUE, TLogSpillType::END); set_config(format("log_spill:=%d", logSpill)); - int logVersion = deterministicRandom()->randomInt(TLogVersion::MIN_RECRUITABLE, TLogVersion::MAX_SUPPORTED + 1); + int logVersion = deterministicRandom()->randomInt(TLogVersion::MIN_RECRUITABLE, testConfig.maxTLogVersion + 1); set_config(format("log_version:=%d", logVersion)); } else { if (deterministicRandom()->random01() < 0.7) - set_config(format("log_version:=%d", TLogVersion::MAX_SUPPORTED)); + set_config(format("log_version:=%d", testConfig.maxTLogVersion)); if (deterministicRandom()->random01() < 0.5) set_config(format("log_spill:=%d", TLogSpillType::DEFAULT)); } @@ -1663,8 +1668,17 @@ void checkTestConf(const char* testFile, TestConfig* testConfig) { sscanf(value.c_str(), "%d", &testConfig->logAntiQuorum); } - if (attrib == "storageEngineExcludeType") { - sscanf(value.c_str(), "%d", &testConfig->storageEngineExcludeType); + if (attrib == "storageEngineExcludeTypes") { + std::stringstream ss(value); + for (int i; ss >> i;) { + testConfig->storageEngineExcludeTypes.push_back(i); + if (ss.peek() == ',') { + ss.ignore(); + } + } + } + if (attrib == "maxTLogVersion") { + sscanf(value.c_str(), "%d", &testConfig->maxTLogVersion); } } diff --git a/fdbserver/TesterInterface.actor.h b/fdbserver/TesterInterface.actor.h index f5e84a2a58..061128a9a7 100644 --- a/fdbserver/TesterInterface.actor.h +++ b/fdbserver/TesterInterface.actor.h @@ -109,12 +109,15 @@ struct TestConfig { bool startIncompatibleProcess = false; int logAntiQuorum = -1; // Storage Engine Types: Verify match with SimulationConfig::generateNormalConfig - // -1 = None // 0 = "ssd" // 1 = "memory" // 2 = "memory-radixtree-beta" // 3 = "ssd-redwood-experimental" - int storageEngineExcludeType = -1; + // Requires a comma-separated list of numbers WITHOUT whitespaces + std::vector storageEngineExcludeTypes; + // Set the maximum TLog version that can be selected for a test + // Refer to FDBTypes.h::TLogVersion. Defaults to the maximum supported version. + int maxTLogVersion = TLogVersion::MAX_SUPPORTED; }; struct TesterInterface { diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index 839df40999..714747ec32 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -1036,8 +1036,10 @@ std::map> testSpecGlobalKey } }, { "startIncompatibleProcess", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedStartIncompatibleProcess", value); } }, - { "storageEngineExcludeType", - [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedStorageEngineExcludeType", ""); } } + { "storageEngineExcludeTypes", + [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedStorageEngineExcludeTypes", ""); } }, + { "maxTLogVersion", + [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedMaxTLogVersion", ""); } } }; std::map> testSpecTestKeys = { diff --git a/tests/restarting/to_6.3.10/CycleTestRestart-1.txt b/tests/restarting/to_6.3.10/CycleTestRestart-1.txt index 59e764c697..fe2a95fd46 100644 --- a/tests/restarting/to_6.3.10/CycleTestRestart-1.txt +++ b/tests/restarting/to_6.3.10/CycleTestRestart-1.txt @@ -1,4 +1,5 @@ -storageEngineExcludeType=-1 +storageEngineExcludeTypes=-1,-2 +maxTLogVersion=6 testTitle=Clogged clearAfterTest=false testName=Cycle diff --git a/tests/restarting/to_6.3.10/CycleTestRestart-2.txt b/tests/restarting/to_6.3.10/CycleTestRestart-2.txt index ecd3c77b52..8af5b92392 100644 --- a/tests/restarting/to_6.3.10/CycleTestRestart-2.txt +++ b/tests/restarting/to_6.3.10/CycleTestRestart-2.txt @@ -1,4 +1,5 @@ -storageEngineExcludeType=-1 +storageEngineExcludeTypes=-1,-2 +maxTLogVersion=6 testTitle=Clogged runSetup=false testName=Cycle From c27d82cecdd11f91f3b32d924003fb0a72272eb3 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Wed, 7 Apr 2021 16:04:08 -0700 Subject: [PATCH 06/60] tlog recruitment used a degraded LogClass process over a non-degraded TransactionClass process tlog recruitment would not use TransactionClass processes if it fulfulled the required amount with LogClass processes Better master exists did not account for how many times a process had been used when comparing recruitments Better master exists did not account for the fact that tlogs prefer to be in a different dc than the cluster controller RoleFitness comparison did not properly order count before degraded or bestFit betterCount was returning worstFit when worstIsDegraded did not match backupWorker recruitment did not attempt to avoid sharing processes with other roles If any of the commit_proxy, grv_proxy, or resolver are forced to share a process, allow the recruitment for all of them to share to an equal degree, this change allows BetterMasterExists to be refactors as a tuple comparison --- fdbserver/ClusterController.actor.cpp | 491 ++++++++++++++++---------- 1 file changed, 310 insertions(+), 181 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 55770d6f3b..87dcbd7c11 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -407,16 +407,17 @@ public: // This worker is a candidate for TLog recruitment. bool inCCDC = worker_details.interf.locality.dcId() == clusterControllerDcId; + // Prefer recruiting a TransactionClass non-degraded process over a LogClass degraded process + if (worker_details.degraded) { + fitness = std::max(fitness, ProcessClass::GoodFit); + } fitness_workers[std::make_tuple(fitness, id_used[worker_process_id], worker_details.degraded, inCCDC)] .push_back(worker_details); } - // FIXME: it's not clear whether this is necessary. - for (int fitness = ProcessClass::BestFit; fitness != ProcessClass::NeverAssign; fitness++) { - auto fitnessEnum = (ProcessClass::Fitness)fitness; - for (int addingDegraded = 0; addingDegraded < 2; addingDegraded++) { - fitness_workers[std::make_tuple(fitnessEnum, 0, addingDegraded, false)]; - } + // Make sure we check for tlogs at the required size before adding processses from the next fitness level + for (int fitness = ProcessClass::GoodFit; fitness != ProcessClass::NeverAssign; fitness++) { + fitness_workers[std::make_tuple((ProcessClass::Fitness)fitness, 0, true, false)]; } results.reserve(results.size() + id_worker.size()); for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { @@ -432,7 +433,7 @@ public: logServerMap->add(worker.interf.locality, &worker); } - if (logServerSet->size() < (std::get<2>(workerIter->first) ? required : desired)) { + if (logServerSet->size() < (addingDegraded ? required : desired)) { } else if (logServerSet->size() == required || logServerSet->size() <= desired) { if (logServerSet->validate(policy)) { for (auto& object : logServerMap->getObjects()) { @@ -742,29 +743,40 @@ public: return results; } + // Allows the comparison of two different recruitments to determine which one is better + // Tlog recruitment is different from all the other roles, in that it avoids degraded processes + // And tried to avoid recruitment in the same DC as the cluster controller struct RoleFitness { ProcessClass::Fitness bestFit; ProcessClass::Fitness worstFit; ProcessClass::ClusterRole role; int count; + int worstUsed; bool worstIsDegraded; + bool inClusterControllerDC; RoleFitness(int bestFit, int worstFit, int count, ProcessClass::ClusterRole role) : bestFit((ProcessClass::Fitness)bestFit), worstFit((ProcessClass::Fitness)worstFit), count(count), - role(role), worstIsDegraded(false) {} + role(role), worstUsed(1), worstIsDegraded(false), inClusterControllerDC(false) {} RoleFitness(int fitness, int count, ProcessClass::ClusterRole role) : bestFit((ProcessClass::Fitness)fitness), worstFit((ProcessClass::Fitness)fitness), count(count), role(role), - worstIsDegraded(false) {} + worstUsed(1), worstIsDegraded(false), inClusterControllerDC(false) {} RoleFitness() : bestFit(ProcessClass::NeverAssign), worstFit(ProcessClass::NeverAssign), role(ProcessClass::NoRole), - count(0), worstIsDegraded(false) {} + count(0), worstUsed(1), worstIsDegraded(false), inClusterControllerDC(false) {} - RoleFitness(vector workers, ProcessClass::ClusterRole role) : role(role) { - worstFit = ProcessClass::GoodFit; + RoleFitness(const vector& workers, + ProcessClass::ClusterRole role, + const std::map>, int>& id_used, + Optional> ccDcId) + : role(role) { + worstFit = ProcessClass::BestFit; worstIsDegraded = false; + inClusterControllerDC = false; bestFit = ProcessClass::NeverAssign; + worstUsed = 1; for (auto& it : workers) { auto thisFit = it.processClass.machineClassFitness(role); if (thisFit > worstFit) { @@ -774,7 +786,24 @@ public: worstIsDegraded = worstIsDegraded || it.degraded; } bestFit = std::min(bestFit, thisFit); + auto thisUsed = id_used.find(it.interf.locality.processId()); + if (thisUsed == id_used.end()) { + TraceEvent(SevError, "UsedNotFound").detail("ProcessId", it.interf.locality.processId().get()); + ASSERT(false); + } + if (thisUsed->second == 0) { + TraceEvent(SevError, "UsedIsZero").detail("ProcessId", it.interf.locality.processId().get()); + ASSERT(false); + } + worstUsed = std::max(worstUsed, thisUsed->second); + // only tlogs avoid the cluster controller dc + if (role == ProcessClass::TLog && it.interf.locality.dcId() == ccDcId) { + inClusterControllerDC = true; + } } + // Every recruitment will attempt to recruit the preferred amount through GoodFit, + // So a recruitment which only has BestFit is not better than one that has a GoodFit process + worstFit = std::max(worstFit, ProcessClass::GoodFit); count = workers.size(); // degraded is only used for recruitment of tlogs if (role != ProcessClass::TLog) { @@ -785,87 +814,45 @@ public: bool operator<(RoleFitness const& r) const { if (worstFit != r.worstFit) return worstFit < r.worstFit; + if (worstUsed != r.worstUsed) + return worstUsed < r.worstUsed; + if (count != r.count) + return count > r.count; if (worstIsDegraded != r.worstIsDegraded) return r.worstIsDegraded; + if (inClusterControllerDC != r.inClusterControllerDC) + return r.inClusterControllerDC; // FIXME: TLog recruitment process does not guarantee the best fit is not worsened. if (role != ProcessClass::TLog && role != ProcessClass::LogRouter && bestFit != r.bestFit) return bestFit < r.bestFit; - return count > r.count; + return false; } bool operator>(RoleFitness const& r) const { return r < *this; } bool operator<=(RoleFitness const& r) const { return !(*this > r); } bool operator>=(RoleFitness const& r) const { return !(*this < r); } - bool betterFitness(RoleFitness const& r) const { - if (worstFit != r.worstFit) - return worstFit < r.worstFit; - if (worstIsDegraded != r.worstIsDegraded) - return r.worstFit; - if (bestFit != r.bestFit) - return bestFit < r.bestFit; - return false; - } - bool betterCount(RoleFitness const& r) const { if (count > r.count) return true; if (worstFit != r.worstFit) return worstFit < r.worstFit; + if (worstUsed != r.worstUsed) + return worstUsed < r.worstUsed; if (worstIsDegraded != r.worstIsDegraded) - return r.worstFit; + return r.worstIsDegraded; + if (inClusterControllerDC != r.inClusterControllerDC) + return r.inClusterControllerDC; return false; } bool operator==(RoleFitness const& r) const { - return worstFit == r.worstFit && bestFit == r.bestFit && count == r.count && - worstIsDegraded == r.worstIsDegraded; + return worstFit == r.worstFit && worstUsed == r.worstUsed && bestFit == r.bestFit && count == r.count && + worstIsDegraded == r.worstIsDegraded && inClusterControllerDC == r.inClusterControllerDC; } - std::string toString() const { return format("%d %d %d %d", bestFit, worstFit, count, worstIsDegraded); } - }; - - struct RoleFitnessPair { - RoleFitness proxy; - RoleFitness grvProxy; - RoleFitness resolver; - - RoleFitnessPair() {} - RoleFitnessPair(RoleFitness const& proxy, RoleFitness const& grvProxy, RoleFitness const& resolver) - : proxy(proxy), grvProxy(grvProxy), resolver(resolver) {} - - bool operator<(RoleFitnessPair const& r) const { - if (proxy.betterFitness(r.proxy)) { - return true; - } - if (r.proxy.betterFitness(proxy)) { - return false; - } - if (grvProxy.betterFitness(r.grvProxy)) { - return true; - } - if (r.grvProxy.betterFitness(grvProxy)) { - return false; - } - if (resolver.betterFitness(r.resolver)) { - return true; - } - if (r.resolver.betterFitness(resolver)) { - return false; - } - if (proxy.count != r.proxy.count) { - return proxy.count > r.proxy.count; - } - if (grvProxy.count != r.grvProxy.count) { - return grvProxy.count > r.grvProxy.count; - } - return resolver.count > r.resolver.count; - } - bool operator>(RoleFitnessPair const& r) const { return r < *this; } - bool operator<=(RoleFitnessPair const& r) const { return !(*this > r); } - bool operator>=(RoleFitnessPair const& r) const { return !(*this < r); } - - bool operator==(RoleFitnessPair const& r) const { - return proxy == r.proxy && grvProxy == r.grvProxy && resolver == r.resolver; + std::string toString() const { + return format( + "%d %d %d %d %d %d", worstFit, worstUsed, count, worstIsDegraded, inClusterControllerDC, bestFit); } }; @@ -914,9 +901,9 @@ public: if (!goodRemoteRecruitmentTime.isReady() && ((RoleFitness( SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredRemoteLogs(), ProcessClass::TLog) - .betterCount(RoleFitness(remoteLogs, ProcessClass::TLog))) || + .betterCount(RoleFitness(remoteLogs, ProcessClass::TLog, id_used, clusterControllerDcId))) || (RoleFitness(SERVER_KNOBS->EXPECTED_LOG_ROUTER_FITNESS, req.logRouterCount, ProcessClass::LogRouter) - .betterCount(RoleFitness(logRouters, ProcessClass::LogRouter))))) { + .betterCount(RoleFitness(logRouters, ProcessClass::LogRouter, id_used, clusterControllerDcId))))) { throw operation_failed(); } @@ -980,6 +967,13 @@ public: auto first_resolver = getWorkerForRoleInDatacenter( dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, req.configuration, id_used); + // If one of the first process recruitments is forced to share a process, allow all of next recruitments + // to also share a process. + auto maxUsed = std::max(std::max(first_commit_proxy.used, first_grv_proxy.used), first_resolver.used); + first_commit_proxy.used = maxUsed; + first_grv_proxy.used = maxUsed; + first_resolver.used = maxUsed; + auto commit_proxies = getWorkersForRoleInDatacenter(dcId, ProcessClass::CommitProxy, req.configuration.getDesiredCommitProxies(), @@ -1031,24 +1025,24 @@ public: if (!goodRecruitmentTime.isReady() && (RoleFitness(SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs(), ProcessClass::TLog) - .betterCount(RoleFitness(tlogs, ProcessClass::TLog)) || + .betterCount(RoleFitness(tlogs, ProcessClass::TLog, id_used, clusterControllerDcId)) || (region.satelliteTLogReplicationFactor > 0 && req.configuration.usableRegions > 1 && RoleFitness(SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredSatelliteLogs(dcId), ProcessClass::TLog) - .betterCount(RoleFitness(satelliteLogs, ProcessClass::TLog))) || + .betterCount(RoleFitness(satelliteLogs, ProcessClass::TLog, id_used, clusterControllerDcId))) || RoleFitness(SERVER_KNOBS->EXPECTED_COMMIT_PROXY_FITNESS, req.configuration.getDesiredCommitProxies(), ProcessClass::CommitProxy) - .betterCount(RoleFitness(commit_proxies, ProcessClass::CommitProxy)) || + .betterCount(RoleFitness(commit_proxies, ProcessClass::CommitProxy, id_used, clusterControllerDcId)) || RoleFitness(SERVER_KNOBS->EXPECTED_GRV_PROXY_FITNESS, req.configuration.getDesiredGrvProxies(), ProcessClass::GrvProxy) - .betterCount(RoleFitness(grv_proxies, ProcessClass::GrvProxy)) || + .betterCount(RoleFitness(grv_proxies, ProcessClass::GrvProxy, id_used, clusterControllerDcId)) || RoleFitness(SERVER_KNOBS->EXPECTED_RESOLVER_FITNESS, req.configuration.getDesiredResolvers(), ProcessClass::Resolver) - .betterCount(RoleFitness(resolvers, ProcessClass::Resolver)))) { + .betterCount(RoleFitness(resolvers, ProcessClass::Resolver, id_used, clusterControllerDcId)))) { return operation_failed(); } @@ -1149,7 +1143,7 @@ public: auto datacenters = getDatacenters(req.configuration); - RoleFitnessPair bestFitness; + std::tuple bestFitness; int numEquivalent = 1; Optional bestDC; @@ -1165,6 +1159,14 @@ public: auto first_resolver = getWorkerForRoleInDatacenter( dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, req.configuration, used); + // If one of the first process recruitments is forced to share a process, allow all of next + // recruitments to also share a process. + auto maxUsed = + std::max(std::max(first_commit_proxy.used, first_grv_proxy.used), first_resolver.used); + first_commit_proxy.used = maxUsed; + first_grv_proxy.used = maxUsed; + first_resolver.used = maxUsed; + auto commit_proxies = getWorkersForRoleInDatacenter(dcId, ProcessClass::CommitProxy, req.configuration.getDesiredCommitProxies(), @@ -1186,9 +1188,10 @@ public: used, first_resolver); - RoleFitnessPair fitness(RoleFitness(commit_proxies, ProcessClass::CommitProxy), - RoleFitness(grv_proxies, ProcessClass::GrvProxy), - RoleFitness(resolvers, ProcessClass::Resolver)); + auto fitness = std::make_tuple( + RoleFitness(commit_proxies, ProcessClass::CommitProxy, used, clusterControllerDcId), + RoleFitness(grv_proxies, ProcessClass::GrvProxy, used, clusterControllerDcId), + RoleFitness(resolvers, ProcessClass::Resolver, used, clusterControllerDcId)); if (dcId == clusterControllerDcId) { bestFitness = fitness; @@ -1206,7 +1209,7 @@ public: if (req.configuration.backupWorkerEnabled) { const int nBackup = std::max(tlogs.size(), req.maxOldLogRouters); auto backupWorkers = getWorkersForRoleInDatacenter( - dcId, ProcessClass::Backup, nBackup, req.configuration, id_used); + dcId, ProcessClass::Backup, nBackup, req.configuration, used); std::transform(backupWorkers.begin(), backupWorkers.end(), std::back_inserter(result.backupWorkers), @@ -1254,19 +1257,19 @@ public: if (!goodRecruitmentTime.isReady() && (RoleFitness( SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs(), ProcessClass::TLog) - .betterCount(RoleFitness(tlogs, ProcessClass::TLog)) || + .betterCount(RoleFitness(tlogs, ProcessClass::TLog, id_used, clusterControllerDcId)) || RoleFitness(SERVER_KNOBS->EXPECTED_COMMIT_PROXY_FITNESS, req.configuration.getDesiredCommitProxies(), ProcessClass::CommitProxy) - .betterCount(bestFitness.proxy) || + .betterCount(std::get<0>(bestFitness)) || RoleFitness(SERVER_KNOBS->EXPECTED_GRV_PROXY_FITNESS, req.configuration.getDesiredGrvProxies(), ProcessClass::GrvProxy) - .betterCount(bestFitness.grvProxy) || + .betterCount(std::get<1>(bestFitness)) || RoleFitness(SERVER_KNOBS->EXPECTED_RESOLVER_FITNESS, req.configuration.getDesiredResolvers(), ProcessClass::Resolver) - .betterCount(bestFitness.resolver))) { + .betterCount(std::get<2>(bestFitness)))) { throw operation_failed(); } @@ -1337,6 +1340,12 @@ public: } } + void updateIdUsed(const vector& workers, std::map>, int>& id_used) { + for (auto& it : workers) { + id_used[it.interf.locality.processId()]++; + } + } + // FIXME: determine when to fail the cluster controller when a primaryDC has not been set // This function returns true when the cluster controller determines it is worth forcing @@ -1351,6 +1360,7 @@ public: // Do not trigger better master exists if the cluster controller is excluded, since the master will change // anyways once the cluster controller is moved if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) { + TraceEvent("WorseMasterExists", id).detail("Reason", "ClusterControllerExcluded"); return false; } @@ -1363,6 +1373,9 @@ public: // Get master process auto masterWorker = id_worker.find(dbi.master.locality.processId()); if (masterWorker == id_worker.end()) { + TraceEvent("WorseMasterExists", id) + .detail("Reason", "CannotFindMaster") + .detail("ProcessID", dbi.master.locality.processId()); return false; } @@ -1378,10 +1391,18 @@ public: for (auto& logSet : dbi.logSystemConfig.tLogs) { for (auto& it : logSet.tLogs) { auto tlogWorker = id_worker.find(it.interf().filteredLocality.processId()); - if (tlogWorker == id_worker.end()) + if (tlogWorker == id_worker.end()) { + TraceEvent("WorseMasterExists", id) + .detail("Reason", "CannotFindTLog") + .detail("ProcessID", it.interf().filteredLocality.processId()); return false; - if (tlogWorker->second.priorityInfo.isExcluded) + } + if (tlogWorker->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "TLogExcluded") + .detail("ProcessID", it.interf().filteredLocality.processId()); return true; + } if (logSet.isLocal && logSet.locality == tagLocalitySatellite) { satellite_tlogs.push_back(tlogWorker->second.details); @@ -1394,10 +1415,18 @@ public: for (auto& it : logSet.logRouters) { auto tlogWorker = id_worker.find(it.interf().filteredLocality.processId()); - if (tlogWorker == id_worker.end()) + if (tlogWorker == id_worker.end()) { + TraceEvent("WorseMasterExists", id) + .detail("Reason", "CannotFindLogRouter") + .detail("ProcessID", it.interf().filteredLocality.processId()); return false; - if (tlogWorker->second.priorityInfo.isExcluded) + } + if (tlogWorker->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "LogRouterExcluded") + .detail("ProcessID", it.interf().filteredLocality.processId()); return true; + } if (!logRouterAddresses.count(tlogWorker->second.details.interf.address())) { logRouterAddresses.insert(tlogWorker->second.details.interf.address()); log_routers.push_back(tlogWorker->second.details); @@ -1406,10 +1435,18 @@ public: for (const auto& worker : logSet.backupWorkers) { auto workerIt = id_worker.find(worker.interf().locality.processId()); - if (workerIt == id_worker.end()) + if (workerIt == id_worker.end()) { + TraceEvent("WorseMasterExists", id) + .detail("Reason", "CannotFindBackupWorker") + .detail("ProcessID", worker.interf().locality.processId()); return false; - if (workerIt->second.priorityInfo.isExcluded) + } + if (workerIt->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "BackupWorkerExcluded") + .detail("ProcessID", worker.interf().locality.processId()); return true; + } if (backup_addresses.count(workerIt->second.details.interf.address()) == 0) { backup_addresses.insert(workerIt->second.details.interf.address()); backup_workers.push_back(workerIt->second.details); @@ -1421,10 +1458,18 @@ public: std::vector commitProxyClasses; for (auto& it : dbi.client.commitProxies) { auto commitProxyWorker = id_worker.find(it.processId); - if (commitProxyWorker == id_worker.end()) + if (commitProxyWorker == id_worker.end()) { + TraceEvent("WorseMasterExists", id) + .detail("Reason", "CannotFindCommitProxy") + .detail("ProcessID", it.processId); return false; - if (commitProxyWorker->second.priorityInfo.isExcluded) + } + if (commitProxyWorker->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "CommitProxyExcluded") + .detail("ProcessID", it.processId); return true; + } commitProxyClasses.push_back(commitProxyWorker->second.details); } @@ -1432,10 +1477,18 @@ public: std::vector grvProxyClasses; for (auto& it : dbi.client.grvProxies) { auto grvProxyWorker = id_worker.find(it.processId); - if (grvProxyWorker == id_worker.end()) + if (grvProxyWorker == id_worker.end()) { + TraceEvent("WorseMasterExists", id) + .detail("Reason", "CannotFindGrvProxy") + .detail("ProcessID", it.processId); return false; - if (grvProxyWorker->second.priorityInfo.isExcluded) + } + if (grvProxyWorker->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "GrvProxyExcluded") + .detail("ProcessID", it.processId); return true; + } grvProxyClasses.push_back(grvProxyWorker->second.details); } @@ -1443,10 +1496,18 @@ public: std::vector resolverClasses; for (auto& it : dbi.resolvers) { auto resolverWorker = id_worker.find(it.locality.processId()); - if (resolverWorker == id_worker.end()) + if (resolverWorker == id_worker.end()) { + TraceEvent("WorseMasterExists", id) + .detail("Reason", "CannotFindResolver") + .detail("ProcessID", it.locality.processId()); return false; - if (resolverWorker->second.priorityInfo.isExcluded) + } + if (resolverWorker->second.priorityInfo.isExcluded) { + TraceEvent("BetterMasterExists", id) + .detail("Reason", "ResolverExcluded") + .detail("ProcessID", it.locality.processId()); return true; + } resolverClasses.push_back(resolverWorker->second.details); } @@ -1459,7 +1520,9 @@ public: } std::map>, int> id_used; + std::map>, int> old_id_used; id_used[clusterControllerProcessId]++; + old_id_used[clusterControllerProcessId]++; WorkerFitnessInfo mworker = getWorkerForRoleInDatacenter( clusterControllerDcId, ProcessClass::Master, ProcessClass::NeverAssign, db.config, id_used, true); auto newMasterFit = mworker.worker.processClass.machineClassFitness(ProcessClass::Master); @@ -1467,11 +1530,25 @@ public: newMasterFit = std::max(newMasterFit, ProcessClass::ExcludeFit); } - if (oldMasterFit < newMasterFit) + old_id_used[masterWorker->first]++; + if (oldMasterFit < newMasterFit) { + TraceEvent("WorseMasterExists", id) + .detail("OldMasterFit", oldMasterFit) + .detail("NewMasterFit", newMasterFit) + .detail("OldIsCC", dbi.master.locality.processId() == clusterControllerProcessId) + .detail("NewIsCC", mworker.worker.interf.locality.processId() == clusterControllerProcessId); + ; return false; + } if (oldMasterFit > newMasterFit || (dbi.master.locality.processId() == clusterControllerProcessId && - mworker.worker.interf.locality.processId() != clusterControllerProcessId)) + mworker.worker.interf.locality.processId() != clusterControllerProcessId)) { + TraceEvent("BetterMasterExists", id) + .detail("OldMasterFit", oldMasterFit) + .detail("NewMasterFit", newMasterFit) + .detail("OldIsCC", dbi.master.locality.processId() == clusterControllerProcessId) + .detail("NewIsCC", mworker.worker.interf.locality.processId() == clusterControllerProcessId); return true; + } std::set> primaryDC; std::set> remoteDC; @@ -1493,7 +1570,8 @@ public: } // Check tLog fitness - RoleFitness oldTLogFit(tlogs, ProcessClass::TLog); + updateIdUsed(tlogs, old_id_used); + RoleFitness oldTLogFit(tlogs, ProcessClass::TLog, old_id_used, clusterControllerDcId); auto newTLogs = getWorkersForTlogs(db.config, db.config.tLogReplicationFactor, db.config.getDesiredLogs(), @@ -1501,10 +1579,7 @@ public: id_used, true, primaryDC); - RoleFitness newTLogFit(newTLogs, ProcessClass::TLog); - - if (oldTLogFit < newTLogFit) - return false; + RoleFitness newTLogFit(newTLogs, ProcessClass::TLog, id_used, clusterControllerDcId); bool oldSatelliteFallback = false; @@ -1520,13 +1595,16 @@ public: } } - RoleFitness oldSatelliteTLogFit(satellite_tlogs, ProcessClass::TLog); + updateIdUsed(satellite_tlogs, old_id_used); + RoleFitness oldSatelliteTLogFit(satellite_tlogs, ProcessClass::TLog, old_id_used, clusterControllerDcId); bool newSatelliteFallback = false; - auto newSatelliteTLogs = - (region.satelliteTLogReplicationFactor > 0 && db.config.usableRegions > 1) - ? getWorkersForSatelliteLogs(db.config, region, remoteRegion, id_used, newSatelliteFallback, true) - : satellite_tlogs; - RoleFitness newSatelliteTLogFit(newSatelliteTLogs, ProcessClass::TLog); + auto newSatelliteTLogs = satellite_tlogs; + RoleFitness newSatelliteTLogFit = oldSatelliteTLogFit; + if (region.satelliteTLogReplicationFactor > 0 && db.config.usableRegions > 1) { + newSatelliteTLogs = + getWorkersForSatelliteLogs(db.config, region, remoteRegion, id_used, newSatelliteFallback, true); + newSatelliteTLogFit = RoleFitness(newSatelliteTLogs, ProcessClass::TLog, id_used, clusterControllerDcId); + } std::map, int32_t> satellite_priority; for (auto& r : region.satellites) { @@ -1551,55 +1629,72 @@ public: } } - if (oldSatelliteFallback && !newSatelliteFallback) + if (oldSatelliteFallback && !newSatelliteFallback) { + TraceEvent("BetterMasterExists", id) + .detail("OldSatelliteFallback", oldSatelliteFallback) + .detail("NewSatelliteFallback", newSatelliteFallback); return true; - if (!oldSatelliteFallback && newSatelliteFallback) + } + if (!oldSatelliteFallback && newSatelliteFallback) { + TraceEvent("WorseMasterExists", id) + .detail("OldSatelliteFallback", oldSatelliteFallback) + .detail("NewSatelliteFallback", newSatelliteFallback); return false; + } - if (oldSatelliteRegionFit < newSatelliteRegionFit) + if (oldSatelliteRegionFit < newSatelliteRegionFit) { + TraceEvent("BetterMasterExists", id) + .detail("OldSatelliteRegionFit", oldSatelliteRegionFit) + .detail("NewSatelliteRegionFit", newSatelliteRegionFit); return true; - if (oldSatelliteRegionFit > newSatelliteRegionFit) + } + if (oldSatelliteRegionFit > newSatelliteRegionFit) { + TraceEvent("WorseMasterExists", id) + .detail("OldSatelliteRegionFit", oldSatelliteRegionFit) + .detail("NewSatelliteRegionFit", newSatelliteRegionFit); return false; + } - if (oldSatelliteTLogFit < newSatelliteTLogFit) - return false; - - RoleFitness oldRemoteTLogFit(remote_tlogs, ProcessClass::TLog); + updateIdUsed(remote_tlogs, old_id_used); + RoleFitness oldRemoteTLogFit(remote_tlogs, ProcessClass::TLog, old_id_used, clusterControllerDcId); std::vector exclusionWorkerIds; auto fn = [](const WorkerDetails& in) { return in.interf.id(); }; std::transform(newTLogs.begin(), newTLogs.end(), std::back_inserter(exclusionWorkerIds), fn); std::transform(newSatelliteTLogs.begin(), newSatelliteTLogs.end(), std::back_inserter(exclusionWorkerIds), fn); - RoleFitness newRemoteTLogFit( - (db.config.usableRegions > 1 && (dbi.recoveryState == RecoveryState::ALL_LOGS_RECRUITED || - dbi.recoveryState == RecoveryState::FULLY_RECOVERED)) - ? getWorkersForTlogs(db.config, - db.config.getRemoteTLogReplicationFactor(), - db.config.getDesiredRemoteLogs(), - db.config.getRemoteTLogPolicy(), - id_used, - true, - remoteDC, - exclusionWorkerIds) - : remote_tlogs, - ProcessClass::TLog); - if (oldRemoteTLogFit < newRemoteTLogFit) - return false; + RoleFitness newRemoteTLogFit = oldRemoteTLogFit; + if (db.config.usableRegions > 1 && (dbi.recoveryState == RecoveryState::ALL_LOGS_RECRUITED || + dbi.recoveryState == RecoveryState::FULLY_RECOVERED)) { + newRemoteTLogFit = RoleFitness(getWorkersForTlogs(db.config, + db.config.getRemoteTLogReplicationFactor(), + db.config.getDesiredRemoteLogs(), + db.config.getRemoteTLogPolicy(), + id_used, + true, + remoteDC, + exclusionWorkerIds), + ProcessClass::TLog, + id_used, + clusterControllerDcId); + } int oldRouterCount = oldTLogFit.count * std::max(1, db.config.desiredLogRouterCount / std::max(1, oldTLogFit.count)); int newRouterCount = newTLogFit.count * std::max(1, db.config.desiredLogRouterCount / std::max(1, newTLogFit.count)); - RoleFitness oldLogRoutersFit(log_routers, ProcessClass::LogRouter); - RoleFitness newLogRoutersFit( - (db.config.usableRegions > 1 && dbi.recoveryState == RecoveryState::FULLY_RECOVERED) - ? getWorkersForRoleInDatacenter(*remoteDC.begin(), - ProcessClass::LogRouter, - newRouterCount, - db.config, - id_used, - Optional(), - true) - : log_routers, - ProcessClass::LogRouter); + updateIdUsed(log_routers, old_id_used); + RoleFitness oldLogRoutersFit(log_routers, ProcessClass::LogRouter, old_id_used, clusterControllerDcId); + RoleFitness newLogRoutersFit = oldLogRoutersFit; + if (db.config.usableRegions > 1 && dbi.recoveryState == RecoveryState::FULLY_RECOVERED) { + newLogRoutersFit = RoleFitness(getWorkersForRoleInDatacenter(*remoteDC.begin(), + ProcessClass::LogRouter, + newRouterCount, + db.config, + id_used, + Optional(), + true), + ProcessClass::LogRouter, + id_used, + clusterControllerDcId); + } if (oldLogRoutersFit.count < oldRouterCount) { oldLogRoutersFit.worstFit = ProcessClass::NeverAssign; @@ -1607,13 +1702,15 @@ public: if (newLogRoutersFit.count < newRouterCount) { newLogRoutersFit.worstFit = ProcessClass::NeverAssign; } - if (oldLogRoutersFit < newLogRoutersFit) - return false; // Check proxy/grvProxy/resolver fitness - RoleFitnessPair oldInFit(RoleFitness(commitProxyClasses, ProcessClass::CommitProxy), - RoleFitness(grvProxyClasses, ProcessClass::GrvProxy), - RoleFitness(resolverClasses, ProcessClass::Resolver)); + updateIdUsed(commitProxyClasses, old_id_used); + updateIdUsed(grvProxyClasses, old_id_used); + updateIdUsed(resolverClasses, old_id_used); + RoleFitness oldCommitProxyFit( + commitProxyClasses, ProcessClass::CommitProxy, old_id_used, clusterControllerDcId); + RoleFitness oldGrvProxyFit(grvProxyClasses, ProcessClass::GrvProxy, old_id_used, clusterControllerDcId); + RoleFitness oldResolverFit(resolverClasses, ProcessClass::Resolver, old_id_used, clusterControllerDcId); auto first_commit_proxy = getWorkerForRoleInDatacenter( clusterControllerDcId, ProcessClass::CommitProxy, ProcessClass::ExcludeFit, db.config, id_used, true); @@ -1621,6 +1718,10 @@ public: clusterControllerDcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, db.config, id_used, true); auto first_resolver = getWorkerForRoleInDatacenter( clusterControllerDcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, db.config, id_used, true); + auto maxUsed = std::max(std::max(first_commit_proxy.used, first_grv_proxy.used), first_resolver.used); + first_commit_proxy.used = maxUsed; + first_grv_proxy.used = maxUsed; + first_resolver.used = maxUsed; auto commit_proxies = getWorkersForRoleInDatacenter(clusterControllerDcId, ProcessClass::CommitProxy, db.config.getDesiredCommitProxies(), @@ -1643,25 +1744,13 @@ public: first_resolver, true); - RoleFitnessPair newInFit(RoleFitness(commit_proxies, ProcessClass::CommitProxy), - RoleFitness(grv_proxies, ProcessClass::GrvProxy), - RoleFitness(resolvers, ProcessClass::Resolver)); - if (oldInFit.proxy.betterFitness(newInFit.proxy) || oldInFit.grvProxy.betterFitness(newInFit.grvProxy) || - oldInFit.resolver.betterFitness(newInFit.resolver)) { - return false; - } - - // Because a configuration with fewer proxies or resolvers does not cause this function to fail, - // we need an extra check to determine if the total number of processes has been reduced. - // This is mainly helpful in avoiding situations where killing a degraded process - // would result in a configuration with less total processes than desired. - if (oldTLogFit.count + oldInFit.proxy.count + oldInFit.grvProxy.count + oldInFit.resolver.count > - newTLogFit.count + newInFit.proxy.count + newInFit.grvProxy.count + newInFit.resolver.count) { - return false; - } + RoleFitness newCommitProxyFit(commit_proxies, ProcessClass::CommitProxy, id_used, clusterControllerDcId); + RoleFitness newGrvProxyFit(grv_proxies, ProcessClass::GrvProxy, id_used, clusterControllerDcId); + RoleFitness newResolverFit(resolvers, ProcessClass::Resolver, id_used, clusterControllerDcId); // Check backup worker fitness - RoleFitness oldBackupWorkersFit(backup_workers, ProcessClass::Backup); + updateIdUsed(backup_workers, old_id_used); + RoleFitness oldBackupWorkersFit(backup_workers, ProcessClass::Backup, old_id_used, clusterControllerDcId); const int nBackup = backup_addresses.size(); RoleFitness newBackupWorkersFit(getWorkersForRoleInDatacenter(clusterControllerDcId, ProcessClass::Backup, @@ -1670,35 +1759,75 @@ public: id_used, Optional(), true), - ProcessClass::Backup); + ProcessClass::Backup, + id_used, + clusterControllerDcId); - if (oldTLogFit > newTLogFit || oldInFit > newInFit || oldSatelliteTLogFit > newSatelliteTLogFit || - oldRemoteTLogFit > newRemoteTLogFit || oldLogRoutersFit > newLogRoutersFit || - oldBackupWorkersFit > newBackupWorkersFit) { + auto oldFit = std::make_tuple(oldTLogFit, + oldSatelliteTLogFit, + oldCommitProxyFit, + oldGrvProxyFit, + oldResolverFit, + oldBackupWorkersFit, + oldRemoteTLogFit, + oldLogRoutersFit); + auto newFit = std::make_tuple(newTLogFit, + newSatelliteTLogFit, + newCommitProxyFit, + newGrvProxyFit, + newResolverFit, + newBackupWorkersFit, + newRemoteTLogFit, + newLogRoutersFit); + + if (oldFit > newFit) { TraceEvent("BetterMasterExists", id) .detail("OldMasterFit", oldMasterFit) .detail("NewMasterFit", newMasterFit) .detail("OldTLogFit", oldTLogFit.toString()) .detail("NewTLogFit", newTLogFit.toString()) - .detail("OldProxyFit", oldInFit.proxy.toString()) - .detail("NewProxyFit", newInFit.proxy.toString()) - .detail("OldGrvProxyFit", oldInFit.grvProxy.toString()) - .detail("NewGrvProxyFit", newInFit.grvProxy.toString()) - .detail("OldResolverFit", oldInFit.resolver.toString()) - .detail("NewResolverFit", newInFit.resolver.toString()) .detail("OldSatelliteFit", oldSatelliteTLogFit.toString()) .detail("NewSatelliteFit", newSatelliteTLogFit.toString()) + .detail("OldCommitProxyFit", oldCommitProxyFit.toString()) + .detail("NewCommitProxyFit", newCommitProxyFit.toString()) + .detail("OldGrvProxyFit", oldGrvProxyFit.toString()) + .detail("NewGrvProxyFit", newGrvProxyFit.toString()) + .detail("OldResolverFit", oldResolverFit.toString()) + .detail("NewResolverFit", newResolverFit.toString()) + .detail("OldBackupWorkerFit", oldBackupWorkersFit.toString()) + .detail("NewBackupWorkerFit", newBackupWorkersFit.toString()) .detail("OldRemoteFit", oldRemoteTLogFit.toString()) .detail("NewRemoteFit", newRemoteTLogFit.toString()) .detail("OldRouterFit", oldLogRoutersFit.toString()) .detail("NewRouterFit", newLogRoutersFit.toString()) - .detail("OldBackupWorkerFit", oldBackupWorkersFit.toString()) - .detail("NewBackupWorkerFit", newBackupWorkersFit.toString()) .detail("OldSatelliteFallback", oldSatelliteFallback) .detail("NewSatelliteFallback", newSatelliteFallback); return true; } + if (oldFit < newFit) { + TraceEvent("WorseMasterExists", id) + .detail("OldMasterFit", oldMasterFit) + .detail("NewMasterFit", newMasterFit) + .detail("OldTLogFit", oldTLogFit.toString()) + .detail("NewTLogFit", newTLogFit.toString()) + .detail("OldSatelliteFit", oldSatelliteTLogFit.toString()) + .detail("NewSatelliteFit", newSatelliteTLogFit.toString()) + .detail("OldCommitProxyFit", oldCommitProxyFit.toString()) + .detail("NewCommitProxyFit", newCommitProxyFit.toString()) + .detail("OldGrvProxyFit", oldGrvProxyFit.toString()) + .detail("NewGrvProxyFit", newGrvProxyFit.toString()) + .detail("OldResolverFit", oldResolverFit.toString()) + .detail("NewResolverFit", newResolverFit.toString()) + .detail("OldBackupWorkerFit", oldBackupWorkersFit.toString()) + .detail("NewBackupWorkerFit", newBackupWorkersFit.toString()) + .detail("OldRemoteFit", oldRemoteTLogFit.toString()) + .detail("NewRemoteFit", newRemoteTLogFit.toString()) + .detail("OldRouterFit", oldLogRoutersFit.toString()) + .detail("NewRouterFit", newLogRoutersFit.toString()) + .detail("OldSatelliteFallback", oldSatelliteFallback) + .detail("NewSatelliteFallback", newSatelliteFallback); + } return false; } From 434f41a0937031338bf28f9ffca9300eef6be5c3 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 7 Apr 2021 18:14:44 -0700 Subject: [PATCH 07/60] Renamed members of UnitTestParameters to look cleaner. Added getDouble(). Updated more Redwood unit test parameters to be initialized from params. --- fdbserver/VersionedBTree.actor.cpp | 115 +++++++++++++----------- fdbserver/fdbserver.actor.cpp | 2 +- fdbserver/networktest.actor.cpp | 18 ++-- fdbserver/workloads/UnitTests.actor.cpp | 6 +- flow/UnitTest.cpp | 24 +++-- flow/UnitTest.h | 14 ++- 6 files changed, 105 insertions(+), 74 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index c12015ade8..fab29a7034 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -7029,7 +7029,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") { bytes += deltaTest(a, b); } double elapsed = timer() - start; - printf("DeltaTest() on random large records %g M/s %g MB/s\n", count / elapsed / 1e6, bytes / elapsed / 1e6); + printf("DeltaTest() on random large records %f M/s %f MB/s\n", count / elapsed / 1e6, bytes / elapsed / 1e6); keyBuffer.resize(30); valueBuffer.resize(100); @@ -7041,7 +7041,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") { RedwoodRecordRef b = randomRedwoodRecordRef(keyBuffer, valueBuffer); bytes += deltaTest(a, b); } - printf("DeltaTest() on random small records %g M/s %g MB/s\n", count / elapsed / 1e6, bytes / elapsed / 1e6); + printf("DeltaTest() on random small records %f M/s %f MB/s\n", count / elapsed / 1e6, bytes / elapsed / 1e6); RedwoodRecordRef rec1; RedwoodRecordRef rec2; @@ -7058,7 +7058,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") { for (i = 0; i < count; ++i) { total += rec1.getCommonPrefixLen(rec2, 50); } - printf("%" PRId64 " getCommonPrefixLen(skip=50) %g M/s\n", total, count / (timer() - start) / 1e6); + printf("%" PRId64 " getCommonPrefixLen(skip=50) %f M/s\n", total, count / (timer() - start) / 1e6); start = timer(); total = 0; @@ -7066,7 +7066,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") { for (i = 0; i < count; ++i) { total += rec1.getCommonPrefixLen(rec2, 0); } - printf("%" PRId64 " getCommonPrefixLen(skip=0) %g M/s\n", total, count / (timer() - start) / 1e6); + printf("%" PRId64 " getCommonPrefixLen(skip=0) %f M/s\n", total, count / (timer() - start) / 1e6); char buf[1000]; RedwoodRecordRef::Delta& d = *(RedwoodRecordRef::Delta*)buf; @@ -7079,7 +7079,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") { for (i = 0; i < count; ++i) { total += rec1.writeDelta(d, rec2, commonPrefix); } - printf("%" PRId64 " writeDelta(commonPrefix=%d) %g M/s\n", total, commonPrefix, count / (timer() - start) / 1e6); + printf("%" PRId64 " writeDelta(commonPrefix=%d) %f M/s\n", total, commonPrefix, count / (timer() - start) / 1e6); start = timer(); total = 0; @@ -7087,7 +7087,7 @@ TEST_CASE("/redwood/correctness/unit/RedwoodRecordRef") { for (i = 0; i < count; ++i) { total += rec1.writeDelta(d, rec2); } - printf("%" PRId64 " writeDelta() %g M/s\n", total, count / (timer() - start) / 1e6); + printf("%" PRId64 " writeDelta() %f M/s\n", total, count / (timer() - start) / 1e6); return Void(); } @@ -7647,30 +7647,43 @@ TEST_CASE("/redwood/correctness/btree") { g_redwoodMetricsActor = Void(); // Prevent trace event metrics from starting g_redwoodMetrics.clear(); - state std::string pagerFile = "unittest_pageFile.redwood"; + state std::string fileName = params.get("fileName").orDefault("unittest_pageFile.redwood"); IPager2* pager; - state bool serialTest = deterministicRandom()->coinflip(); - state bool shortTest = deterministicRandom()->coinflip(); + state bool serialTest = params.getInt("serialTest").orDefault(deterministicRandom()->coinflip()); + state bool shortTest = params.getInt("shortTest").orDefault(deterministicRandom()->coinflip()); state int pageSize = shortTest ? 200 : (deterministicRandom()->coinflip() ? 4096 : deterministicRandom()->randomInt(200, 400)); - state int64_t targetPageOps = shortTest ? 50000 : 1000000; - state bool pagerMemoryOnly = shortTest && (deterministicRandom()->random01() < .001); - state int maxKeySize = deterministicRandom()->randomInt(1, pageSize * 2); - state int maxValueSize = randomSize(pageSize * 25); - state int maxCommitSize = shortTest ? 1000 : randomSize(std::min((maxKeySize + maxValueSize) * 20000, 10e6)); - state double clearProbability = deterministicRandom()->random01() * .1; - state double clearSingleKeyProbability = deterministicRandom()->random01(); - state double clearPostSetProbability = deterministicRandom()->random01() * .1; - state double coldStartProbability = pagerMemoryOnly ? 0 : (deterministicRandom()->random01() * 0.3); - state double advanceOldVersionProbability = deterministicRandom()->random01(); + state int64_t targetPageOps = params.getInt("targetPageOps").orDefault(shortTest ? 50000 : 1000000); + state bool pagerMemoryOnly = + params.getInt("pagerMemoryOnly").orDefault(shortTest && (deterministicRandom()->random01() < .001)); + state int maxKeySize = params.getInt("maxKeySize").orDefault(deterministicRandom()->randomInt(1, pageSize * 2)); + state int maxValueSize = params.getInt("maxValueSize").orDefault(randomSize(pageSize * 25)); + state int maxCommitSize = + params.getInt("maxCommitSize") + .orDefault(shortTest ? 1000 : randomSize(std::min((maxKeySize + maxValueSize) * 20000, 10e6))); + state double clearProbability = + params.getDouble("clearProbability").orDefault(deterministicRandom()->random01() * .1); + state double clearSingleKeyProbability = + params.getDouble("clearSingleKeyProbability").orDefault(deterministicRandom()->random01()); + state double clearPostSetProbability = + params.getDouble("clearPostSetProbability").orDefault(deterministicRandom()->random01() * .1); + state double coldStartProbability = params.getDouble("coldStartProbability") + .orDefault(pagerMemoryOnly ? 0 : (deterministicRandom()->random01() * 0.3)); + state double advanceOldVersionProbability = + params.getDouble("advanceOldVersionProbability").orDefault(deterministicRandom()->random01()); state int64_t cacheSizeBytes = - pagerMemoryOnly ? 2e9 : (pageSize * deterministicRandom()->randomInt(1, (BUGGIFY ? 2 : 10000) + 1)); - state Version versionIncrement = deterministicRandom()->randomInt64(1, 1e8); - state Version remapCleanupWindow = BUGGIFY ? 0 : deterministicRandom()->randomInt64(1, versionIncrement * 50); - state int maxVerificationMapEntries = 300e3; + params.getInt("cacheSizeBytes") + .orDefault(pagerMemoryOnly ? 2e9 + : (pageSize * deterministicRandom()->randomInt(1, (BUGGIFY ? 2 : 10000) + 1))); + state Version versionIncrement = + params.getInt("versionIncrement").orDefault(deterministicRandom()->randomInt64(1, 1e8)); + state Version remapCleanupWindow = + params.getInt("remapCleanupWindow") + .orDefault(BUGGIFY ? 0 : deterministicRandom()->randomInt64(1, versionIncrement * 50)); + state int maxVerificationMapEntries = params.getInt("maxVerificationMapEntries").orDefault(300e3); printf("\n"); printf("targetPageOps: %" PRId64 "\n", targetPageOps); @@ -7693,11 +7706,11 @@ TEST_CASE("/redwood/correctness/btree") { printf("\n"); printf("Deleting existing test data...\n"); - deleteFile(pagerFile); + deleteFile(fileName); printf("Initializing...\n"); - pager = new DWALPager(pageSize, pagerFile, cacheSizeBytes, remapCleanupWindow, pagerMemoryOnly); - state VersionedBTree* btree = new VersionedBTree(pager, pagerFile); + pager = new DWALPager(pageSize, fileName, cacheSizeBytes, remapCleanupWindow, pagerMemoryOnly); + state VersionedBTree* btree = new VersionedBTree(pager, fileName); wait(btree->init()); state std::map, Optional> written; @@ -7900,8 +7913,8 @@ TEST_CASE("/redwood/correctness/btree") { wait(closedFuture); printf("Reopening btree from disk.\n"); - IPager2* pager = new DWALPager(pageSize, pagerFile, cacheSizeBytes, remapCleanupWindow); - btree = new VersionedBTree(pager, pagerFile); + IPager2* pager = new DWALPager(pageSize, fileName, cacheSizeBytes, remapCleanupWindow); + btree = new VersionedBTree(pager, fileName); wait(btree->init()); Version v = btree->getLatestVersion(); @@ -7937,7 +7950,7 @@ TEST_CASE("/redwood/correctness/btree") { state Future closedFuture = btree->onClosed(); btree->close(); wait(closedFuture); - btree = new VersionedBTree(new DWALPager(pageSize, pagerFile, cacheSizeBytes, 0), pagerFile); + btree = new VersionedBTree(new DWALPager(pageSize, fileName, cacheSizeBytes, 0), fileName); wait(btree->init()); wait(btree->clearAllAndCheckSanity()); @@ -8045,22 +8058,22 @@ TEST_CASE(":/redwood/performance/set") { deleteFile(pagerFile); } - state int pageSize = params.getIntParam("pageSize").orDefault(SERVER_KNOBS->REDWOOD_DEFAULT_PAGE_SIZE); - state int64_t pageCacheBytes = params.getIntParam("pageCacheBytes").orDefault(FLOW_KNOBS->PAGE_CACHE_4K); - state int nodeCount = params.getIntParam("nodeCount").orDefault(1e9); - state int maxRecordsPerCommit = params.getIntParam("maxRecordsPerCommit").orDefault(20000); - state int maxKVBytesPerCommit = params.getIntParam("maxKVBytesPerCommit").orDefault(20e6); - state int64_t kvBytesTarget = params.getIntParam("kvBytesTarget").orDefault(4e9); - state int minKeyPrefixBytes = params.getIntParam("minKeyPrefixBytes").orDefault(25); - state int maxKeyPrefixBytes = params.getIntParam("maxKeyPrefixBytes").orDefault(25); - state int minValueSize = params.getIntParam("minValueSize").orDefault(100); - state int maxValueSize = params.getIntParam("maxValueSize").orDefault(500); - state int minConsecutiveRun = params.getIntParam("minConsecutiveRun").orDefault(1); - state int maxConsecutiveRun = params.getIntParam("maxConsecutiveRun").orDefault(100); - state char firstKeyChar = params.getParam("firstKeyChar").orDefault("a")[0]; - state char lastKeyChar = params.getParam("lastKeyChar").orDefault("m")[0]; + state int pageSize = params.getInt("pageSize").orDefault(SERVER_KNOBS->REDWOOD_DEFAULT_PAGE_SIZE); + state int64_t pageCacheBytes = params.getInt("pageCacheBytes").orDefault(FLOW_KNOBS->PAGE_CACHE_4K); + state int nodeCount = params.getInt("nodeCount").orDefault(1e9); + state int maxRecordsPerCommit = params.getInt("maxRecordsPerCommit").orDefault(20000); + state int maxKVBytesPerCommit = params.getInt("maxKVBytesPerCommit").orDefault(20e6); + state int64_t kvBytesTarget = params.getInt("kvBytesTarget").orDefault(4e9); + state int minKeyPrefixBytes = params.getInt("minKeyPrefixBytes").orDefault(25); + state int maxKeyPrefixBytes = params.getInt("maxKeyPrefixBytes").orDefault(25); + state int minValueSize = params.getInt("minValueSize").orDefault(100); + state int maxValueSize = params.getInt("maxValueSize").orDefault(500); + state int minConsecutiveRun = params.getInt("minConsecutiveRun").orDefault(1); + state int maxConsecutiveRun = params.getInt("maxConsecutiveRun").orDefault(100); + state char firstKeyChar = params.get("firstKeyChar").orDefault("a")[0]; + state char lastKeyChar = params.get("lastKeyChar").orDefault("m")[0]; state Version remapCleanupWindow = - params.getIntParam("remapCleanupWindow").orDefault(SERVER_KNOBS->REDWOOD_REMAP_CLEANUP_WINDOW); + params.getInt("remapCleanupWindow").orDefault(SERVER_KNOBS->REDWOOD_REMAP_CLEANUP_WINDOW); printf("pageSize: %d\n", pageSize); printf("pageCacheBytes: %" PRId64 "\n", pageCacheBytes); @@ -8543,10 +8556,10 @@ ACTOR Future doPrefixInsertComparison(int suffixSize, } TEST_CASE(":/redwood/performance/prefixSizeComparison") { - state int suffixSize = 12; - state int valueSize = 100; - state int recordCountTarget = 100e6; - state int usePrefixesInOrder = false; + state int suffixSize = params.getInt("suffixSize").orDefault(12); + state int valueSize = params.getInt("valueSize").orDefault(100); + state int recordCountTarget = params.getInt("recordCountTarget").orDefault(100e6); + state bool usePrefixesInOrder = params.getInt("usePrefixesInOrder").orDefault(0); wait(doPrefixInsertComparison( suffixSize, valueSize, recordCountTarget, usePrefixesInOrder, KVSource({ { 10, 100000 } }))); @@ -8564,9 +8577,9 @@ TEST_CASE(":/redwood/performance/prefixSizeComparison") { } TEST_CASE(":/redwood/performance/sequentialInsert") { - state int prefixLen = 30; - state int valueSize = 100; - state int recordCountTarget = 100e6; + state int prefixLen = params.getInt("prefixLen").orDefault(30); + state int valueSize = params.getInt("valueSize").orDefault(100); + state int recordCountTarget = params.getInt("recordCountTarget").orDefault(100e6); deleteFile("test.redwood"); wait(delay(5)); diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 325caa10c5..610410dec9 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -1059,7 +1059,7 @@ private: fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str()); flushAndExit(FDB_EXIT_ERROR); } - testParams.setParam(syn.substr(7), args.OptionArg()); + testParams.set(syn.substr(7), args.OptionArg()); break; } case OPT_LOCALITY: { diff --git a/fdbserver/networktest.actor.cpp b/fdbserver/networktest.actor.cpp index cea0c2fca8..d9ef7e4857 100644 --- a/fdbserver/networktest.actor.cpp +++ b/fdbserver/networktest.actor.cpp @@ -570,15 +570,15 @@ struct P2PNetworkTest { // The client will close the connection after a random idleMilliseconds. // Reads and writes can optionally preceded by random delays, waitReadMilliseconds and waitWriteMilliseconds. TEST_CASE(":/network/p2ptest") { - state P2PNetworkTest p2p(params.getParam("listenerAddresses").orDefault(""), - params.getParam("remoteAddresses").orDefault(""), - params.getIntParam("connectionsOut").orDefault(1), - params.getParam("requestBytes").orDefault("50:100"), - params.getParam("replyBytes").orDefault("500:1000"), - params.getParam("requests").orDefault("10:10000"), - params.getParam("idleMilliseconds").orDefault("0"), - params.getParam("waitReadMilliseconds").orDefault("0"), - params.getParam("waitWriteMilliseconds").orDefault("0")); + state P2PNetworkTest p2p(params.get("listenerAddresses").orDefault(""), + params.get("remoteAddresses").orDefault(""), + params.getInt("connectionsOut").orDefault(1), + params.get("requestBytes").orDefault("50:100"), + params.get("replyBytes").orDefault("500:1000"), + params.get("requests").orDefault("10:10000"), + params.get("idleMilliseconds").orDefault("0"), + params.get("waitReadMilliseconds").orDefault("0"), + params.get("waitWriteMilliseconds").orDefault("0")); wait(p2p.run()); return Void(); diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp index 024cfc9973..db816fe4c7 100644 --- a/fdbserver/workloads/UnitTests.actor.cpp +++ b/fdbserver/workloads/UnitTests.actor.cpp @@ -48,9 +48,9 @@ struct UnitTestWorkload : TestWorkload { testRunLimit = getOption(options, LiteralStringRef("maxTestCases"), -1); // Consume all remaining options as testParams which the unit test can access - for(auto &kv : options) { - if(kv.value.size() != 0) { - testParams.setParam(kv.key.toString(), getOption(options, kv.key, StringRef()).toString()); + for (auto& kv : options) { + if (kv.value.size() != 0) { + testParams.set(kv.key.toString(), getOption(options, kv.key, StringRef()).toString()); } } diff --git a/flow/UnitTest.cpp b/flow/UnitTest.cpp index 7303cd33c7..f797fc32c1 100644 --- a/flow/UnitTest.cpp +++ b/flow/UnitTest.cpp @@ -27,12 +27,12 @@ UnitTest::UnitTest(const char* name, const char* file, int line, TestFunction fu g_unittests.tests = this; } -void UnitTestParameters::setParam(const std::string& name, const std::string& value) { +void UnitTestParameters::set(const std::string& name, const std::string& value) { printf("setting %s = %s\n", name.c_str(), value.c_str()); params[name] = value; } -Optional UnitTestParameters::getParam(const std::string& name) const { +Optional UnitTestParameters::get(const std::string& name) const { auto it = params.find(name); if (it != params.end()) { return it->second; @@ -40,14 +40,26 @@ Optional UnitTestParameters::getParam(const std::string& name) cons return {}; } -void UnitTestParameters::setParam(const std::string& name, int64_t value) { - setParam(name, format("%" PRId64, value)); +void UnitTestParameters::set(const std::string& name, int64_t value) { + set(name, format("%" PRId64, value)); }; -Optional UnitTestParameters::getIntParam(const std::string& name) const { - auto opt = getParam(name); +void UnitTestParameters::set(const std::string& name, double value) { + set(name, format("%g", value)); +}; + +Optional UnitTestParameters::getInt(const std::string& name) const { + auto opt = get(name); if (opt.present()) { return atoll(opt.get().c_str()); } return {}; } + +Optional UnitTestParameters::getDouble(const std::string& name) const { + auto opt = get(name); + if (opt.present()) { + return atof(opt.get().c_str()); + } + return {}; +} diff --git a/flow/UnitTest.h b/flow/UnitTest.h index 21d51a158f..3a0d4c1db6 100644 --- a/flow/UnitTest.h +++ b/flow/UnitTest.h @@ -52,16 +52,22 @@ struct UnitTestParameters { std::map params; // Set a named parameter to a string value, replacing any existing value - void setParam(const std::string& name, const std::string& value); + void set(const std::string& name, const std::string& value); // Set a named parameter to an integer converted to a string value, replacing any existing value - void setParam(const std::string& name, int64_t value); + void set(const std::string& name, int64_t value); + + // Set a named parameter to a double converted to a string value, replacing any existing value + void set(const std::string& name, double value); // Get a parameter's value, will return !present() if parameter was not set - Optional getParam(const std::string& name) const; + Optional get(const std::string& name) const; // Get a parameter's value as an integer, will return !present() if parameter was not set - Optional getIntParam(const std::string& name) const; + Optional getInt(const std::string& name) const; + + // Get a parameter's value parsed as a double, will return !present() if parameter was not set + Optional getDouble(const std::string& name) const; }; // Unit test definition structured as a linked list item From 7b08886caffdfc1ec4ac32cd2bff1c27a9feb3cc Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 7 Apr 2021 18:29:17 -0700 Subject: [PATCH 08/60] Updated btree unit test name. --- tests/rare/RedwoodCorrectnessBTree.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/rare/RedwoodCorrectnessBTree.toml b/tests/rare/RedwoodCorrectnessBTree.toml index 1a7320e416..db21848a4b 100644 --- a/tests/rare/RedwoodCorrectnessBTree.toml +++ b/tests/rare/RedwoodCorrectnessBTree.toml @@ -6,4 +6,5 @@ startDelay = 0 [[test.workload]] testName = 'UnitTests' maxTestCases = 0 - testsMatching = ':/redwood/correctness/btree' + testsMatching = '/redwood/correctness/btree' + remapCleanupWindow = 1000000000 From 15e8b43961cec6eb4d0b700d754125a82b11eb57 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Wed, 7 Apr 2021 19:57:24 -0700 Subject: [PATCH 09/60] rewrote getWorkersForTLogs to do a much better job of avoiding degraded processes and processes in the same DC as the cluster controller --- fdbserver/ClusterController.actor.cpp | 292 ++++++++++++++++---------- 1 file changed, 178 insertions(+), 114 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 87dcbd7c11..c5546830c6 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -415,91 +415,37 @@ public: .push_back(worker_details); } - // Make sure we check for tlogs at the required size before adding processses from the next fitness level - for (int fitness = ProcessClass::GoodFit; fitness != ProcessClass::NeverAssign; fitness++) { - fitness_workers[std::make_tuple((ProcessClass::Fitness)fitness, 0, true, false)]; - } - results.reserve(results.size() + id_worker.size()); + int requiredProcesses = 0; + auto requiredFitness = ProcessClass::BestFit; + int requiredUsed = 0; + bool requiredDegraded = false; + bool requiredInCCDC = false; + for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { auto fitness = std::get<0>(workerIter->first); auto used = std::get<1>(workerIter->first); - auto addingDegraded = std::get<2>(workerIter->first); - ASSERT(fitness < ProcessClass::NeverAssign); - if (bCompleted) { - break; + if (fitness > requiredFitness || used > requiredUsed) { + requiredFitness = fitness; + requiredUsed = used; + if (logServerSet->size() >= required && logServerSet->validate(policy)) { + requiredProcesses = logServerSet->size(); + bCompleted = true; + break; + } } + if (std::get<2>(workerIter->first)) { + requiredDegraded = true; + } + if (std::get<3>(workerIter->first)) { + requiredInCCDC = true; + } for (auto& worker : workerIter->second) { logServerMap->add(worker.interf.locality, &worker); } - - if (logServerSet->size() < (addingDegraded ? required : desired)) { - } else if (logServerSet->size() == required || logServerSet->size() <= desired) { - if (logServerSet->validate(policy)) { - for (auto& object : logServerMap->getObjects()) { - results.push_back(*object); - } - bCompleted = true; - break; - } - TraceEvent(SevWarn, "GWFTADNotAcceptable", id) - .detail("DcIds", dcList) - .detail("Fitness", fitness) - .detail("Processes", logServerSet->size()) - .detail("Required", required) - .detail("TLogPolicy", policy->info()) - .detail("DesiredLogs", desired) - .detail("Used", used) - .detail("AddingDegraded", addingDegraded); - } - // Try to select the desired size, if larger - else { - std::vector bestSet; - std::vector tLocalities; - - // Try to find the best team of servers to fulfill the policy - if (findBestPolicySet(bestSet, - logServerSet, - policy, - desired, - SERVER_KNOBS->POLICY_RATING_TESTS, - SERVER_KNOBS->POLICY_GENERATIONS)) { - results.reserve(results.size() + bestSet.size()); - for (auto& entry : bestSet) { - auto object = logServerMap->getObject(entry); - ASSERT(object); - results.push_back(*object); - tLocalities.push_back(object->interf.locality); - } - TraceEvent("GWFTADBestResults", id) - .detail("DcIds", dcList) - .detail("Fitness", fitness) - .detail("Used", used) - .detail("Processes", logServerSet->size()) - .detail("BestCount", bestSet.size()) - .detail("BestZones", ::describeZones(tLocalities)) - .detail("BestDataHalls", ::describeDataHalls(tLocalities)) - .detail("TLogPolicy", policy->info()) - .detail("TotalResults", results.size()) - .detail("DesiredLogs", desired) - .detail("AddingDegraded", addingDegraded); - bCompleted = true; - break; - } - TraceEvent(SevWarn, "GWFTADNoBest", id) - .detail("DcIds", dcList) - .detail("Fitness", fitness) - .detail("Used", used) - .detail("Processes", logServerSet->size()) - .detail("Required", required) - .detail("TLogPolicy", policy->info()) - .detail("DesiredLogs", desired) - .detail("AddingDegraded", addingDegraded); - } } - // If policy cannot be satisfied - if (!bCompleted) { + if (!bCompleted && !(logServerSet->size() >= required && logServerSet->validate(policy))) { std::vector tLocalities; for (auto& object : logServerMap->getObjects()) { tLocalities.push_back(object->interf.locality); @@ -517,33 +463,154 @@ public: .detail("MissingDataHalls", ::describeDataHalls(unavailableLocals)) .detail("Required", required) .detail("DesiredLogs", desired) - .detail("RatingTests", SERVER_KNOBS->POLICY_RATING_TESTS) .detail("CheckStable", checkStable) - .detail("NumExclusionWorkers", exclusionWorkerIds.size()) - .detail("PolicyGenerations", SERVER_KNOBS->POLICY_GENERATIONS) - .backtrace(); + .detail("NumExclusionWorkers", exclusionWorkerIds.size()); logServerSet->clear(); logServerSet.clear(); throw no_more_servers(); } + if (requiredProcesses <= desired) { + for (auto& object : logServerMap->getObjects()) { + results.push_back(*object); + } + for (auto& result : results) { + id_used[result.interf.locality.processId()]++; + } + TraceEvent("GetTLogTeamDone") + .detail("DcIds", dcList) + .detail("Policy", policy->info()) + .detail("Results", results.size()) + .detail("Processes", logServerSet->size()) + .detail("Workers", id_worker.size()) + .detail("Required", required) + .detail("Desired", desired) + .detail("Fitness", requiredFitness) + .detail("Used", requiredUsed) + .detail("AddingDegraded", requiredDegraded) + .detail("InCCDC", requiredInCCDC); + return results; + } + + if (requiredDegraded) { + logServerMap->clear(); + for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { + auto fitness = std::get<0>(workerIter->first); + auto used = std::get<1>(workerIter->first); + auto addingDegraded = std::get<2>(workerIter->first); + if (addingDegraded) { + continue; + } + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } + for (auto& worker : workerIter->second) { + logServerMap->add(worker.interf.locality, &worker); + } + } + if (logServerSet->size() >= desired && logServerSet->validate(policy)) { + requiredDegraded = false; + } + } + + if (requiredInCCDC) { + logServerMap->clear(); + for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { + auto fitness = std::get<0>(workerIter->first); + auto used = std::get<1>(workerIter->first); + auto addingDegraded = std::get<2>(workerIter->first); + auto inCCDC = std::get<3>(workerIter->first); + if (inCCDC || (!requiredDegraded && addingDegraded)) { + continue; + } + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } + for (auto& worker : workerIter->second) { + logServerMap->add(worker.interf.locality, &worker); + } + } + if (logServerSet->size() >= desired && logServerSet->validate(policy)) { + requiredInCCDC = false; + } + } + + logServerMap->clear(); + for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { + auto fitness = std::get<0>(workerIter->first); + auto used = std::get<1>(workerIter->first); + auto addingDegraded = std::get<2>(workerIter->first); + auto inCCDC = std::get<3>(workerIter->first); + if ((!requiredInCCDC && inCCDC) || (!requiredDegraded && addingDegraded)) { + continue; + } + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } + for (auto& worker : workerIter->second) { + logServerMap->add(worker.interf.locality, &worker); + } + } + + if (logServerSet->size() == desired) { + for (auto& object : logServerMap->getObjects()) { + results.push_back(*object); + } + for (auto& result : results) { + id_used[result.interf.locality.processId()]++; + } + TraceEvent("GetTLogTeamDone") + .detail("DcIds", dcList) + .detail("Policy", policy->info()) + .detail("Results", results.size()) + .detail("Processes", logServerSet->size()) + .detail("Workers", id_worker.size()) + .detail("Required", required) + .detail("Desired", desired) + .detail("Fitness", requiredFitness) + .detail("Used", requiredUsed) + .detail("AddingDegraded", requiredDegraded) + .detail("InCCDC", requiredInCCDC); + return results; + } + + std::vector bestSet; + std::vector tLocalities; + + // Try to find the best team of servers to fulfill the policy + bCompleted = findBestPolicySet(bestSet, + logServerSet, + policy, + desired, + SERVER_KNOBS->POLICY_RATING_TESTS, + SERVER_KNOBS->POLICY_GENERATIONS); + ASSERT(bCompleted); + results.reserve(results.size() + bestSet.size()); + for (auto& entry : bestSet) { + auto object = logServerMap->getObject(entry); + ASSERT(object); + results.push_back(*object); + tLocalities.push_back(object->interf.locality); + } for (auto& result : results) { id_used[result.interf.locality.processId()]++; } - TraceEvent("GetTLogTeamDone") .detail("DcIds", dcList) - .detail("Completed", bCompleted) .detail("Policy", policy->info()) .detail("Results", results.size()) .detail("Processes", logServerSet->size()) .detail("Workers", id_worker.size()) .detail("Required", required) .detail("Desired", desired) - .detail("RatingTests", SERVER_KNOBS->POLICY_RATING_TESTS) - .detail("PolicyGenerations", SERVER_KNOBS->POLICY_GENERATIONS); - + .detail("Fitness", requiredFitness) + .detail("Used", requiredUsed) + .detail("AddingDegraded", requiredDegraded) + .detail("InCCDC", requiredInCCDC) + .detail("BestCount", bestSet.size()) + .detail("BestZones", ::describeZones(tLocalities)) + .detail("BestDataHalls", ::describeDataHalls(tLocalities)); return results; } @@ -751,41 +818,42 @@ public: ProcessClass::Fitness worstFit; ProcessClass::ClusterRole role; int count; - int worstUsed; - bool worstIsDegraded; - bool inClusterControllerDC; + int worstUsed = 1; + bool degraded = false; + bool inClusterControllerDC = false; RoleFitness(int bestFit, int worstFit, int count, ProcessClass::ClusterRole role) : bestFit((ProcessClass::Fitness)bestFit), worstFit((ProcessClass::Fitness)worstFit), count(count), - role(role), worstUsed(1), worstIsDegraded(false), inClusterControllerDC(false) {} + role(role) {} RoleFitness(int fitness, int count, ProcessClass::ClusterRole role) - : bestFit((ProcessClass::Fitness)fitness), worstFit((ProcessClass::Fitness)fitness), count(count), role(role), - worstUsed(1), worstIsDegraded(false), inClusterControllerDC(false) {} + : bestFit((ProcessClass::Fitness)fitness), worstFit((ProcessClass::Fitness)fitness), count(count), + role(role) {} RoleFitness() : bestFit(ProcessClass::NeverAssign), worstFit(ProcessClass::NeverAssign), role(ProcessClass::NoRole), - count(0), worstUsed(1), worstIsDegraded(false), inClusterControllerDC(false) {} + count(0) {} RoleFitness(const vector& workers, ProcessClass::ClusterRole role, const std::map>, int>& id_used, Optional> ccDcId) : role(role) { - worstFit = ProcessClass::BestFit; - worstIsDegraded = false; + // Every recruitment will attempt to recruit the preferred amount through GoodFit, + // So a recruitment which only has BestFit is not better than one that has a GoodFit process + worstFit = ProcessClass::GoodFit; + + degraded = false; inClusterControllerDC = false; bestFit = ProcessClass::NeverAssign; worstUsed = 1; for (auto& it : workers) { auto thisFit = it.processClass.machineClassFitness(role); - if (thisFit > worstFit) { - worstFit = thisFit; - worstIsDegraded = it.degraded; - } else if (thisFit == worstFit) { - worstIsDegraded = worstIsDegraded || it.degraded; - } + worstFit = std::max(worstFit, thisFit); bestFit = std::min(bestFit, thisFit); + degraded |= it.degraded; + inClusterControllerDC |= (it.interf.locality.dcId() == ccDcId); + auto thisUsed = id_used.find(it.interf.locality.processId()); if (thisUsed == id_used.end()) { TraceEvent(SevError, "UsedNotFound").detail("ProcessId", it.interf.locality.processId().get()); @@ -796,18 +864,15 @@ public: ASSERT(false); } worstUsed = std::max(worstUsed, thisUsed->second); - // only tlogs avoid the cluster controller dc - if (role == ProcessClass::TLog && it.interf.locality.dcId() == ccDcId) { - inClusterControllerDC = true; - } } - // Every recruitment will attempt to recruit the preferred amount through GoodFit, - // So a recruitment which only has BestFit is not better than one that has a GoodFit process - worstFit = std::max(worstFit, ProcessClass::GoodFit); + count = workers.size(); + // degraded is only used for recruitment of tlogs + // only tlogs avoid the cluster controller dc if (role != ProcessClass::TLog) { - worstIsDegraded = false; + degraded = false; + inClusterControllerDC = false; } } @@ -818,8 +883,8 @@ public: return worstUsed < r.worstUsed; if (count != r.count) return count > r.count; - if (worstIsDegraded != r.worstIsDegraded) - return r.worstIsDegraded; + if (degraded != r.degraded) + return r.degraded; if (inClusterControllerDC != r.inClusterControllerDC) return r.inClusterControllerDC; // FIXME: TLog recruitment process does not guarantee the best fit is not worsened. @@ -838,8 +903,8 @@ public: return worstFit < r.worstFit; if (worstUsed != r.worstUsed) return worstUsed < r.worstUsed; - if (worstIsDegraded != r.worstIsDegraded) - return r.worstIsDegraded; + if (degraded != r.degraded) + return r.degraded; if (inClusterControllerDC != r.inClusterControllerDC) return r.inClusterControllerDC; return false; @@ -847,12 +912,11 @@ public: bool operator==(RoleFitness const& r) const { return worstFit == r.worstFit && worstUsed == r.worstUsed && bestFit == r.bestFit && count == r.count && - worstIsDegraded == r.worstIsDegraded && inClusterControllerDC == r.inClusterControllerDC; + degraded == r.degraded && inClusterControllerDC == r.inClusterControllerDC; } std::string toString() const { - return format( - "%d %d %d %d %d %d", worstFit, worstUsed, count, worstIsDegraded, inClusterControllerDC, bestFit); + return format("%d %d %d %d %d %d", worstFit, worstUsed, count, degraded, inClusterControllerDC, bestFit); } }; From 14213b01519d83f109c85082a1b7c0579208e3f9 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Wed, 7 Apr 2021 20:06:30 -0700 Subject: [PATCH 10/60] code cleanup --- fdbserver/ClusterController.actor.cpp | 31 +++++++++++++-------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index c5546830c6..fb81c571c1 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -1033,7 +1033,7 @@ public: // If one of the first process recruitments is forced to share a process, allow all of next recruitments // to also share a process. - auto maxUsed = std::max(std::max(first_commit_proxy.used, first_grv_proxy.used), first_resolver.used); + auto maxUsed = std::max({ first_commit_proxy.used, first_grv_proxy.used, first_resolver.used }); first_commit_proxy.used = maxUsed; first_grv_proxy.used = maxUsed; first_resolver.used = maxUsed; @@ -1225,8 +1225,7 @@ public: // If one of the first process recruitments is forced to share a process, allow all of next // recruitments to also share a process. - auto maxUsed = - std::max(std::max(first_commit_proxy.used, first_grv_proxy.used), first_resolver.used); + auto maxUsed = std::max({ first_commit_proxy.used, first_grv_proxy.used, first_resolver.used }); first_commit_proxy.used = maxUsed; first_grv_proxy.used = maxUsed; first_resolver.used = maxUsed; @@ -1424,7 +1423,7 @@ public: // Do not trigger better master exists if the cluster controller is excluded, since the master will change // anyways once the cluster controller is moved if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) { - TraceEvent("WorseMasterExists", id).detail("Reason", "ClusterControllerExcluded"); + TraceEvent("NewRecruitmentIsWorse", id).detail("Reason", "ClusterControllerExcluded"); return false; } @@ -1437,7 +1436,7 @@ public: // Get master process auto masterWorker = id_worker.find(dbi.master.locality.processId()); if (masterWorker == id_worker.end()) { - TraceEvent("WorseMasterExists", id) + TraceEvent("NewRecruitmentIsWorse", id) .detail("Reason", "CannotFindMaster") .detail("ProcessID", dbi.master.locality.processId()); return false; @@ -1456,7 +1455,7 @@ public: for (auto& it : logSet.tLogs) { auto tlogWorker = id_worker.find(it.interf().filteredLocality.processId()); if (tlogWorker == id_worker.end()) { - TraceEvent("WorseMasterExists", id) + TraceEvent("NewRecruitmentIsWorse", id) .detail("Reason", "CannotFindTLog") .detail("ProcessID", it.interf().filteredLocality.processId()); return false; @@ -1480,7 +1479,7 @@ public: for (auto& it : logSet.logRouters) { auto tlogWorker = id_worker.find(it.interf().filteredLocality.processId()); if (tlogWorker == id_worker.end()) { - TraceEvent("WorseMasterExists", id) + TraceEvent("NewRecruitmentIsWorse", id) .detail("Reason", "CannotFindLogRouter") .detail("ProcessID", it.interf().filteredLocality.processId()); return false; @@ -1500,7 +1499,7 @@ public: for (const auto& worker : logSet.backupWorkers) { auto workerIt = id_worker.find(worker.interf().locality.processId()); if (workerIt == id_worker.end()) { - TraceEvent("WorseMasterExists", id) + TraceEvent("NewRecruitmentIsWorse", id) .detail("Reason", "CannotFindBackupWorker") .detail("ProcessID", worker.interf().locality.processId()); return false; @@ -1523,7 +1522,7 @@ public: for (auto& it : dbi.client.commitProxies) { auto commitProxyWorker = id_worker.find(it.processId); if (commitProxyWorker == id_worker.end()) { - TraceEvent("WorseMasterExists", id) + TraceEvent("NewRecruitmentIsWorse", id) .detail("Reason", "CannotFindCommitProxy") .detail("ProcessID", it.processId); return false; @@ -1542,7 +1541,7 @@ public: for (auto& it : dbi.client.grvProxies) { auto grvProxyWorker = id_worker.find(it.processId); if (grvProxyWorker == id_worker.end()) { - TraceEvent("WorseMasterExists", id) + TraceEvent("NewRecruitmentIsWorse", id) .detail("Reason", "CannotFindGrvProxy") .detail("ProcessID", it.processId); return false; @@ -1561,7 +1560,7 @@ public: for (auto& it : dbi.resolvers) { auto resolverWorker = id_worker.find(it.locality.processId()); if (resolverWorker == id_worker.end()) { - TraceEvent("WorseMasterExists", id) + TraceEvent("NewRecruitmentIsWorse", id) .detail("Reason", "CannotFindResolver") .detail("ProcessID", it.locality.processId()); return false; @@ -1596,7 +1595,7 @@ public: old_id_used[masterWorker->first]++; if (oldMasterFit < newMasterFit) { - TraceEvent("WorseMasterExists", id) + TraceEvent("NewRecruitmentIsWorse", id) .detail("OldMasterFit", oldMasterFit) .detail("NewMasterFit", newMasterFit) .detail("OldIsCC", dbi.master.locality.processId() == clusterControllerProcessId) @@ -1700,7 +1699,7 @@ public: return true; } if (!oldSatelliteFallback && newSatelliteFallback) { - TraceEvent("WorseMasterExists", id) + TraceEvent("NewRecruitmentIsWorse", id) .detail("OldSatelliteFallback", oldSatelliteFallback) .detail("NewSatelliteFallback", newSatelliteFallback); return false; @@ -1713,7 +1712,7 @@ public: return true; } if (oldSatelliteRegionFit > newSatelliteRegionFit) { - TraceEvent("WorseMasterExists", id) + TraceEvent("NewRecruitmentIsWorse", id) .detail("OldSatelliteRegionFit", oldSatelliteRegionFit) .detail("NewSatelliteRegionFit", newSatelliteRegionFit); return false; @@ -1782,7 +1781,7 @@ public: clusterControllerDcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, db.config, id_used, true); auto first_resolver = getWorkerForRoleInDatacenter( clusterControllerDcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, db.config, id_used, true); - auto maxUsed = std::max(std::max(first_commit_proxy.used, first_grv_proxy.used), first_resolver.used); + auto maxUsed = std::max({ first_commit_proxy.used, first_grv_proxy.used, first_resolver.used }); first_commit_proxy.used = maxUsed; first_grv_proxy.used = maxUsed; first_resolver.used = maxUsed; @@ -1870,7 +1869,7 @@ public: } if (oldFit < newFit) { - TraceEvent("WorseMasterExists", id) + TraceEvent("NewRecruitmentIsWorse", id) .detail("OldMasterFit", oldMasterFit) .detail("NewMasterFit", newMasterFit) .detail("OldTLogFit", oldTLogFit.toString()) From 4d8dd0b0a0d173a3cea08c6ded51b11e8b16e3c9 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Wed, 7 Apr 2021 20:32:45 -0700 Subject: [PATCH 11/60] fix: desired must be greater than or equal to required --- fdbserver/ClusterController.actor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index fb81c571c1..64794cb2ae 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -345,6 +345,7 @@ public: Reference logServerSet; LocalityMap* logServerMap; bool bCompleted = false; + desired = std::max(required, desired); // Construct the list of DCs where the TLog recruitment is happening. This is mainly for logging purpose. std::string dcList; From 1b1f73ea16c7a4510b133a015677fd0c6777eb7b Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Wed, 7 Apr 2021 20:40:42 -0700 Subject: [PATCH 12/60] added comments --- fdbserver/ClusterController.actor.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 64794cb2ae..8480566914 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -416,12 +416,12 @@ public: .push_back(worker_details); } - int requiredProcesses = 0; auto requiredFitness = ProcessClass::BestFit; int requiredUsed = 0; bool requiredDegraded = false; bool requiredInCCDC = false; + // Determine the minimum fitness and used necessary to fulfill the policy for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { auto fitness = std::get<0>(workerIter->first); auto used = std::get<1>(workerIter->first); @@ -429,7 +429,6 @@ public: requiredFitness = fitness; requiredUsed = used; if (logServerSet->size() >= required && logServerSet->validate(policy)) { - requiredProcesses = logServerSet->size(); bCompleted = true; break; } @@ -472,7 +471,8 @@ public: throw no_more_servers(); } - if (requiredProcesses <= desired) { + // If we have less than the desired amount, return all of the processes we have + if (logServerSet->size() <= desired) { for (auto& object : logServerMap->getObjects()) { results.push_back(*object); } @@ -494,6 +494,8 @@ public: return results; } + // If we have added any degraded processes, try and remove them to see if we can still + // have the desired amount of processes if (requiredDegraded) { logServerMap->clear(); for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { @@ -515,6 +517,8 @@ public: } } + // If we have added any processes in the CC DC, try and remove them to see if we can still + // have the desired amount of processes if (requiredInCCDC) { logServerMap->clear(); for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { @@ -579,7 +583,8 @@ public: std::vector bestSet; std::vector tLocalities; - // Try to find the best team of servers to fulfill the policy + // We have more than the desired number of processes, so use the policy engine to + // pick a diverse subset of them bCompleted = findBestPolicySet(bestSet, logServerSet, policy, From 5695a1816f0bad8bc95c690a1c78977aa939914c Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Wed, 7 Apr 2021 21:31:14 -0700 Subject: [PATCH 13/60] fix: requiredFitness was being set to one higher than the actual requirement --- fdbserver/ClusterController.actor.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 8480566914..903c6cf2b8 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -426,12 +426,12 @@ public: auto fitness = std::get<0>(workerIter->first); auto used = std::get<1>(workerIter->first); if (fitness > requiredFitness || used > requiredUsed) { - requiredFitness = fitness; - requiredUsed = used; if (logServerSet->size() >= required && logServerSet->validate(policy)) { bCompleted = true; break; } + requiredFitness = fitness; + requiredUsed = used; } if (std::get<2>(workerIter->first)) { @@ -501,13 +501,13 @@ public: for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { auto fitness = std::get<0>(workerIter->first); auto used = std::get<1>(workerIter->first); + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } auto addingDegraded = std::get<2>(workerIter->first); if (addingDegraded) { continue; } - if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { - break; - } for (auto& worker : workerIter->second) { logServerMap->add(worker.interf.locality, &worker); } @@ -524,14 +524,14 @@ public: for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { auto fitness = std::get<0>(workerIter->first); auto used = std::get<1>(workerIter->first); + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } auto addingDegraded = std::get<2>(workerIter->first); auto inCCDC = std::get<3>(workerIter->first); if (inCCDC || (!requiredDegraded && addingDegraded)) { continue; } - if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { - break; - } for (auto& worker : workerIter->second) { logServerMap->add(worker.interf.locality, &worker); } @@ -545,14 +545,14 @@ public: for (auto workerIter = fitness_workers.begin(); workerIter != fitness_workers.end(); ++workerIter) { auto fitness = std::get<0>(workerIter->first); auto used = std::get<1>(workerIter->first); + if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { + break; + } auto addingDegraded = std::get<2>(workerIter->first); auto inCCDC = std::get<3>(workerIter->first); if ((!requiredInCCDC && inCCDC) || (!requiredDegraded && addingDegraded)) { continue; } - if (fitness > requiredFitness || (fitness == requiredFitness && used > requiredUsed)) { - break; - } for (auto& worker : workerIter->second) { logServerMap->add(worker.interf.locality, &worker); } From a90c26f1d03bb61acc64923f489b622ab6ccf6ea Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Thu, 8 Apr 2021 14:29:12 -0700 Subject: [PATCH 14/60] The master, proxies, and resolver all need to have the same machine class fitness function besides best fit to ensure recruitment is deterministic if the first GRV proxy or resolver is forced to share a process, it should prefer to share with the commit proxy so that the commit proxy has more potential options it can share with --- fdbrpc/Locality.cpp | 22 +--- fdbserver/ClusterController.actor.cpp | 159 ++++++++++++++++++-------- 2 files changed, 116 insertions(+), 65 deletions(-) diff --git a/fdbrpc/Locality.cpp b/fdbrpc/Locality.cpp index 3cf70943e0..8cdc0751c4 100644 --- a/fdbrpc/Locality.cpp +++ b/fdbrpc/Locality.cpp @@ -63,7 +63,7 @@ ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const default: return ProcessClass::NeverAssign; } - case ProcessClass::CommitProxy: + case ProcessClass::CommitProxy: // Resolver, Master, CommitProxy, and GrvProxy need to be the same besides best fit switch (_class) { case ProcessClass::CommitProxyClass: return ProcessClass::BestFit; @@ -71,10 +71,6 @@ ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const return ProcessClass::GoodFit; case ProcessClass::UnsetClass: return ProcessClass::UnsetFit; - case ProcessClass::GrvProxyClass: - return ProcessClass::OkayFit; - case ProcessClass::ResolutionClass: - return ProcessClass::OkayFit; case ProcessClass::TransactionClass: return ProcessClass::OkayFit; case ProcessClass::CoordinatorClass: @@ -84,7 +80,7 @@ ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const default: return ProcessClass::WorstFit; } - case ProcessClass::GrvProxy: + case ProcessClass::GrvProxy: // Resolver, Master, CommitProxy, and GrvProxy need to be the same besides best fit switch (_class) { case ProcessClass::GrvProxyClass: return ProcessClass::BestFit; @@ -92,10 +88,6 @@ ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const return ProcessClass::GoodFit; case ProcessClass::UnsetClass: return ProcessClass::UnsetFit; - case ProcessClass::CommitProxyClass: - return ProcessClass::OkayFit; - case ProcessClass::ResolutionClass: - return ProcessClass::OkayFit; case ProcessClass::TransactionClass: return ProcessClass::OkayFit; case ProcessClass::CoordinatorClass: @@ -105,7 +97,7 @@ ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const default: return ProcessClass::WorstFit; } - case ProcessClass::Master: + case ProcessClass::Master: // Resolver, Master, CommitProxy, and GrvProxy need to be the same besides best fit switch (_class) { case ProcessClass::MasterClass: return ProcessClass::BestFit; @@ -113,7 +105,7 @@ ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const return ProcessClass::GoodFit; case ProcessClass::UnsetClass: return ProcessClass::UnsetFit; - case ProcessClass::ResolutionClass: + case ProcessClass::TransactionClass: return ProcessClass::OkayFit; case ProcessClass::CoordinatorClass: case ProcessClass::TesterClass: @@ -122,7 +114,7 @@ ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const default: return ProcessClass::WorstFit; } - case ProcessClass::Resolver: + case ProcessClass::Resolver: // Resolver, Master, CommitProxy, and GrvProxy need to be the same besides best fit switch (_class) { case ProcessClass::ResolutionClass: return ProcessClass::BestFit; @@ -147,8 +139,6 @@ ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const return ProcessClass::GoodFit; case ProcessClass::UnsetClass: return ProcessClass::UnsetFit; - case ProcessClass::ResolutionClass: - return ProcessClass::OkayFit; case ProcessClass::TransactionClass: return ProcessClass::OkayFit; case ProcessClass::CoordinatorClass: @@ -167,8 +157,6 @@ ProcessClass::Fitness ProcessClass::machineClassFitness(ClusterRole role) const return ProcessClass::GoodFit; case ProcessClass::UnsetClass: return ProcessClass::UnsetFit; - case ProcessClass::ResolutionClass: - return ProcessClass::OkayFit; case ProcessClass::TransactionClass: return ProcessClass::OkayFit; case ProcessClass::CoordinatorClass: diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 903c6cf2b8..97249dccf8 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -727,14 +727,15 @@ public: return bestFitness; } - WorkerFitnessInfo getWorkerForRoleInDatacenter(Optional> const& dcId, - ProcessClass::ClusterRole role, - ProcessClass::Fitness unacceptableFitness, - DatabaseConfiguration const& conf, - std::map>, int>& id_used, - bool checkStable = false) { - std::map, std::pair, vector>> - fitness_workers; + WorkerFitnessInfo getWorkerForRoleInDatacenter( + Optional> const& dcId, + ProcessClass::ClusterRole role, + ProcessClass::Fitness unacceptableFitness, + DatabaseConfiguration const& conf, + std::map>, int>& id_used, + Optional> preferredSharing = Optional>(), + bool checkStable = false) { + std::map, vector> fitness_workers; for (auto& it : id_worker) { auto fitness = it.second.details.processClass.machineClassFitness(role); @@ -743,23 +744,20 @@ public: } if (workerAvailable(it.second, checkStable) && fitness < unacceptableFitness && it.second.details.interf.locality.dcId() == dcId) { - if (isLongLivedStateless(it.first)) { - fitness_workers[std::make_pair(fitness, id_used[it.first])].second.push_back(it.second.details); - } else { - fitness_workers[std::make_pair(fitness, id_used[it.first])].first.push_back(it.second.details); - } + fitness_workers[std::make_tuple(fitness, + id_used[it.first], + isLongLivedStateless(it.first), + preferredSharing != it.first)] + .push_back(it.second.details); } } - for (auto& it : fitness_workers) { - for (int j = 0; j < 2; j++) { - auto& w = j == 0 ? it.second.first : it.second.second; - deterministicRandom()->randomShuffle(w); - for (int i = 0; i < w.size(); i++) { - id_used[w[i].interf.locality.processId()]++; - return WorkerFitnessInfo(w[i], std::max(ProcessClass::GoodFit, it.first.first), it.first.second); - } - } + if (fitness_workers.size()) { + auto worker = deterministicRandom()->randomChoice(fitness_workers.begin()->second); + id_used[worker.interf.locality.processId()]++; + return WorkerFitnessInfo(worker, + std::max(ProcessClass::GoodFit, std::get<0>(fitness_workers.begin()->first)), + std::get<1>(fitness_workers.begin()->first)); } throw no_more_servers(); @@ -1032,10 +1030,18 @@ public: auto first_commit_proxy = getWorkerForRoleInDatacenter( dcId, ProcessClass::CommitProxy, ProcessClass::ExcludeFit, req.configuration, id_used); - auto first_grv_proxy = getWorkerForRoleInDatacenter( - dcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, req.configuration, id_used); - auto first_resolver = getWorkerForRoleInDatacenter( - dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, req.configuration, id_used); + auto first_grv_proxy = getWorkerForRoleInDatacenter(dcId, + ProcessClass::GrvProxy, + ProcessClass::ExcludeFit, + req.configuration, + id_used, + first_commit_proxy.worker.interf.locality.processId()); + auto first_resolver = getWorkerForRoleInDatacenter(dcId, + ProcessClass::Resolver, + ProcessClass::ExcludeFit, + req.configuration, + id_used, + first_commit_proxy.worker.interf.locality.processId()); // If one of the first process recruitments is forced to share a process, allow all of next recruitments // to also share a process. @@ -1224,10 +1230,20 @@ public: auto used = id_used; auto first_commit_proxy = getWorkerForRoleInDatacenter( dcId, ProcessClass::CommitProxy, ProcessClass::ExcludeFit, req.configuration, used); - auto first_grv_proxy = getWorkerForRoleInDatacenter( - dcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, req.configuration, used); - auto first_resolver = getWorkerForRoleInDatacenter( - dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, req.configuration, used); + auto first_grv_proxy = + getWorkerForRoleInDatacenter(dcId, + ProcessClass::GrvProxy, + ProcessClass::ExcludeFit, + req.configuration, + used, + first_commit_proxy.worker.interf.locality.processId()); + auto first_resolver = + getWorkerForRoleInDatacenter(dcId, + ProcessClass::Resolver, + ProcessClass::ExcludeFit, + req.configuration, + used, + first_commit_proxy.worker.interf.locality.processId()); // If one of the first process recruitments is forced to share a process, allow all of next // recruitments to also share a process. @@ -1356,10 +1372,20 @@ public: try { std::map>, int> id_used; - getWorkerForRoleInDatacenter( - regions[0].dcId, ProcessClass::ClusterController, ProcessClass::ExcludeFit, db.config, id_used, true); - getWorkerForRoleInDatacenter( - regions[0].dcId, ProcessClass::Master, ProcessClass::ExcludeFit, db.config, id_used, true); + getWorkerForRoleInDatacenter(regions[0].dcId, + ProcessClass::ClusterController, + ProcessClass::ExcludeFit, + db.config, + id_used, + Optional>(), + true); + getWorkerForRoleInDatacenter(regions[0].dcId, + ProcessClass::Master, + ProcessClass::ExcludeFit, + db.config, + id_used, + Optional>(), + true); std::set> primaryDC; primaryDC.insert(regions[0].dcId); @@ -1375,12 +1401,27 @@ public: getWorkersForSatelliteLogs(db.config, regions[0], regions[1], id_used, satelliteFallback, true); } - getWorkerForRoleInDatacenter( - regions[0].dcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, db.config, id_used, true); - getWorkerForRoleInDatacenter( - regions[0].dcId, ProcessClass::CommitProxy, ProcessClass::ExcludeFit, db.config, id_used, true); - getWorkerForRoleInDatacenter( - regions[0].dcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, db.config, id_used, true); + getWorkerForRoleInDatacenter(regions[0].dcId, + ProcessClass::Resolver, + ProcessClass::ExcludeFit, + db.config, + id_used, + Optional>(), + true); + getWorkerForRoleInDatacenter(regions[0].dcId, + ProcessClass::CommitProxy, + ProcessClass::ExcludeFit, + db.config, + id_used, + Optional>(), + true); + getWorkerForRoleInDatacenter(regions[0].dcId, + ProcessClass::GrvProxy, + ProcessClass::ExcludeFit, + db.config, + id_used, + Optional>(), + true); vector> dcPriority; dcPriority.push_back(regions[0].dcId); @@ -1592,8 +1633,13 @@ public: std::map>, int> old_id_used; id_used[clusterControllerProcessId]++; old_id_used[clusterControllerProcessId]++; - WorkerFitnessInfo mworker = getWorkerForRoleInDatacenter( - clusterControllerDcId, ProcessClass::Master, ProcessClass::NeverAssign, db.config, id_used, true); + WorkerFitnessInfo mworker = getWorkerForRoleInDatacenter(clusterControllerDcId, + ProcessClass::Master, + ProcessClass::NeverAssign, + db.config, + id_used, + Optional>(), + true); auto newMasterFit = mworker.worker.processClass.machineClassFitness(ProcessClass::Master); if (db.config.isExcludedServer(mworker.worker.interf.addresses())) { newMasterFit = std::max(newMasterFit, ProcessClass::ExcludeFit); @@ -1781,12 +1827,27 @@ public: RoleFitness oldGrvProxyFit(grvProxyClasses, ProcessClass::GrvProxy, old_id_used, clusterControllerDcId); RoleFitness oldResolverFit(resolverClasses, ProcessClass::Resolver, old_id_used, clusterControllerDcId); - auto first_commit_proxy = getWorkerForRoleInDatacenter( - clusterControllerDcId, ProcessClass::CommitProxy, ProcessClass::ExcludeFit, db.config, id_used, true); - auto first_grv_proxy = getWorkerForRoleInDatacenter( - clusterControllerDcId, ProcessClass::GrvProxy, ProcessClass::ExcludeFit, db.config, id_used, true); - auto first_resolver = getWorkerForRoleInDatacenter( - clusterControllerDcId, ProcessClass::Resolver, ProcessClass::ExcludeFit, db.config, id_used, true); + auto first_commit_proxy = getWorkerForRoleInDatacenter(clusterControllerDcId, + ProcessClass::CommitProxy, + ProcessClass::ExcludeFit, + db.config, + id_used, + Optional>(), + true); + auto first_grv_proxy = getWorkerForRoleInDatacenter(clusterControllerDcId, + ProcessClass::GrvProxy, + ProcessClass::ExcludeFit, + db.config, + id_used, + first_commit_proxy.worker.interf.locality.processId(), + true); + auto first_resolver = getWorkerForRoleInDatacenter(clusterControllerDcId, + ProcessClass::Resolver, + ProcessClass::ExcludeFit, + db.config, + id_used, + first_commit_proxy.worker.interf.locality.processId(), + true); auto maxUsed = std::max({ first_commit_proxy.used, first_grv_proxy.used, first_resolver.used }); first_commit_proxy.used = maxUsed; first_grv_proxy.used = maxUsed; @@ -2266,6 +2327,7 @@ void checkBetterDDOrRK(ClusterControllerData* self) { ProcessClass::NeverAssign, self->db.config, id_used, + Optional>(), true) .worker; if (self->onMasterIsBetter(newRKWorker, ProcessClass::Ratekeeper)) { @@ -2281,6 +2343,7 @@ void checkBetterDDOrRK(ClusterControllerData* self) { ProcessClass::NeverAssign, self->db.config, id_used, + Optional>(), true) .worker; if (self->onMasterIsBetter(newDDWorker, ProcessClass::DataDistributor)) { From 1d701e8bcfcd01b31949f92e095fd405b4826cfd Mon Sep 17 00:00:00 2001 From: RenxuanW Date: Thu, 8 Apr 2021 14:38:37 -0700 Subject: [PATCH 15/60] Log a warning when remote dc's priority doesn't match the original primary. --- fdbserver/ClusterController.actor.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 55770d6f3b..f73720bf41 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -1058,8 +1058,14 @@ public: RecruitFromConfigurationReply findWorkersForConfiguration(RecruitFromConfigurationRequest const& req) { if (req.configuration.regions.size() > 1) { std::vector regions = req.configuration.regions; - if (regions[0].priority == regions[1].priority && regions[1].dcId == clusterControllerDcId.get()) { - std::swap(regions[0], regions[1]); + if (regions[1].dcId == clusterControllerDcId.get()) { + if (regions[1].priority == regions[0].priority) { + std::swap(regions[0], regions[1]); + } + } else { + TraceEvent(SevWarn, "DcPriorityUnmatch") + .detail("DcId", regions[1].dcId) + .detail("Priority", regions[1].priority); } if (regions[1].dcId == clusterControllerDcId.get() && regions[1].priority >= 0 && From f3d5fa47502a2ae40b7540097131adde84bab229 Mon Sep 17 00:00:00 2001 From: RenxuanW Date: Thu, 8 Apr 2021 15:19:43 -0700 Subject: [PATCH 16/60] Revert "Log a warning when remote dc's priority doesn't match the original primary." This reverts commit 1d701e8bcfcd01b31949f92e095fd405b4826cfd. --- fdbserver/ClusterController.actor.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index f73720bf41..55770d6f3b 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -1058,14 +1058,8 @@ public: RecruitFromConfigurationReply findWorkersForConfiguration(RecruitFromConfigurationRequest const& req) { if (req.configuration.regions.size() > 1) { std::vector regions = req.configuration.regions; - if (regions[1].dcId == clusterControllerDcId.get()) { - if (regions[1].priority == regions[0].priority) { - std::swap(regions[0], regions[1]); - } - } else { - TraceEvent(SevWarn, "DcPriorityUnmatch") - .detail("DcId", regions[1].dcId) - .detail("Priority", regions[1].priority); + if (regions[0].priority == regions[1].priority && regions[1].dcId == clusterControllerDcId.get()) { + std::swap(regions[0], regions[1]); } if (regions[1].dcId == clusterControllerDcId.get() && regions[1].priority >= 0 && From 738e7402f7ad5bdfe047ced9864086837187ec79 Mon Sep 17 00:00:00 2001 From: RenxuanW Date: Thu, 8 Apr 2021 15:36:52 -0700 Subject: [PATCH 17/60] Log a warning when remote dc is disabled (priority < 0) --- fdbserver/ClusterController.actor.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 55770d6f3b..0d00bd2086 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -1062,9 +1062,15 @@ public: std::swap(regions[0], regions[1]); } - if (regions[1].dcId == clusterControllerDcId.get() && regions[1].priority >= 0 && + if (regions[1].dcId == clusterControllerDcId.get() && (!versionDifferenceUpdated || datacenterVersionDifference >= SERVER_KNOBS->MAX_VERSION_DIFFERENCE)) { - std::swap(regions[0], regions[1]); + if (regions[1].priority >= 0) { + std::swap(regions[0], regions[1]); + } else { + TraceEvent(SevWarn, "DcPriorityNegative") + .detail("DcId", regions[1].dcId) + .detail("Priority", regions[1].priority); + } } bool setPrimaryDesired = false; From 7be8dab045a725028a83e97371d559f58bbee61f Mon Sep 17 00:00:00 2001 From: RenxuanW Date: Thu, 8 Apr 2021 16:00:37 -0700 Subject: [PATCH 18/60] Change DcPriorityNegative to CCDcPriorityNegative --- fdbserver/ClusterController.actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 0d00bd2086..7fc5d51d37 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -1067,7 +1067,7 @@ public: if (regions[1].priority >= 0) { std::swap(regions[0], regions[1]); } else { - TraceEvent(SevWarn, "DcPriorityNegative") + TraceEvent(SevWarnAlways, "CCDcPriorityNegative") .detail("DcId", regions[1].dcId) .detail("Priority", regions[1].priority); } From eb4c80db39deb94cb4eb1e2d72364eb097db030a Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Mon, 12 Apr 2021 23:15:17 +0000 Subject: [PATCH 19/60] Respect the version constraints for restart tests in ctest --- tests/TestRunner/TestRunner.py | 49 ++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/tests/TestRunner/TestRunner.py b/tests/TestRunner/TestRunner.py index 4e9bb1d5c0..207bec08c0 100755 --- a/tests/TestRunner/TestRunner.py +++ b/tests/TestRunner/TestRunner.py @@ -264,6 +264,40 @@ def process_traces(basedir, testname, path, out, aggregationPolicy, symbolicateB parser.writeObject({'CMakeSEED': str(cmake_seed)}) return res +class RestartTestPolicy: + def __init__(self, name, old_binary, new_binary): + # Default is to use the same binary for the restart test, unless constraints are satisfied. + self._first_binary = new_binary + self._second_binary = new_binary + if old_binary is None: + _logger.info("No old binary provided") + old_binary_version_raw = subprocess.check_output([old_binary, '--version']).decode('utf-8') + match = re.match('FoundationDB.*\(v([0-9]+\.[0-9]+\.[0-9]+)\)', old_binary_version_raw) + assert match, old_binary_version_raw + old_binary_version = tuple(map(int, match.group(1).split('.'))) + match = re.match('.*/restarting/from_([0-9]+\.[0-9]+\.[0-9]+)/', name) + if match: # upgrading _from_ + lower_bound = tuple(map(int, match.group(1).split('.'))) + if old_binary_version >= lower_bound: + self._first_binary = old_binary + _logger.info("Using old binary as first binary: {} >= {}".format(old_binary_version, lower_bound)) + else: + _logger.info("Using new binary as first binary: {} < {}".format(old_binary_version, lower_bound)) + match = re.match('.*/restarting/to_([0-9]+\.[0-9]+\.[0-9]+)/', name) + if match: # downgrading _to_ + lower_bound = tuple(map(int, match.group(1).split('.'))) + if old_binary_version >= lower_bound: + self._second_binary = old_binary + _logger.info("Using old binary as second binary: {} >= {}".format(old_binary_version, lower_bound)) + else: + _logger.info("Using new binary as second binary: {} < {}".format(old_binary_version, lower_bound)) + + def first_binary(self): + return self._first_binary + + def second_binary(self): + return self._second_binary + def run_simulation_test(basedir, options): fdbserver = os.path.join(basedir, 'bin', 'fdbserver') pargs = [fdbserver, @@ -298,14 +332,19 @@ def run_simulation_test(basedir, options): os.mkdir(wd) return_codes = {} # {command: return_code} first = True + restart_test_policy = None + if len(options.testfile) > 1: + restart_test_policy = RestartTestPolicy(options.testfile[0], options.old_binary, fdbserver) for testfile in options.testfile: tmp = list(pargs) - # old_binary is not under test, so don't run under valgrind valgrind_args = [] - if first and options.old_binary is not None and len(options.testfile) > 1: - _logger.info("Run old binary at {}".format(options.old_binary)) - tmp[0] = options.old_binary - elif options.use_valgrind: + if restart_test_policy is not None: + if first: + tmp[0] = restart_test_policy.first_binary() + else: + tmp[0] = restart_test_policy.second_binary() + # old_binary is not under test, so don't run under valgrind + if options.use_valgrind and tmp[0] == fdbserver: valgrind_args = ['valgrind', '--error-exitcode=99', '--'] if not first: tmp.append('-R') From bd6db9ca7cc4a789c5738d16f128e895c8ad15b6 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Tue, 13 Apr 2021 15:13:45 -0700 Subject: [PATCH 20/60] Update fdbserver/ClusterController.actor.cpp Co-authored-by: Markus Pilman --- fdbserver/ClusterController.actor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 97249dccf8..543abc8dad 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -855,8 +855,8 @@ public: auto thisFit = it.processClass.machineClassFitness(role); worstFit = std::max(worstFit, thisFit); bestFit = std::min(bestFit, thisFit); - degraded |= it.degraded; - inClusterControllerDC |= (it.interf.locality.dcId() == ccDcId); + degraded = it.degraded || degraded; + inClusterControllerDC = (it.interf.locality.dcId() == ccDcId) || inClusterControllerDC; auto thisUsed = id_used.find(it.interf.locality.processId()); if (thisUsed == id_used.end()) { From 9475b6a5dd6ca87a2b42f5a950ffe7039e8e046e Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 13 Apr 2021 20:15:19 -0700 Subject: [PATCH 21/60] Correctness fix, prevent AsyncFileNonDurable from always making file writes take up to 5 seconds. --- fdbrpc/AsyncFileNonDurable.actor.h | 6 +++--- flow/Knobs.cpp | 1 + flow/Knobs.h | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h index b682a7741b..c55cd8494e 100644 --- a/fdbrpc/AsyncFileNonDurable.actor.h +++ b/fdbrpc/AsyncFileNonDurable.actor.h @@ -197,7 +197,7 @@ private: this->file = file; this->filename = filename; this->diskParameters = diskParameters; - maxWriteDelay = 5.0; + maxWriteDelay = deterministicRandom()->random01() * FLOW_KNOBS->NON_DURABLE_MAX_WRITE_DELAY; hasBeenSynced = false; killMode = (KillMode)deterministicRandom()->randomInt(1, 3); @@ -434,7 +434,7 @@ private: state TaskPriority currentTaskID = g_network->getCurrentTask(); wait(g_simulator.onMachine(currentProcess)); - state double delayDuration = deterministicRandom()->random01() * self->maxWriteDelay; + state double delayDuration = g_simulator.speedUpSimulation ? 0.0001 : deterministicRandom()->random01() * self->maxWriteDelay; state Standalone dataCopy(StringRef((uint8_t*)data, length)); state Future startSyncFuture = self->startSyncPromise.getFuture(); @@ -606,7 +606,7 @@ private: state TaskPriority currentTaskID = g_network->getCurrentTask(); wait(g_simulator.onMachine(currentProcess)); - state double delayDuration = deterministicRandom()->random01() * self->maxWriteDelay; + state double delayDuration = g_simulator.speedUpSimulation ? 0.0001 : deterministicRandom()->random01() * self->maxWriteDelay; state Future startSyncFuture = self->startSyncPromise.getFuture(); try { diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index a173ba43bf..6dc77e2fb2 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -135,6 +135,7 @@ void FlowKnobs::initialize(bool randomize, bool isSimulated) { init( DISABLE_POSIX_KERNEL_AIO, 0 ); //AsyncFileNonDurable + init( NON_DURABLE_MAX_WRITE_DELAY, 0.0001 ); if( randomize && BUGGIFY ) NON_DURABLE_MAX_WRITE_DELAY = 5.0; init( MAX_PRIOR_MODIFICATION_DELAY, 1.0 ); if( randomize && BUGGIFY ) MAX_PRIOR_MODIFICATION_DELAY = 10.0; //GenericActors diff --git a/flow/Knobs.h b/flow/Knobs.h index ab088382f8..67ec3b82b7 100644 --- a/flow/Knobs.h +++ b/flow/Knobs.h @@ -149,6 +149,7 @@ public: int DISABLE_POSIX_KERNEL_AIO; // AsyncFileNonDurable + double NON_DURABLE_MAX_WRITE_DELAY; double MAX_PRIOR_MODIFICATION_DELAY; // GenericActors From 1c5013f6ecc04eb89541abcd14dfe7ddc7b27445 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 7 Apr 2021 18:39:06 -0700 Subject: [PATCH 22/60] Removed btree cleanup parameter override. --- tests/rare/RedwoodCorrectnessBTree.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/rare/RedwoodCorrectnessBTree.toml b/tests/rare/RedwoodCorrectnessBTree.toml index db21848a4b..c39098e4cc 100644 --- a/tests/rare/RedwoodCorrectnessBTree.toml +++ b/tests/rare/RedwoodCorrectnessBTree.toml @@ -7,4 +7,3 @@ startDelay = 0 testName = 'UnitTests' maxTestCases = 0 testsMatching = '/redwood/correctness/btree' - remapCleanupWindow = 1000000000 From f74748ebac9fda2626cc71f8370c6b1d7ed2bd9a Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 13 Apr 2021 20:43:12 -0700 Subject: [PATCH 23/60] Applied clang-format. --- fdbrpc/AsyncFileNonDurable.actor.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h index c55cd8494e..9997a8f4b2 100644 --- a/fdbrpc/AsyncFileNonDurable.actor.h +++ b/fdbrpc/AsyncFileNonDurable.actor.h @@ -434,7 +434,8 @@ private: state TaskPriority currentTaskID = g_network->getCurrentTask(); wait(g_simulator.onMachine(currentProcess)); - state double delayDuration = g_simulator.speedUpSimulation ? 0.0001 : deterministicRandom()->random01() * self->maxWriteDelay; + state double delayDuration = + g_simulator.speedUpSimulation ? 0.0001 : deterministicRandom()->random01() * self->maxWriteDelay; state Standalone dataCopy(StringRef((uint8_t*)data, length)); state Future startSyncFuture = self->startSyncPromise.getFuture(); @@ -606,7 +607,8 @@ private: state TaskPriority currentTaskID = g_network->getCurrentTask(); wait(g_simulator.onMachine(currentProcess)); - state double delayDuration = g_simulator.speedUpSimulation ? 0.0001 : deterministicRandom()->random01() * self->maxWriteDelay; + state double delayDuration = + g_simulator.speedUpSimulation ? 0.0001 : deterministicRandom()->random01() * self->maxWriteDelay; state Future startSyncFuture = self->startSyncPromise.getFuture(); try { From 1958fde5c6751f13e633f9197610cbc22d1cf0a8 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 13 Apr 2021 20:49:04 -0700 Subject: [PATCH 24/60] Added parentheses for clarity. --- fdbrpc/AsyncFileNonDurable.actor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h index 9997a8f4b2..fe3d3a4137 100644 --- a/fdbrpc/AsyncFileNonDurable.actor.h +++ b/fdbrpc/AsyncFileNonDurable.actor.h @@ -435,7 +435,7 @@ private: wait(g_simulator.onMachine(currentProcess)); state double delayDuration = - g_simulator.speedUpSimulation ? 0.0001 : deterministicRandom()->random01() * self->maxWriteDelay; + g_simulator.speedUpSimulation ? 0.0001 : (deterministicRandom()->random01() * self->maxWriteDelay); state Standalone dataCopy(StringRef((uint8_t*)data, length)); state Future startSyncFuture = self->startSyncPromise.getFuture(); @@ -608,7 +608,7 @@ private: wait(g_simulator.onMachine(currentProcess)); state double delayDuration = - g_simulator.speedUpSimulation ? 0.0001 : deterministicRandom()->random01() * self->maxWriteDelay; + g_simulator.speedUpSimulation ? 0.0001 : (deterministicRandom()->random01() * self->maxWriteDelay); state Future startSyncFuture = self->startSyncPromise.getFuture(); try { From eab468fecca3c166296db892f5d297add4da8df8 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 14 Apr 2021 09:32:48 -0700 Subject: [PATCH 25/60] Remove extra line caused by commit issue --- fdbclient/NativeAPI.actor.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index df9e08169f..9f6784e279 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -2477,7 +2477,6 @@ ACTOR Future watchValue(Future version, cx->invalidateCache(key); wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, info.taskID)); } else if (e.code() == error_code_watch_cancelled || e.code() == error_code_process_behind) { - TEST(e.code() == error_code_watch_cancelled); // Too many watches on the storage server, poll for changes instead TEST(e.code() == error_code_watch_cancelled); // Too many watches on storage server, poll for changes TEST(e.code() == error_code_process_behind); // The storage servers are all behind wait(delay(CLIENT_KNOBS->WATCH_POLLING_TIME, info.taskID)); From f1415412f1bae3da4931eaa88f1413c179f02f41 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 12 Feb 2021 18:55:01 -0800 Subject: [PATCH 26/60] Add global configuration framework implementation --- fdbclient/CMakeLists.txt | 2 + fdbclient/CommitProxyInterface.h | 12 +-- fdbclient/GlobalConfig.actor.cpp | 60 ++++++++++++ fdbclient/GlobalConfig.actor.h | 132 ++++++++++++++++++++++++++ fdbclient/NativeAPI.actor.cpp | 13 ++- fdbclient/SpecialKeySpace.actor.cpp | 127 ++++++++++++++++++++++++- fdbclient/SpecialKeySpace.actor.h | 11 +++ fdbclient/SystemData.cpp | 13 ++- fdbclient/SystemData.h | 24 +++++ fdbserver/ClusterController.actor.cpp | 55 +++++++++-- flow/network.h | 3 +- 11 files changed, 426 insertions(+), 26 deletions(-) create mode 100644 fdbclient/GlobalConfig.actor.cpp create mode 100644 fdbclient/GlobalConfig.actor.h diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt index 129f9e7d3e..e733259611 100644 --- a/fdbclient/CMakeLists.txt +++ b/fdbclient/CMakeLists.txt @@ -28,6 +28,8 @@ set(FDBCLIENT_SRCS FDBOptions.h FDBTypes.h FileBackupAgent.actor.cpp + GlobalConfig.actor.h + GlobalConfig.actor.cpp GrvProxyInterface.h HTTP.actor.cpp IClientApi.h diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h index a166a87dfa..f29d7369b3 100644 --- a/fdbclient/CommitProxyInterface.h +++ b/fdbclient/CommitProxyInterface.h @@ -113,6 +113,7 @@ struct ClientDBInfo { vector commitProxies; Optional firstCommitProxy; // not serialized, used for commitOnFirstProxy when the commit proxies vector has been shrunk + vector>>> history; double clientTxnInfoSampleRate; int64_t clientTxnInfoSizeLimit; Optional forward; @@ -132,15 +133,8 @@ struct ClientDBInfo { if constexpr (!is_fb_function) { ASSERT(ar.protocolVersion().isValid()); } - serializer(ar, - grvProxies, - commitProxies, - id, - clientTxnInfoSampleRate, - clientTxnInfoSizeLimit, - forward, - transactionTagSampleRate, - transactionTagSampleCost); + serializer(ar, grvProxies, commitProxies, id, history, clientTxnInfoSampleRate, clientTxnInfoSizeLimit, + forward, transactionTagSampleRate, transactionTagSampleCost); } }; diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp new file mode 100644 index 0000000000..f4c2c81e5d --- /dev/null +++ b/fdbclient/GlobalConfig.actor.cpp @@ -0,0 +1,60 @@ +/* + * GlobalConfig.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbclient/GlobalConfig.actor.h" + +#include "flow/actorcompiler.h" // This must be the last #include. + +GlobalConfig::GlobalConfig() : lastUpdate(0) {} + +void GlobalConfig::create(DatabaseContext* cx, Reference> dbInfo) { + auto config = new GlobalConfig{}; // TODO: memory leak? + config->cx = Database(cx); + g_network->setGlobal(INetwork::enGlobalConfig, config); + config->_updater = updater(config, dbInfo); +} + +GlobalConfig& GlobalConfig::globalConfig() { + void* res = g_network->global(INetwork::enGlobalConfig); + ASSERT(res); + return *reinterpret_cast(res); +} + +const std::any GlobalConfig::get(StringRef name) { + auto it = data.find(name); + if (it == data.end()) { + return nullptr; + } + return it->second; +} + +Future GlobalConfig::onInitialized() { + return initialized.getFuture(); +} + +void GlobalConfig::insert(KeyRef key, ValueRef value) { + Tuple t = Tuple::unpack(value); + // TODO: Add more Tuple types + if (t.getType(0) == Tuple::ElementType::UTF8) { + data[key] = t.getString(0); + } else if (t.getType(0) == Tuple::ElementType::INT) { + data[key] = t.getInt(0); + } +} diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h new file mode 100644 index 0000000000..323a5e953c --- /dev/null +++ b/fdbclient/GlobalConfig.actor.h @@ -0,0 +1,132 @@ +/* + * GlobalConfig.actor.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_GLOBALCONFIG_ACTOR_G_H) +#define FDBCLIENT_GLOBALCONFIG_ACTOR_G_H +#include "fdbclient/GlobalConfig.actor.g.h" +#elif !defined(FDBCLIENT_GLOBALCONFIG_ACTOR_H) +#define FDBCLIENT_GLOBALCONFIG_ACTOR_H + +#include +#include + +#include "fdbclient/CommitProxyInterface.h" +#include "fdbclient/ReadYourWrites.h" +#include "fdbclient/SystemData.h" +#include "fdbclient/Tuple.h" +#include "flow/flow.h" +#include "flow/genericactors.actor.h" +#include "flow/Knobs.h" + +#include "flow/actorcompiler.h" // has to be last include + +class GlobalConfig { +public: + GlobalConfig(); + GlobalConfig(const GlobalConfig&) = delete; + GlobalConfig& operator=(const GlobalConfig&) = delete; + + static void create(DatabaseContext* cx, Reference> dbInfo); + static GlobalConfig& globalConfig(); + const std::any get(StringRef name); + Future onInitialized(); + +private: + void insert(KeyRef key, ValueRef value); + + ACTOR static Future refresh(GlobalConfig* self) { + Transaction tr(self->cx); + Standalone result = wait(tr.getRange(globalConfigDataKeys, CLIENT_KNOBS->TOO_MANY)); + for (const auto& kv : result) { + KeyRef systemKey = kv.key.removePrefix(globalConfigDataPrefix); + self->insert(systemKey, kv.value); + } + return Void(); + } + + ACTOR static Future updater(GlobalConfig* self, Reference> dbInfo) { + wait(refresh(self)); + self->initialized.send(Void()); + + loop { + try { + wait(dbInfo->onChange()); + + auto& history = dbInfo->get().history; + if (history.size() == 0 || (self->lastUpdate < history[0].first && self->lastUpdate != 0)) { + // This process missed too many global configuration + // history updates or the protocol version changed, so it + // must re-read the entire configuration range. + wait(refresh(self)); + self->lastUpdate = dbInfo->get().history.back().contents().first; + } else { + // Apply history in order, from lowest version to highest + // version. Mutation history should already be stored in + // ascending version order. + for (int i = 0; i < history.size(); ++i) { + std::pair> pair = history[i].contents(); + + Version version = pair.first; + if (version <= self->lastUpdate) { + continue; // already applied this mutation + } + + VectorRef& mutations = pair.second; + for (const auto& mutation : mutations) { + if (mutation.type == MutationRef::SetValue) { + self->insert(mutation.param1, mutation.param2); + } else if (mutation.type == MutationRef::ClearRange) { + // TODO: Could be optimized if using std::map.. + KeyRangeRef range(mutation.param1, mutation.param2); + auto it = self->data.begin(); + while (it != self->data.end()) { + if (range.contains(it->first)) { + it = self->data.erase(it); + } else { + ++it; + } + } + } else { + ASSERT(false); + } + } + + ASSERT(version > self->lastUpdate); + self->lastUpdate = version; + } + } + } catch (Error& e) { + throw; + } + } + } + + Database cx; + Future _updater; + Promise initialized; + // TODO: Arena to store all data in + // TODO: Change to std::map for faster range access + std::unordered_map data; + Version lastUpdate; +}; + +#endif diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 9f6784e279..d350a39974 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -36,6 +36,7 @@ #include "fdbclient/ClusterInterface.h" #include "fdbclient/CoordinationInterface.h" #include "fdbclient/DatabaseContext.h" +#include "fdbclient/GlobalConfig.actor.h" #include "fdbclient/JsonBuilder.h" #include "fdbclient/KeyRangeMap.h" #include "fdbclient/Knobs.h" @@ -962,6 +963,8 @@ DatabaseContext::DatabaseContext(ReferenceINIT_MID_SHARD_BYTES); + GlobalConfig::create(this, clientInfo); + if (apiVersionAtLeast(700)) { registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::ERRORMSG, SpecialKeySpace::IMPLTYPE::READONLY, @@ -1018,9 +1021,13 @@ DatabaseContext::DatabaseContext(Reference(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::TRACING))); + SpecialKeySpace::MODULE::GLOBALCONFIG, SpecialKeySpace::IMPLTYPE::READWRITE, + std::make_unique( + SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG))); + registerSpecialKeySpaceModule( + SpecialKeySpace::MODULE::TRACING, SpecialKeySpace::IMPLTYPE::READWRITE, + std::make_unique( + SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::TRACING))); registerSpecialKeySpaceModule( SpecialKeySpace::MODULE::CONFIGURATION, SpecialKeySpace::IMPLTYPE::READWRITE, diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 5fb7360b0d..8e681b9aba 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -21,7 +21,7 @@ #include "boost/lexical_cast.hpp" #include "boost/algorithm/string.hpp" -#include "fdbclient/Knobs.h" +#include "fdbclient/GlobalConfig.actor.h" #include "fdbclient/SpecialKeySpace.actor.h" #include "flow/Arena.h" #include "flow/UnitTest.h" @@ -64,6 +64,8 @@ std::unordered_map SpecialKeySpace::moduleToB { SpecialKeySpace::MODULE::ERRORMSG, singleKeyRange(LiteralStringRef("\xff\xff/error_message")) }, { SpecialKeySpace::MODULE::CONFIGURATION, KeyRangeRef(LiteralStringRef("\xff\xff/configuration/"), LiteralStringRef("\xff\xff/configuration0")) }, + { SpecialKeySpace::MODULE::GLOBALCONFIG, + KeyRangeRef(LiteralStringRef("\xff\xff/global_config/"), LiteralStringRef("\xff\xff/global_config0")) }, { SpecialKeySpace::MODULE::TRACING, KeyRangeRef(LiteralStringRef("\xff\xff/tracing/"), LiteralStringRef("\xff\xff/tracing0")) } }; @@ -1369,11 +1371,128 @@ Future> ConsistencyCheckImpl::commit(ReadYourWritesTransac return Optional(); } -TracingOptionsImpl::TracingOptionsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) { - TraceEvent("TracingOptionsImpl::TracingOptionsImpl").detail("Range", kr); +GlobalConfigImpl::GlobalConfigImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} + +Future> GlobalConfigImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr) const { + Standalone result; + + // if (kr.begin != kr.end) { + // ryw->setSpecialKeySpaceErrorMsg("get range disabled, please fetch a single key"); + // throw special_keys_api_failure(); + // } + + auto& globalConfig = GlobalConfig::globalConfig(); + KeyRef key = kr.begin.removePrefix(getKeyRange().begin); + const std::any& any = globalConfig.get(key); + if (any.has_value()) { + if (any.type() == typeid(Standalone)) { + result.push_back_deep(result.arena(), KeyValueRef(kr.begin, std::any_cast>(globalConfig.get(key)).contents())); + } else if (any.type() == typeid(int64_t)) { + result.push_back_deep(result.arena(), KeyValueRef(kr.begin, std::to_string(std::any_cast(globalConfig.get(key))))); + } else { + ASSERT(false); + } + } + return result; } -Future> TracingOptionsImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { +void GlobalConfigImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) { + ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(value))); +} + +ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* globalConfig, ReadYourWritesTransaction* ryw) { + state Transaction& tr = ryw->getTransaction(); + + // History should only contain three most recent updates. If it currently + // has three items, remove the oldest to make room for a new item. + Standalone history = wait(tr.getRange(globalConfigHistoryKeys, CLIENT_KNOBS->TOO_MANY, false, true)); + constexpr int kGlobalConfigMaxHistorySize = 3; + if (history.size() > kGlobalConfigMaxHistorySize - 1) { + std::vector keys; + for (const auto& kv : history) { + keys.push_back(kv.key); + } + // Fix ordering of returned keys. This will ensure versions are ordered + // numerically; for example \xff/globalConfig/h/1000 should come after + // \xff/globalConfig/h/999. + std::sort(keys.begin(), keys.end(), [](const KeyRef& lhs, const KeyRef& rhs) { + if (lhs.size() != rhs.size()) { + return lhs.size() < rhs.size(); + } + return lhs.compare(rhs) < 0; + }); + + // Cannot use a range clear because of how keys are ordered in FDB. + // \xff/globalConfig/h/999 -> ... + // \xff/globalConfig/h/1000 -> ... + // \xff/globalConfig/h/1001 -> ... + // + // clear_range(\xff/globalConfig/h, \xff/globalConfig/h/1000) results + // in zero key-value pairs being deleted (999 is lexicographically + // larger than 1000, and the range is exclusive). + for (int i = 0; i < keys.size() - (kGlobalConfigMaxHistorySize - 1); ++i) { + tr.clear(keys[i]); + } + } + + // TODO: Should probably be using the commit version... + Version readVersion = wait(ryw->getReadVersion()); + BinaryWriter wr = BinaryWriter(AssumeVersion(g_network->protocolVersion())); + + Arena arena; + VectorRef mutations; + + // Transform writes from special-key-space (\xff\xff/global_config/) to + // system key space (\xff/globalConfig/). + state RangeMap>, KeyRangeRef>::Ranges ranges = + ryw->getSpecialKeySpaceWriteMap().containedRanges(specialKeys); + state RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); + while (iter != ranges.end()) { + Key bareKey = iter->begin().removePrefix(globalConfig->getKeyRange().begin); + Key systemKey = bareKey.withPrefix(globalConfigDataPrefix); + std::pair> entry = iter->value(); + if (entry.first) { + if (entry.second.present()) { + mutations.emplace_back_deep(arena, MutationRef(MutationRef::SetValue, bareKey, entry.second.get())); + tr.set(systemKey, entry.second.get()); + } else { + mutations.emplace_back_deep(arena, MutationRef(MutationRef::ClearRange, bareKey, keyAfter(bareKey))); + tr.clear(systemKey); + } + } + ++iter; + } + + wr << std::make_pair(readVersion, mutations); + + // Record the mutations in this commit into the global configuration history. + Key historyVersionKey = globalConfigHistoryPrefix.withSuffix(std::to_string(readVersion)); + tr.set(historyVersionKey, wr.toValue()); + + ProtocolVersion protocolVersion = g_network->protocolVersion(); + BinaryWriter versionWriter = BinaryWriter(AssumeVersion(protocolVersion)); + versionWriter << readVersion << protocolVersion; + tr.set(globalConfigVersionKey, versionWriter.toValue()); + + return Optional(); + +} + +Future> GlobalConfigImpl::commit(ReadYourWritesTransaction* ryw) { + return globalConfigCommitActor(this, ryw); +} + +void GlobalConfigImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { + // TODO +} + +void GlobalConfigImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { + ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); +} + +Future> TracingOptionsImpl::getRange(ReadYourWritesTransaction* ryw, + KeyRangeRef kr) const { Standalone result; for (const auto& option : SpecialKeySpace::getTracingOptions()) { auto key = getKeyRange().begin.withSuffix(option); diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index c760a10724..2f605385c1 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -146,6 +146,7 @@ public: CONFIGURATION, // Configuration of the cluster CONNECTIONSTRING, ERRORMSG, // A single key space contains a json string which describes the last error in special-key-space + GLOBALCONFIG, // Global configuration options synchronized to all nodes MANAGEMENT, // Management-API METRICS, // data-distribution metrics TESTONLY, // only used by correctness tests @@ -336,6 +337,16 @@ public: Future> commit(ReadYourWritesTransaction* ryw) override; }; +class GlobalConfigImpl : public SpecialKeyRangeRWImpl { +public: + explicit GlobalConfigImpl(KeyRangeRef kr); + Future> getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const override; + void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) override; + Future> commit(ReadYourWritesTransaction* ryw) override; + void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override; + void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; +}; + class TracingOptionsImpl : public SpecialKeyRangeRWImpl { public: explicit TracingOptionsImpl(KeyRangeRef kr); diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 0b15f8f91d..aaf115e3b8 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -632,7 +632,18 @@ std::string encodeFailedServersKey(AddressExclusion const& addr) { return failedServersPrefix.toString() + addr.toString(); } -const KeyRangeRef workerListKeys(LiteralStringRef("\xff/worker/"), LiteralStringRef("\xff/worker0")); +const KeyRangeRef globalConfigKeys( LiteralStringRef("\xff/globalConfig/"), LiteralStringRef("\xff/globalConfig0") ); +const KeyRef globalConfigPrefix = globalConfigKeys.begin; + +const KeyRangeRef globalConfigDataKeys( LiteralStringRef("\xff/globalConfig/k/"), LiteralStringRef("\xff/globalConfig/k0") ); +const KeyRef globalConfigDataPrefix = globalConfigDataKeys.begin; + +const KeyRangeRef globalConfigHistoryKeys( LiteralStringRef("\xff/globalConfig/h/"), LiteralStringRef("\xff/globalConfig/h0") ); +const KeyRef globalConfigHistoryPrefix = globalConfigHistoryKeys.begin; + +const KeyRef globalConfigVersionKey = LiteralStringRef("\xff/globalConfig/v"); + +const KeyRangeRef workerListKeys( LiteralStringRef("\xff/worker/"), LiteralStringRef("\xff/worker0") ); const KeyRef workerListPrefix = workerListKeys.begin; const Key workerListKeyFor(StringRef processID) { diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index bbeb7489f9..15117a867e 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -230,6 +230,30 @@ extern const KeyRef failedServersVersionKey; // The value of this key shall be c const AddressExclusion decodeFailedServersKey(KeyRef const& key); // where key.startsWith(failedServersPrefix) std::string encodeFailedServersKey(AddressExclusion const&); +// "\xff/globalConfig/[[option]]" := "value" +// An umbrella prefix for global configuration data synchronized to all nodes. +extern const KeyRangeRef globalConfigData; +extern const KeyRef globalConfigDataPrefix; + +// "\xff/globalConfig/k/[[key]]" := "value" +// Key-value pairs that have been set. The range this keyspace represents +// contains all globally configured options. +extern const KeyRangeRef globalConfigDataKeys; +extern const KeyRef globalConfigDataPrefix; + +// "\xff/globalConfig/h/[[version]]" := "value" +// Maps a commit version to a list of mutations made to the global +// configuration at that commit. Shipped to nodes periodically. In general, +// clients should not write to keys in this keyspace; it will be written +// automatically when updating global configuration keys. +extern const KeyRangeRef globalConfigHistoryKeys; +extern const KeyRef globalConfigHistoryPrefix; + +// "\xff/globalConfig/v" := "version,protocol" +// Read-only key which returns the version and protocol of the most recent +// data written to the global configuration keyspace. +extern const KeyRef globalConfigVersionKey; + // "\xff/workers/[[processID]]" := "" // Asynchronously updated by the cluster controller, this is a list of fdbserver processes that have joined the cluster // and are currently (recently) available diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 543abc8dad..e599e3feae 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -3198,26 +3198,65 @@ ACTOR Future monitorClientTxnInfoConfigs(ClusterControllerData::DBInfo* db try { tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + state Optional globalConfigVersion = wait(tr.get(globalConfigVersionKey)); state Optional rateVal = wait(tr.get(fdbClientInfoTxnSampleRate)); state Optional limitVal = wait(tr.get(fdbClientInfoTxnSizeLimit)); - ClientDBInfo clientInfo = db->clientInfo->get(); - double sampleRate = rateVal.present() - ? BinaryReader::fromStringRef(rateVal.get(), Unversioned()) - : std::numeric_limits::infinity(); - int64_t sizeLimit = - limitVal.present() ? BinaryReader::fromStringRef(limitVal.get(), Unversioned()) : -1; - if (sampleRate != clientInfo.clientTxnInfoSampleRate || - sizeLimit != clientInfo.clientTxnInfoSampleRate) { + state ClientDBInfo clientInfo = db->clientInfo->get(); + + if (globalConfigVersion.present()) { + BinaryReader versionReader = BinaryReader(globalConfigVersion.get(), AssumeVersion(g_network->protocolVersion())); + Version version; + ProtocolVersion protocolVersion; + versionReader >> version >> protocolVersion; + + state Arena arena; + if (protocolVersion == g_network->protocolVersion()) { + Standalone globalConfigHistory = wait(tr.getRange(globalConfigHistoryKeys, CLIENT_KNOBS->TOO_MANY)); + // If the global configuration version key has been + // set, the history should contain at least one item. + ASSERT(globalConfigHistory.size() > 0); + clientInfo.history.clear(); + + for (const auto& kv : globalConfigHistory) { + BinaryReader rd = BinaryReader(kv.value, AssumeVersion(g_network->protocolVersion())); + Standalone>> data; + rd >> data >> arena; + clientInfo.history.push_back(data); + } + + // History should be ordered by version, ascending. + std::sort(clientInfo.history.begin(), clientInfo.history.end(), [](const auto& lhs, const auto& rhs) { + return lhs.first < rhs.first; + }); + } else { + // If the protocol version has changed, the + // GlobalConfig actor should refresh its view by + // reading the entire global configuration key range. + // An empty mutation list will signal the actor to + // refresh. + clientInfo.history.clear(); + } + + clientInfo.id = deterministicRandom()->randomUniqueID(); + db->clientInfo->set(clientInfo); + } + + // TODO: Remove this and move to global config space + double sampleRate = rateVal.present() ? BinaryReader::fromStringRef(rateVal.get(), Unversioned()) : std::numeric_limits::infinity(); + int64_t sizeLimit = limitVal.present() ? BinaryReader::fromStringRef(limitVal.get(), Unversioned()) : -1; + if (sampleRate != clientInfo.clientTxnInfoSampleRate || sizeLimit != clientInfo.clientTxnInfoSampleRate) { clientInfo.id = deterministicRandom()->randomUniqueID(); clientInfo.clientTxnInfoSampleRate = sampleRate; clientInfo.clientTxnInfoSizeLimit = sizeLimit; db->clientInfo->set(clientInfo); } + state Future globalConfigFuture = tr.watch(globalConfigVersionKey); state Future watchRateFuture = tr.watch(fdbClientInfoTxnSampleRate); state Future watchLimitFuture = tr.watch(fdbClientInfoTxnSizeLimit); wait(tr.commit()); choose { + when (wait(globalConfigFuture)) { break; } when(wait(watchRateFuture)) { break; } when(wait(watchLimitFuture)) { break; } } diff --git a/flow/network.h b/flow/network.h index d0f117dede..1eeb5bdc2d 100644 --- a/flow/network.h +++ b/flow/network.h @@ -481,7 +481,8 @@ public: enBlobCredentialFiles = 10, enNetworkAddressesFunc = 11, enClientFailureMonitor = 12, - enSQLiteInjectedError = 13 + enSQLiteInjectedError = 13, + enGlobalConfig = 14 }; virtual void longTaskCheck(const char* name) {} From 9e20b08976b49bbbe7216c55d757b7b4fb43577d Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Wed, 17 Feb 2021 16:04:23 -0800 Subject: [PATCH 27/60] Add float and double parsing --- fdbclient/GlobalConfig.actor.cpp | 9 ++++- fdbclient/Tuple.cpp | 64 ++++++++++++++++++++++++++++++++ fdbclient/Tuple.h | 4 +- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index f4c2c81e5d..8997d11973 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -40,7 +40,7 @@ GlobalConfig& GlobalConfig::globalConfig() { const std::any GlobalConfig::get(StringRef name) { auto it = data.find(name); if (it == data.end()) { - return nullptr; + return std::any{}; } return it->second; } @@ -51,10 +51,15 @@ Future GlobalConfig::onInitialized() { void GlobalConfig::insert(KeyRef key, ValueRef value) { Tuple t = Tuple::unpack(value); - // TODO: Add more Tuple types if (t.getType(0) == Tuple::ElementType::UTF8) { data[key] = t.getString(0); } else if (t.getType(0) == Tuple::ElementType::INT) { data[key] = t.getInt(0); + } else if (t.getType(0) == Tuple::ElementType::FLOAT) { + data[key] = t.getFloat(0); + } else if (t.getType(0) == Tuple::ElementType::DOUBLE) { + data[key] = t.getDouble(0); + } else { + ASSERT(false); } } diff --git a/fdbclient/Tuple.cpp b/fdbclient/Tuple.cpp index 3d4427079f..535be3d7fc 100644 --- a/fdbclient/Tuple.cpp +++ b/fdbclient/Tuple.cpp @@ -20,6 +20,18 @@ #include "fdbclient/Tuple.h" +static float bigEndianFloat(float orig) { + int32_t big = *(int32_t*)&orig; + big = bigEndian32(big); + return *(float*)&big; +} + +static double bigEndianDouble(double orig) { + int64_t big = *(int64_t*)&orig; + big = bigEndian64(big); + return *(double*)&big; +} + static size_t find_string_terminator(const StringRef data, size_t offset) { size_t i = offset; while (i < data.size() - 1 && !(data[i] == '\x00' && data[i + 1] != (uint8_t)'\xff')) { @@ -29,6 +41,19 @@ static size_t find_string_terminator(const StringRef data, size_t offset) { return i; } +// If encoding and the sign bit is 1 (the number is negative), flip all the bits. +// If decoding and the sign bit is 0 (the number is negative), flip all the bits. +// Otherwise, the number is positive, so flip the sign bit. +static void adjust_floating_point(uint8_t *bytes, size_t size, bool encode) { + if((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) { + for(size_t i = 0; i < size; i++) { + bytes[i] ^= (uint8_t)0xff; + } + } else { + bytes[0] ^= (uint8_t)0x80; + } +} + Tuple::Tuple(StringRef const& str, bool exclude_incomplete) { data.append(data.arena(), str.begin(), str.size()); @@ -228,6 +253,45 @@ int64_t Tuple::getInt(size_t index, bool allow_incomplete) const { return swap; } +// TODO: Combine with bindings/flow/Tuple.*. This code is copied from there. +float Tuple::getFloat(size_t index) const { + if(index >= offsets.size()) { + throw invalid_tuple_index(); + } + ASSERT_LT(offsets[index], data.size()); + uint8_t code = data[offsets[index]]; + if(code != 0x20) { + throw invalid_tuple_data_type(); + } + + float swap; + uint8_t* bytes = (uint8_t*)&swap; + ASSERT_LE(offsets[index] + 1 + sizeof(float), data.size()); + swap = *(float*)(data.begin() + offsets[index] + 1); + adjust_floating_point( bytes, sizeof(float), false ); + + return bigEndianFloat(swap); +} + +double Tuple::getDouble(size_t index) const { + if(index >= offsets.size()) { + throw invalid_tuple_index(); + } + ASSERT_LT(offsets[index], data.size()); + uint8_t code = data[offsets[index]]; + if(code != 0x21) { + throw invalid_tuple_data_type(); + } + + double swap; + uint8_t* bytes = (uint8_t*)&swap; + ASSERT_LE(offsets[index] + 1 + sizeof(double), data.size()); + swap = *(double*)(data.begin() + offsets[index] + 1); + adjust_floating_point( bytes, sizeof(double), false ); + + return bigEndianDouble(swap); +} + KeyRange Tuple::range(Tuple const& tuple) const { VectorRef begin; VectorRef end; diff --git a/fdbclient/Tuple.h b/fdbclient/Tuple.h index b44edd73cc..4497f19441 100644 --- a/fdbclient/Tuple.h +++ b/fdbclient/Tuple.h @@ -47,7 +47,7 @@ struct Tuple { return append(t); } - enum ElementType { NULL_TYPE, INT, BYTES, UTF8 }; + enum ElementType { NULL_TYPE, INT, BYTES, UTF8, FLOAT, DOUBLE }; // this is number of elements, not length of data size_t size() const { return offsets.size(); } @@ -55,6 +55,8 @@ struct Tuple { ElementType getType(size_t index) const; Standalone getString(size_t index) const; int64_t getInt(size_t index, bool allow_incomplete = false) const; + float getFloat(size_t index) const; + double getDouble(size_t index) const; KeyRange range(Tuple const& tuple = Tuple()) const; From 7bb0b3d8995a6d40232f8bbd107f010f2e94193d Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Thu, 18 Feb 2021 17:44:42 -0800 Subject: [PATCH 28/60] Use commit version for global configuration updates FIXME: There is a memory issue where the underlying data for values set in the `data` field of GlobalConfig will be freed shortly after being set. --- fdbclient/SpecialKeySpace.actor.cpp | 26 +++++++++++++++----------- fdbserver/ClusterController.actor.cpp | 23 ++++++++++++++++++----- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 8e681b9aba..d2d553f0ca 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1436,10 +1436,6 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl } } - // TODO: Should probably be using the commit version... - Version readVersion = wait(ryw->getReadVersion()); - BinaryWriter wr = BinaryWriter(AssumeVersion(g_network->protocolVersion())); - Arena arena; VectorRef mutations; @@ -1464,16 +1460,24 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl ++iter; } - wr << std::make_pair(readVersion, mutations); + ProtocolVersion protocolVersion = g_network->protocolVersion(); // Record the mutations in this commit into the global configuration history. - Key historyVersionKey = globalConfigHistoryPrefix.withSuffix(std::to_string(readVersion)); - tr.set(historyVersionKey, wr.toValue()); + BinaryWriter historyKeyWriter(AssumeVersion(protocolVersion)); + historyKeyWriter.serializeBytes(globalConfigHistoryPrefix); + Key historyKey = addVersionStampAtEnd(historyKeyWriter.toValue()); - ProtocolVersion protocolVersion = g_network->protocolVersion(); - BinaryWriter versionWriter = BinaryWriter(AssumeVersion(protocolVersion)); - versionWriter << readVersion << protocolVersion; - tr.set(globalConfigVersionKey, versionWriter.toValue()); + BinaryWriter historyMutationsWriter(AssumeVersion(protocolVersion)); + historyMutationsWriter << mutations; + + tr.atomicOp(historyKey, historyMutationsWriter.toValue(), MutationRef::SetVersionstampedKey); + + // Write version key to trigger update in cluster controller. + tr.atomicOp(globalConfigVersionKey, + BinaryWriter::toValue(protocolVersion, AssumeVersion(protocolVersion)) + .withPrefix(LiteralStringRef("0123456789")) // placeholder for versionstamp + .withSuffix(LiteralStringRef("\x00\x00\x00\x00")), + MutationRef::SetVersionstampedValue); return Optional(); diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index e599e3feae..43b53c908c 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -3205,9 +3205,10 @@ ACTOR Future monitorClientTxnInfoConfigs(ClusterControllerData::DBInfo* db if (globalConfigVersion.present()) { BinaryReader versionReader = BinaryReader(globalConfigVersion.get(), AssumeVersion(g_network->protocolVersion())); - Version version; - ProtocolVersion protocolVersion; - versionReader >> version >> protocolVersion; + int64_t commitVersion; + int16_t serializationOrder; + state ProtocolVersion protocolVersion; + versionReader >> commitVersion >> serializationOrder >> protocolVersion; state Arena arena; if (protocolVersion == g_network->protocolVersion()) { @@ -3218,9 +3219,21 @@ ACTOR Future monitorClientTxnInfoConfigs(ClusterControllerData::DBInfo* db clientInfo.history.clear(); for (const auto& kv : globalConfigHistory) { - BinaryReader rd = BinaryReader(kv.value, AssumeVersion(g_network->protocolVersion())); Standalone>> data; - rd >> data >> arena; + + // Read commit version out of versionstamp at end of key. + BinaryReader versionReader = BinaryReader(kv.key.removePrefix(globalConfigHistoryPrefix), AssumeVersion(protocolVersion)); + Version historyCommitVersion; + versionReader >> historyCommitVersion; + historyCommitVersion = bigEndian64(historyCommitVersion); + data.first = historyCommitVersion; + + // Read the list of mutations that occurred at this version. + BinaryReader mutationReader = BinaryReader(kv.value, AssumeVersion(protocolVersion)); + VectorRef mutations; + mutationReader >> mutations; + data.second = VectorRef(arena, mutations); + clientInfo.history.push_back(data); } From c9b0d3dd4e203853c8a78904671ae38d98c5b394 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 19 Feb 2021 14:00:07 -0800 Subject: [PATCH 29/60] Fix memory leak The map containing global configuration data had keys of type StringRef, referencing data allocated in history arenas. When the old history was deleted, this memory was no longer valid and some keys would point to garbage memory. --- fdbclient/GlobalConfig.actor.cpp | 37 +++++++++++++++++++++++++---- fdbclient/GlobalConfig.actor.h | 24 +++++++------------ fdbclient/SpecialKeySpace.actor.cpp | 31 +++++++++++++----------- 3 files changed, 58 insertions(+), 34 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index 8997d11973..04a72bb443 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -37,7 +37,7 @@ GlobalConfig& GlobalConfig::globalConfig() { return *reinterpret_cast(res); } -const std::any GlobalConfig::get(StringRef name) { +const std::any GlobalConfig::get(KeyRef name) { auto it = data.find(name); if (it == data.end()) { return std::any{}; @@ -45,21 +45,48 @@ const std::any GlobalConfig::get(StringRef name) { return it->second; } +const std::map GlobalConfig::get(KeyRangeRef range) { + std::map results; + for (const auto& [key, value] : data) { + if (range.contains(key)) { + results[key] = value; + } + } + return results; +} + Future GlobalConfig::onInitialized() { return initialized.getFuture(); } void GlobalConfig::insert(KeyRef key, ValueRef value) { + KeyRef stableKey = KeyRef(arena, key); Tuple t = Tuple::unpack(value); if (t.getType(0) == Tuple::ElementType::UTF8) { - data[key] = t.getString(0); + data[stableKey] = t.getString(0); } else if (t.getType(0) == Tuple::ElementType::INT) { - data[key] = t.getInt(0); + data[stableKey] = t.getInt(0); } else if (t.getType(0) == Tuple::ElementType::FLOAT) { - data[key] = t.getFloat(0); + data[stableKey] = t.getFloat(0); } else if (t.getType(0) == Tuple::ElementType::DOUBLE) { - data[key] = t.getDouble(0); + data[stableKey] = t.getDouble(0); } else { ASSERT(false); } } + +void GlobalConfig::erase(KeyRef key) { + erase(KeyRangeRef(key, keyAfter(key))); +} + +void GlobalConfig::erase(KeyRangeRef range) { + // TODO: Memory leak -- memory for key remains allocated in arena + auto it = data.begin(); + while (it != data.end()) { + if (range.contains(it->first)) { + it = data.erase(it); + } else { + ++it; + } + } +} diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index 323a5e953c..8d0bd21a17 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -27,6 +27,7 @@ #define FDBCLIENT_GLOBALCONFIG_ACTOR_H #include +#include #include #include "fdbclient/CommitProxyInterface.h" @@ -47,11 +48,14 @@ public: static void create(DatabaseContext* cx, Reference> dbInfo); static GlobalConfig& globalConfig(); - const std::any get(StringRef name); + const std::any get(KeyRef name); + const std::map get(KeyRangeRef range); Future onInitialized(); private: void insert(KeyRef key, ValueRef value); + void erase(KeyRef key); + void erase(KeyRangeRef range); ACTOR static Future refresh(GlobalConfig* self) { Transaction tr(self->cx); @@ -83,28 +87,19 @@ private: // version. Mutation history should already be stored in // ascending version order. for (int i = 0; i < history.size(); ++i) { - std::pair> pair = history[i].contents(); + const std::pair>& pair = history[i].contents(); Version version = pair.first; if (version <= self->lastUpdate) { continue; // already applied this mutation } - VectorRef& mutations = pair.second; + const VectorRef& mutations = pair.second; for (const auto& mutation : mutations) { if (mutation.type == MutationRef::SetValue) { self->insert(mutation.param1, mutation.param2); } else if (mutation.type == MutationRef::ClearRange) { - // TODO: Could be optimized if using std::map.. - KeyRangeRef range(mutation.param1, mutation.param2); - auto it = self->data.begin(); - while (it != self->data.end()) { - if (range.contains(it->first)) { - it = self->data.erase(it); - } else { - ++it; - } - } + self->erase(KeyRangeRef(mutation.param1, mutation.param2)); } else { ASSERT(false); } @@ -123,8 +118,7 @@ private: Database cx; Future _updater; Promise initialized; - // TODO: Arena to store all data in - // TODO: Change to std::map for faster range access + Arena arena; std::unordered_map data; Version lastUpdate; }; diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index d2d553f0ca..ea577777e5 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1377,23 +1377,26 @@ Future> GlobalConfigImpl::getRange(ReadYourWritesTran KeyRangeRef kr) const { Standalone result; - // if (kr.begin != kr.end) { - // ryw->setSpecialKeySpaceErrorMsg("get range disabled, please fetch a single key"); - // throw special_keys_api_failure(); - // } - auto& globalConfig = GlobalConfig::globalConfig(); - KeyRef key = kr.begin.removePrefix(getKeyRange().begin); - const std::any& any = globalConfig.get(key); - if (any.has_value()) { - if (any.type() == typeid(Standalone)) { - result.push_back_deep(result.arena(), KeyValueRef(kr.begin, std::any_cast>(globalConfig.get(key)).contents())); - } else if (any.type() == typeid(int64_t)) { - result.push_back_deep(result.arena(), KeyValueRef(kr.begin, std::to_string(std::any_cast(globalConfig.get(key))))); - } else { - ASSERT(false); + KeyRangeRef modified = KeyRangeRef(kr.begin.removePrefix(getKeyRange().begin), kr.end.removePrefix(getKeyRange().begin)); + std::map values = globalConfig.get(modified); + for (const auto& [key, any] : values) { + Key prefixedKey = key.withPrefix(getKeyRange().begin); + if (any.has_value()) { + if (any.type() == typeid(Standalone)) { + result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::any_cast>(any).contents())); + } else if (any.type() == typeid(int64_t)) { + result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(any)))); + } else if (any.type() == typeid(float)) { + result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(any)))); + } else if (any.type() == typeid(double)) { + result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(any)))); + } else { + ASSERT(false); + } } } + return result; } From 96732810ffd4ba4dfdbf1c2830b76c02fbd86938 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 19 Feb 2021 14:22:58 -0800 Subject: [PATCH 30/60] Move actor implementation --- fdbclient/GlobalConfig.actor.cpp | 60 ++++++++++++++++++++++++++++++- fdbclient/GlobalConfig.actor.h | 61 +++----------------------------- 2 files changed, 63 insertions(+), 58 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index 04a72bb443..476c291167 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -25,7 +25,7 @@ GlobalConfig::GlobalConfig() : lastUpdate(0) {} void GlobalConfig::create(DatabaseContext* cx, Reference> dbInfo) { - auto config = new GlobalConfig{}; // TODO: memory leak? + auto config = new GlobalConfig{}; config->cx = Database(cx); g_network->setGlobal(INetwork::enGlobalConfig, config); config->_updater = updater(config, dbInfo); @@ -90,3 +90,61 @@ void GlobalConfig::erase(KeyRangeRef range) { } } } + +ACTOR Future GlobalConfig::refresh(GlobalConfig* self) { + Transaction tr(self->cx); + Standalone result = wait(tr.getRange(globalConfigDataKeys, CLIENT_KNOBS->TOO_MANY)); + for (const auto& kv : result) { + KeyRef systemKey = kv.key.removePrefix(globalConfigDataPrefix); + self->insert(systemKey, kv.value); + } + return Void(); +} + +ACTOR Future GlobalConfig::updater(GlobalConfig* self, Reference> dbInfo) { + wait(self->refresh(self)); + self->initialized.send(Void()); + + loop { + try { + wait(dbInfo->onChange()); + + auto& history = dbInfo->get().history; + if (history.size() == 0 || (self->lastUpdate < history[0].first && self->lastUpdate != 0)) { + // This process missed too many global configuration + // history updates or the protocol version changed, so it + // must re-read the entire configuration range. + wait(self->refresh(self)); + self->lastUpdate = dbInfo->get().history.back().contents().first; + } else { + // Apply history in order, from lowest version to highest + // version. Mutation history should already be stored in + // ascending version order. + for (int i = 0; i < history.size(); ++i) { + const std::pair>& pair = history[i].contents(); + + Version version = pair.first; + if (version <= self->lastUpdate) { + continue; // already applied this mutation + } + + const VectorRef& mutations = pair.second; + for (const auto& mutation : mutations) { + if (mutation.type == MutationRef::SetValue) { + self->insert(mutation.param1, mutation.param2); + } else if (mutation.type == MutationRef::ClearRange) { + self->erase(KeyRangeRef(mutation.param1, mutation.param2)); + } else { + ASSERT(false); + } + } + + ASSERT(version > self->lastUpdate); + self->lastUpdate = version; + } + } + } catch (Error& e) { + throw; + } + } +} diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index 8d0bd21a17..ec43ff5a97 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -48,8 +48,10 @@ public: static void create(DatabaseContext* cx, Reference> dbInfo); static GlobalConfig& globalConfig(); + const std::any get(KeyRef name); const std::map get(KeyRangeRef range); + Future onInitialized(); private: @@ -57,63 +59,8 @@ private: void erase(KeyRef key); void erase(KeyRangeRef range); - ACTOR static Future refresh(GlobalConfig* self) { - Transaction tr(self->cx); - Standalone result = wait(tr.getRange(globalConfigDataKeys, CLIENT_KNOBS->TOO_MANY)); - for (const auto& kv : result) { - KeyRef systemKey = kv.key.removePrefix(globalConfigDataPrefix); - self->insert(systemKey, kv.value); - } - return Void(); - } - - ACTOR static Future updater(GlobalConfig* self, Reference> dbInfo) { - wait(refresh(self)); - self->initialized.send(Void()); - - loop { - try { - wait(dbInfo->onChange()); - - auto& history = dbInfo->get().history; - if (history.size() == 0 || (self->lastUpdate < history[0].first && self->lastUpdate != 0)) { - // This process missed too many global configuration - // history updates or the protocol version changed, so it - // must re-read the entire configuration range. - wait(refresh(self)); - self->lastUpdate = dbInfo->get().history.back().contents().first; - } else { - // Apply history in order, from lowest version to highest - // version. Mutation history should already be stored in - // ascending version order. - for (int i = 0; i < history.size(); ++i) { - const std::pair>& pair = history[i].contents(); - - Version version = pair.first; - if (version <= self->lastUpdate) { - continue; // already applied this mutation - } - - const VectorRef& mutations = pair.second; - for (const auto& mutation : mutations) { - if (mutation.type == MutationRef::SetValue) { - self->insert(mutation.param1, mutation.param2); - } else if (mutation.type == MutationRef::ClearRange) { - self->erase(KeyRangeRef(mutation.param1, mutation.param2)); - } else { - ASSERT(false); - } - } - - ASSERT(version > self->lastUpdate); - self->lastUpdate = version; - } - } - } catch (Error& e) { - throw; - } - } - } + ACTOR static Future refresh(GlobalConfig* self); + ACTOR static Future updater(GlobalConfig* self, Reference> dbInfo); Database cx; Future _updater; From 2acefa2c821071bf0337be99ab195902c24da9fa Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 19 Feb 2021 16:46:08 -0800 Subject: [PATCH 31/60] Add double and float support to tuples Note that this functionality is copied from bindings/flow/Tuple.cpp. These classes should eventually be combined (see #4351). --- fdbclient/GlobalConfig.actor.cpp | 30 ++++++++++++++--------- fdbclient/GlobalConfig.actor.h | 13 ++++++---- fdbclient/Tuple.cpp | 41 ++++++++++++++++++++++++++++++-- fdbclient/Tuple.h | 4 ++++ 4 files changed, 70 insertions(+), 18 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index 476c291167..319c143b5a 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -19,6 +19,10 @@ */ #include "fdbclient/GlobalConfig.actor.h" +#include "fdbclient/SystemData.h" +#include "fdbclient/Tuple.h" +#include "flow/flow.h" +#include "flow/genericactors.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -61,17 +65,21 @@ Future GlobalConfig::onInitialized() { void GlobalConfig::insert(KeyRef key, ValueRef value) { KeyRef stableKey = KeyRef(arena, key); - Tuple t = Tuple::unpack(value); - if (t.getType(0) == Tuple::ElementType::UTF8) { - data[stableKey] = t.getString(0); - } else if (t.getType(0) == Tuple::ElementType::INT) { - data[stableKey] = t.getInt(0); - } else if (t.getType(0) == Tuple::ElementType::FLOAT) { - data[stableKey] = t.getFloat(0); - } else if (t.getType(0) == Tuple::ElementType::DOUBLE) { - data[stableKey] = t.getDouble(0); - } else { - ASSERT(false); + try { + Tuple t = Tuple::unpack(value); + if (t.getType(0) == Tuple::ElementType::UTF8) { + data[stableKey] = t.getString(0); + } else if (t.getType(0) == Tuple::ElementType::INT) { + data[stableKey] = t.getInt(0); + } else if (t.getType(0) == Tuple::ElementType::FLOAT) { + data[stableKey] = t.getFloat(0); + } else if (t.getType(0) == Tuple::ElementType::DOUBLE) { + data[stableKey] = t.getDouble(0); + } else { + ASSERT(false); + } + } catch (Error& e) { + TraceEvent("GlobalConfigTupleError").detail("What", e.what()); } } diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index ec43ff5a97..b472ea4718 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -32,14 +32,13 @@ #include "fdbclient/CommitProxyInterface.h" #include "fdbclient/ReadYourWrites.h" -#include "fdbclient/SystemData.h" -#include "fdbclient/Tuple.h" -#include "flow/flow.h" -#include "flow/genericactors.actor.h" -#include "flow/Knobs.h" #include "flow/actorcompiler.h" // has to be last include +// The global configuration is a series of typed key-value pairs synced to all +// nodes (server and client) in an FDB cluster in an eventually consistent +// manner. + class GlobalConfig { public: GlobalConfig(); @@ -52,6 +51,10 @@ public: const std::any get(KeyRef name); const std::map get(KeyRangeRef range); + // To write into the global configuration, submit a transaction to + // \xff\xff/global_config/ with encoded using the + // FDB tuple typecodes. + Future onInitialized(); private: diff --git a/fdbclient/Tuple.cpp b/fdbclient/Tuple.cpp index 535be3d7fc..96f806c791 100644 --- a/fdbclient/Tuple.cpp +++ b/fdbclient/Tuple.cpp @@ -65,7 +65,14 @@ Tuple::Tuple(StringRef const& str, bool exclude_incomplete) { i = find_string_terminator(str, i + 1) + 1; } else if (data[i] >= '\x0c' && data[i] <= '\x1c') { i += abs(data[i] - '\x14') + 1; - } else if (data[i] == '\x00') { + } + else if(data[i] == 0x20) { + i += sizeof(float) + 1; + } + else if(data[i] == 0x21) { + i += sizeof(double) + 1; + } + else if(data[i] == '\x00') { i += 1; } else { throw invalid_tuple_data_type(); @@ -138,6 +145,29 @@ Tuple& Tuple::append(int64_t value) { return *this; } +Tuple& Tuple::appendFloat( float value ) { + offsets.push_back( data.size() ); + float swap = bigEndianFloat(value); + uint8_t *bytes = (uint8_t*)&swap; + adjust_floating_point(bytes, sizeof(float), true); + + data.push_back( data.arena(), 0x20 ); + data.append( data.arena(), bytes, sizeof(float) ); + return *this; +} + +Tuple& Tuple::appendDouble( double value ) { + offsets.push_back( data.size() ); + double swap = value; + swap = bigEndianDouble(swap); + uint8_t *bytes = (uint8_t*)&swap; + adjust_floating_point(bytes, sizeof(double), true); + + data.push_back( data.arena(), 0x21 ); + data.append( data.arena(), bytes, sizeof(double) ); + return *this; +} + Tuple& Tuple::appendNull() { offsets.push_back(data.size()); data.push_back(data.arena(), (uint8_t)'\x00'); @@ -159,7 +189,14 @@ Tuple::ElementType Tuple::getType(size_t index) const { return ElementType::UTF8; } else if (code >= '\x0c' && code <= '\x1c') { return ElementType::INT; - } else { + } + else if(code == 0x20) { + return ElementType::FLOAT; + } + else if(code == 0x21) { + return ElementType::DOUBLE; + } + else { throw invalid_tuple_data_type(); } } diff --git a/fdbclient/Tuple.h b/fdbclient/Tuple.h index 4497f19441..3dc597f262 100644 --- a/fdbclient/Tuple.h +++ b/fdbclient/Tuple.h @@ -38,6 +38,10 @@ struct Tuple { Tuple& append(Tuple const& tuple); Tuple& append(StringRef const& str, bool utf8 = false); Tuple& append(int64_t); + // There are some ambiguous append calls in fdbclient, so to make it easier + // to add append for floats and doubles, name them differently for now. + Tuple& appendFloat(float); + Tuple& appendDouble(double); Tuple& appendNull(); StringRef pack() const { return StringRef(data.begin(), data.size()); } From 9587318696c54a932a53d31a7ccdb91cf35b2401 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 19 Feb 2021 18:00:04 -0800 Subject: [PATCH 32/60] Fix crash when history size is 0 This shouldn't happen in normal operation (if ClientDBInfo has been updated, that means at least one item should have been added to the history). But there is old functionality that uses other ClientDBInfo fields to send updates to all nodes, and until this functionality is removed this check needs to be here. --- fdbclient/GlobalConfig.actor.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index 319c143b5a..b411e06f56 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -123,7 +123,13 @@ ACTOR Future GlobalConfig::updater(GlobalConfig* self, Referencerefresh(self)); - self->lastUpdate = dbInfo->get().history.back().contents().first; + // TODO: This check is a temporary fix while old functionality + // for setting ClientDBInfo fields exist, but eventually it + // should be replaced with an assert that the size of `history` + // is greater than 0. + if (dbInfo->get().history.size() > 0) { + self->lastUpdate = dbInfo->get().history.back().contents().first; + } } else { // Apply history in order, from lowest version to highest // version. Mutation history should already be stored in From 80c6048a01abb641fbfa54b9b33bf83486ffcbf6 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Sat, 20 Feb 2021 00:43:54 -0800 Subject: [PATCH 33/60] Naming fixes --- fdbclient/GlobalConfig.actor.cpp | 2 +- fdbclient/SpecialKeySpace.actor.cpp | 4 +++- fdbclient/SystemData.cpp | 6 +++--- fdbclient/SystemData.h | 6 +++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index b411e06f56..c51429a694 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -103,7 +103,7 @@ ACTOR Future GlobalConfig::refresh(GlobalConfig* self) { Transaction tr(self->cx); Standalone result = wait(tr.getRange(globalConfigDataKeys, CLIENT_KNOBS->TOO_MANY)); for (const auto& kv : result) { - KeyRef systemKey = kv.key.removePrefix(globalConfigDataPrefix); + KeyRef systemKey = kv.key.removePrefix(globalConfigKeysPrefix); self->insert(systemKey, kv.value); } return Void(); diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index ea577777e5..c84b4a5d42 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1449,7 +1449,7 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl state RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); while (iter != ranges.end()) { Key bareKey = iter->begin().removePrefix(globalConfig->getKeyRange().begin); - Key systemKey = bareKey.withPrefix(globalConfigDataPrefix); + Key systemKey = bareKey.withPrefix(globalConfigKeysPrefix); std::pair> entry = iter->value(); if (entry.first) { if (entry.second.present()) { @@ -1498,6 +1498,8 @@ void GlobalConfigImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); } +TracingOptionsImpl::TracingOptionsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} + Future> TracingOptionsImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { Standalone result; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index aaf115e3b8..785a42828b 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -632,11 +632,11 @@ std::string encodeFailedServersKey(AddressExclusion const& addr) { return failedServersPrefix.toString() + addr.toString(); } -const KeyRangeRef globalConfigKeys( LiteralStringRef("\xff/globalConfig/"), LiteralStringRef("\xff/globalConfig0") ); -const KeyRef globalConfigPrefix = globalConfigKeys.begin; +// const KeyRangeRef globalConfigKeys( LiteralStringRef("\xff/globalConfig/"), LiteralStringRef("\xff/globalConfig0") ); +// const KeyRef globalConfigPrefix = globalConfigKeys.begin; const KeyRangeRef globalConfigDataKeys( LiteralStringRef("\xff/globalConfig/k/"), LiteralStringRef("\xff/globalConfig/k0") ); -const KeyRef globalConfigDataPrefix = globalConfigDataKeys.begin; +const KeyRef globalConfigKeysPrefix = globalConfigDataKeys.begin; const KeyRangeRef globalConfigHistoryKeys( LiteralStringRef("\xff/globalConfig/h/"), LiteralStringRef("\xff/globalConfig/h0") ); const KeyRef globalConfigHistoryPrefix = globalConfigHistoryKeys.begin; diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 15117a867e..489da42f83 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -232,14 +232,14 @@ std::string encodeFailedServersKey(AddressExclusion const&); // "\xff/globalConfig/[[option]]" := "value" // An umbrella prefix for global configuration data synchronized to all nodes. -extern const KeyRangeRef globalConfigData; -extern const KeyRef globalConfigDataPrefix; +// extern const KeyRangeRef globalConfigData; +// extern const KeyRef globalConfigDataPrefix; // "\xff/globalConfig/k/[[key]]" := "value" // Key-value pairs that have been set. The range this keyspace represents // contains all globally configured options. extern const KeyRangeRef globalConfigDataKeys; -extern const KeyRef globalConfigDataPrefix; +extern const KeyRef globalConfigKeysPrefix; // "\xff/globalConfig/h/[[version]]" := "value" // Maps a commit version to a list of mutations made to the global From c3f68831af6ee2e59c0cee6ced085396775c120b Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Tue, 23 Feb 2021 16:17:05 -0800 Subject: [PATCH 34/60] Move existing ClientDBInfo variables to global configuration --- fdbcli/fdbcli.actor.cpp | 32 +++++++--------- fdbclient/CommitProxyInterface.h | 14 ++----- fdbclient/GlobalConfig.actor.cpp | 6 +++ fdbclient/GlobalConfig.actor.h | 27 +++++++++++++ fdbclient/NativeAPI.actor.cpp | 38 +++++++++---------- fdbclient/SystemData.cpp | 6 +-- fdbclient/SystemData.h | 2 - fdbserver/ClusterController.actor.cpp | 22 +---------- ...entTransactionProfileCorrectness.actor.cpp | 19 ++++++---- 9 files changed, 82 insertions(+), 84 deletions(-) diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index e608e96086..35f7fdd884 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -24,6 +24,7 @@ #include "fdbclient/Status.h" #include "fdbclient/StatusClient.h" #include "fdbclient/DatabaseContext.h" +#include "fdbclient/GlobalConfig.actor.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" #include "fdbclient/ClusterInterface.h" @@ -3841,25 +3842,14 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { is_error = true; continue; } - state Future>> sampleRateFuture = - tr->get(fdbClientInfoTxnSampleRate); - state Future>> sizeLimitFuture = - tr->get(fdbClientInfoTxnSizeLimit); - wait(makeInterruptable(success(sampleRateFuture) && success(sizeLimitFuture))); + const double sampleRateDbl = GlobalConfig::globalConfig().get(fdbClientInfoTxnSampleRate, std::numeric_limits::infinity()); + const int64_t sizeLimit = GlobalConfig::globalConfig().get(fdbClientInfoTxnSizeLimit, -1); std::string sampleRateStr = "default", sizeLimitStr = "default"; - if (sampleRateFuture.get().present()) { - const double sampleRateDbl = - BinaryReader::fromStringRef(sampleRateFuture.get().get(), Unversioned()); - if (!std::isinf(sampleRateDbl)) { - sampleRateStr = boost::lexical_cast(sampleRateDbl); - } + if (!std::isinf(sampleRateDbl)) { + sampleRateStr = boost::lexical_cast(sampleRateDbl); } - if (sizeLimitFuture.get().present()) { - const int64_t sizeLimit = - BinaryReader::fromStringRef(sizeLimitFuture.get().get(), Unversioned()); - if (sizeLimit != -1) { - sizeLimitStr = boost::lexical_cast(sizeLimit); - } + if (sizeLimit != -1) { + sizeLimitStr = boost::lexical_cast(sizeLimit); } printf("Client profiling rate is set to %s and size limit is set to %s.\n", sampleRateStr.c_str(), @@ -3897,8 +3887,12 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { continue; } } - tr->set(fdbClientInfoTxnSampleRate, BinaryWriter::toValue(sampleRate, Unversioned())); - tr->set(fdbClientInfoTxnSizeLimit, BinaryWriter::toValue(sizeLimit, Unversioned())); + + Tuple rate = Tuple().appendDouble(sampleRate); + Tuple size = Tuple().append(sizeLimit); + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + tr->set(fdbClientInfoTxnSampleRate.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG).begin), rate.pack()); + tr->set(fdbClientInfoTxnSizeLimit.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG).begin), size.pack()); if (!intrans) { wait(commitTransaction(tr)); } diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h index f29d7369b3..4fffa116ac 100644 --- a/fdbclient/CommitProxyInterface.h +++ b/fdbclient/CommitProxyInterface.h @@ -113,17 +113,10 @@ struct ClientDBInfo { vector commitProxies; Optional firstCommitProxy; // not serialized, used for commitOnFirstProxy when the commit proxies vector has been shrunk - vector>>> history; - double clientTxnInfoSampleRate; - int64_t clientTxnInfoSizeLimit; Optional forward; - double transactionTagSampleRate; - double transactionTagSampleCost; + vector>>> history; - ClientDBInfo() - : clientTxnInfoSampleRate(std::numeric_limits::infinity()), clientTxnInfoSizeLimit(-1), - transactionTagSampleRate(CLIENT_KNOBS->READ_TAG_SAMPLE_RATE), - transactionTagSampleCost(CLIENT_KNOBS->COMMIT_SAMPLE_COST) {} + ClientDBInfo() {} bool operator==(ClientDBInfo const& r) const { return id == r.id; } bool operator!=(ClientDBInfo const& r) const { return id != r.id; } @@ -133,8 +126,7 @@ struct ClientDBInfo { if constexpr (!is_fb_function) { ASSERT(ar.protocolVersion().isValid()); } - serializer(ar, grvProxies, commitProxies, id, history, clientTxnInfoSampleRate, clientTxnInfoSizeLimit, - forward, transactionTagSampleRate, transactionTagSampleCost); + serializer(ar, grvProxies, commitProxies, id, forward, history); } }; diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index c51429a694..261680f4d4 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -26,6 +26,12 @@ #include "flow/actorcompiler.h" // This must be the last #include. +const KeyRef fdbClientInfoTxnSampleRate = LiteralStringRef("fdbClientInfo/client_txn_sample_rate"); +const KeyRef fdbClientInfoTxnSizeLimit = LiteralStringRef("fdbClientInfo/client_txn_size_limit"); + +const KeyRef transactionTagSampleRate = LiteralStringRef("transactionTagSampleRate"); +const KeyRef transactionTagSampleCost = LiteralStringRef("transactionTagSampleCost"); + GlobalConfig::GlobalConfig() : lastUpdate(0) {} void GlobalConfig::create(DatabaseContext* cx, Reference> dbInfo) { diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index b472ea4718..601802e57c 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -39,6 +39,13 @@ // nodes (server and client) in an FDB cluster in an eventually consistent // manner. +// Keys +extern const KeyRef fdbClientInfoTxnSampleRate; +extern const KeyRef fdbClientInfoTxnSizeLimit; + +extern const KeyRef transactionTagSampleRate; +extern const KeyRef transactionTagSampleCost; + class GlobalConfig { public: GlobalConfig(); @@ -51,6 +58,26 @@ public: const std::any get(KeyRef name); const std::map get(KeyRangeRef range); + template + const T get(KeyRef name) { + try { + auto any = get(name); + return std::any_cast(any); + } catch (Error& e) { + throw; + } + } + + template + const T get(KeyRef name, T defaultVal) { + auto any = get(name); + if (any.has_value()) { + return std::any_cast(any); + } + + return defaultVal; + } + // To write into the global configuration, submit a transaction to // \xff\xff/global_config/ with encoded using the // FDB tuple typecodes. diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index d350a39974..f45040982f 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -506,12 +506,11 @@ ACTOR static Future clientStatusUpdateActor(DatabaseContext* cx) { } } cx->clientStatusUpdater.outStatusQ.clear(); - double clientSamplingProbability = std::isinf(cx->clientInfo->get().clientTxnInfoSampleRate) - ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY - : cx->clientInfo->get().clientTxnInfoSampleRate; - int64_t clientTxnInfoSizeLimit = cx->clientInfo->get().clientTxnInfoSizeLimit == -1 - ? CLIENT_KNOBS->CSI_SIZE_LIMIT - : cx->clientInfo->get().clientTxnInfoSizeLimit; + wait(GlobalConfig::globalConfig().onInitialized()); + double sampleRate = GlobalConfig::globalConfig().get(fdbClientInfoTxnSampleRate, std::numeric_limits::infinity()); + double clientSamplingProbability = std::isinf(sampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : sampleRate; + double sizeLimit = GlobalConfig::globalConfig().get(fdbClientInfoTxnSizeLimit, -1); + int64_t clientTxnInfoSizeLimit = sizeLimit == -1 ? CLIENT_KNOBS->CSI_SIZE_LIMIT : sizeLimit; if (!trChunksQ.empty() && deterministicRandom()->random01() < clientSamplingProbability) wait(delExcessClntTxnEntriesActor(&tr, clientTxnInfoSizeLimit)); @@ -957,14 +956,14 @@ DatabaseContext::DatabaseContext(ReferenceINIT_MID_SHARD_BYTES); - GlobalConfig::create(this, clientInfo); - if (apiVersionAtLeast(700)) { registerSpecialKeySpaceModule(SpecialKeySpace::MODULE::ERRORMSG, SpecialKeySpace::IMPLTYPE::READONLY, @@ -1273,14 +1272,14 @@ Future DatabaseContext::onProxiesChanged() { } bool DatabaseContext::sampleReadTags() const { - return clientInfo->get().transactionTagSampleRate > 0 && - deterministicRandom()->random01() <= clientInfo->get().transactionTagSampleRate; + double sampleRate = GlobalConfig::globalConfig().get(transactionTagSampleRate, CLIENT_KNOBS->READ_TAG_SAMPLE_RATE); + return sampleRate > 0 && deterministicRandom()->random01() <= sampleRate; } bool DatabaseContext::sampleOnCost(uint64_t cost) const { - if (clientInfo->get().transactionTagSampleCost <= 0) - return false; - return deterministicRandom()->random01() <= (double)cost / clientInfo->get().transactionTagSampleCost; + double sampleCost = GlobalConfig::globalConfig().get(transactionTagSampleCost, CLIENT_KNOBS->COMMIT_SAMPLE_COST); + if (sampleCost <= 0) return false; + return deterministicRandom()->random01() <= (double)cost / sampleCost; } int64_t extractIntOption(Optional value, int64_t minValue, int64_t maxValue) { @@ -5375,14 +5374,11 @@ void Transaction::checkDeferredError() { cx->checkDeferredError(); } -Reference Transaction::createTrLogInfoProbabilistically(const Database& cx) { - if (!cx->isError()) { - double clientSamplingProbability = std::isinf(cx->clientInfo->get().clientTxnInfoSampleRate) - ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY - : cx->clientInfo->get().clientTxnInfoSampleRate; - if (((networkOptions.logClientInfo.present() && networkOptions.logClientInfo.get()) || BUGGIFY) && - deterministicRandom()->random01() < clientSamplingProbability && - (!g_network->isSimulated() || !g_simulator.speedUpSimulation)) { +Reference Transaction::createTrLogInfoProbabilistically(const Database &cx) { + if(!cx->isError()) { + double sampleRate = GlobalConfig::globalConfig().get(fdbClientInfoTxnSampleRate, std::numeric_limits::infinity()); + double clientSamplingProbability = std::isinf(sampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : sampleRate; + if (((networkOptions.logClientInfo.present() && networkOptions.logClientInfo.get()) || BUGGIFY) && deterministicRandom()->random01() < clientSamplingProbability && (!g_network->isSimulated() || !g_simulator.speedUpSimulation)) { return makeReference(TransactionLogInfo::DATABASE); } } diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 785a42828b..7c12a69059 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -757,10 +757,8 @@ const KeyRef tagThrottleLimitKey = LiteralStringRef("\xff\x02/throttledTags/manu const KeyRef tagThrottleCountKey = LiteralStringRef("\xff\x02/throttledTags/manualThrottleCount"); // Client status info prefix -const KeyRangeRef fdbClientInfoPrefixRange(LiteralStringRef("\xff\x02/fdbClientInfo/"), - LiteralStringRef("\xff\x02/fdbClientInfo0")); -const KeyRef fdbClientInfoTxnSampleRate = LiteralStringRef("\xff\x02/fdbClientInfo/client_txn_sample_rate/"); -const KeyRef fdbClientInfoTxnSizeLimit = LiteralStringRef("\xff\x02/fdbClientInfo/client_txn_size_limit/"); +const KeyRangeRef fdbClientInfoPrefixRange(LiteralStringRef("\xff\x02/fdbClientInfo/"), LiteralStringRef("\xff\x02/fdbClientInfo0")); +// See remaining fields in GlobalConfig.actor.h // ConsistencyCheck settings const KeyRef fdbShouldConsistencyCheckBeSuspended = LiteralStringRef("\xff\x02/ConsistencyCheck/Suspend"); diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 489da42f83..5cf56ef7ec 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -379,8 +379,6 @@ extern const KeyRangeRef applyMutationsKeyVersionCountRange; // FdbClient Info prefix extern const KeyRangeRef fdbClientInfoPrefixRange; -extern const KeyRef fdbClientInfoTxnSampleRate; -extern const KeyRef fdbClientInfoTxnSizeLimit; // Consistency Check settings extern const KeyRef fdbShouldConsistencyCheckBeSuspended; diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 43b53c908c..4aca282d62 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -2715,9 +2715,7 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co clientInfo.id = deterministicRandom()->randomUniqueID(); clientInfo.commitProxies = req.commitProxies; clientInfo.grvProxies = req.grvProxies; - clientInfo.clientTxnInfoSampleRate = db->clientInfo->get().clientTxnInfoSampleRate; - clientInfo.clientTxnInfoSizeLimit = db->clientInfo->get().clientTxnInfoSizeLimit; - db->clientInfo->set(clientInfo); + db->clientInfo->set( clientInfo ); dbInfo.client = db->clientInfo->get(); } @@ -3199,13 +3197,11 @@ ACTOR Future monitorClientTxnInfoConfigs(ClusterControllerData::DBInfo* db tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); state Optional globalConfigVersion = wait(tr.get(globalConfigVersionKey)); - state Optional rateVal = wait(tr.get(fdbClientInfoTxnSampleRate)); - state Optional limitVal = wait(tr.get(fdbClientInfoTxnSizeLimit)); state ClientDBInfo clientInfo = db->clientInfo->get(); if (globalConfigVersion.present()) { BinaryReader versionReader = BinaryReader(globalConfigVersion.get(), AssumeVersion(g_network->protocolVersion())); - int64_t commitVersion; + int64_t commitVersion; // Currently unused. Convert to little endian if you want to use it int16_t serializationOrder; state ProtocolVersion protocolVersion; versionReader >> commitVersion >> serializationOrder >> protocolVersion; @@ -3254,24 +3250,10 @@ ACTOR Future monitorClientTxnInfoConfigs(ClusterControllerData::DBInfo* db db->clientInfo->set(clientInfo); } - // TODO: Remove this and move to global config space - double sampleRate = rateVal.present() ? BinaryReader::fromStringRef(rateVal.get(), Unversioned()) : std::numeric_limits::infinity(); - int64_t sizeLimit = limitVal.present() ? BinaryReader::fromStringRef(limitVal.get(), Unversioned()) : -1; - if (sampleRate != clientInfo.clientTxnInfoSampleRate || sizeLimit != clientInfo.clientTxnInfoSampleRate) { - clientInfo.id = deterministicRandom()->randomUniqueID(); - clientInfo.clientTxnInfoSampleRate = sampleRate; - clientInfo.clientTxnInfoSizeLimit = sizeLimit; - db->clientInfo->set(clientInfo); - } - state Future globalConfigFuture = tr.watch(globalConfigVersionKey); - state Future watchRateFuture = tr.watch(fdbClientInfoTxnSampleRate); - state Future watchLimitFuture = tr.watch(fdbClientInfoTxnSizeLimit); wait(tr.commit()); choose { when (wait(globalConfigFuture)) { break; } - when(wait(watchRateFuture)) { break; } - when(wait(watchLimitFuture)) { break; } } } catch (Error& e) { wait(tr.onError(e)); diff --git a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp b/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp index a5d6ca18be..9343d24694 100644 --- a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp +++ b/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp @@ -1,5 +1,6 @@ #include "fdbserver/workloads/workloads.actor.h" #include "fdbserver/ServerDBInfo.h" +#include "fdbclient/GlobalConfig.actor.h" #include "fdbclient/ManagementAPI.actor.h" #include "fdbclient/RunTransaction.actor.h" #include "flow/actorcompiler.h" // has to be last include @@ -268,13 +269,17 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload { ACTOR Future changeProfilingParameters(Database cx, int64_t sizeLimit, double sampleProbability) { - wait(runRYWTransaction(cx, [=](Reference tr) -> Future { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - tr->set(fdbClientInfoTxnSampleRate, BinaryWriter::toValue(sampleProbability, Unversioned())); - tr->set(fdbClientInfoTxnSizeLimit, BinaryWriter::toValue(sizeLimit, Unversioned())); - return Void(); - })); + wait(runRYWTransaction(cx, [=](Reference tr) -> Future + { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + Tuple rate = Tuple().appendDouble(sampleProbability); + Tuple size = Tuple().append(sizeLimit); + tr->set(fdbClientInfoTxnSampleRate.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG).begin), rate.pack()); + tr->set(fdbClientInfoTxnSizeLimit.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG).begin), size.pack()); + return Void(); + } + )); return Void(); } From 4a799baa1d9c720abfd18d1058fff54aa02ae14f Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Wed, 24 Feb 2021 11:23:29 -0800 Subject: [PATCH 35/60] Add clear range for global configuration --- fdbclient/GlobalConfig.actor.h | 3 ++- fdbclient/NativeAPI.actor.cpp | 2 +- fdbclient/SpecialKeySpace.actor.cpp | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index 601802e57c..3902654112 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -48,7 +48,6 @@ extern const KeyRef transactionTagSampleCost; class GlobalConfig { public: - GlobalConfig(); GlobalConfig(const GlobalConfig&) = delete; GlobalConfig& operator=(const GlobalConfig&) = delete; @@ -85,6 +84,8 @@ public: Future onInitialized(); private: + GlobalConfig(); + void insert(KeyRef key, ValueRef value); void erase(KeyRef key); void erase(KeyRangeRef range); diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index f45040982f..92d6d6711e 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -509,7 +509,7 @@ ACTOR static Future clientStatusUpdateActor(DatabaseContext* cx) { wait(GlobalConfig::globalConfig().onInitialized()); double sampleRate = GlobalConfig::globalConfig().get(fdbClientInfoTxnSampleRate, std::numeric_limits::infinity()); double clientSamplingProbability = std::isinf(sampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : sampleRate; - double sizeLimit = GlobalConfig::globalConfig().get(fdbClientInfoTxnSizeLimit, -1); + int64_t sizeLimit = GlobalConfig::globalConfig().get(fdbClientInfoTxnSizeLimit, -1); int64_t clientTxnInfoSizeLimit = sizeLimit == -1 ? CLIENT_KNOBS->CSI_SIZE_LIMIT : sizeLimit; if (!trChunksQ.empty() && deterministicRandom()->random01() < clientSamplingProbability) wait(delExcessClntTxnEntriesActor(&tr, clientTxnInfoSizeLimit)); diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index c84b4a5d42..7fbce93b6e 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1456,7 +1456,9 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl mutations.emplace_back_deep(arena, MutationRef(MutationRef::SetValue, bareKey, entry.second.get())); tr.set(systemKey, entry.second.get()); } else { - mutations.emplace_back_deep(arena, MutationRef(MutationRef::ClearRange, bareKey, keyAfter(bareKey))); + KeyRef clearRangeBegin = iter->range().begin.removePrefix(globalConfig->getKeyRange().begin); + KeyRef clearRangeEnd = iter->range().end.removePrefix(globalConfig->getKeyRange().begin); + mutations.emplace_back_deep(arena, MutationRef(MutationRef::ClearRange, clearRangeBegin, clearRangeEnd)); tr.clear(systemKey); } } @@ -1491,7 +1493,7 @@ Future> GlobalConfigImpl::commit(ReadYourWritesTransaction } void GlobalConfigImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { - // TODO + ryw->getSpecialKeySpaceWriteMap().insert(range, std::make_pair(true, Optional())); } void GlobalConfigImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { From 70c4bbe119f40bcd7d44f45ee5e05819374c3657 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Wed, 24 Feb 2021 11:49:25 -0800 Subject: [PATCH 36/60] Fix clear range persistence issue --- fdbclient/SpecialKeySpace.actor.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 7fbce93b6e..395e791839 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1448,18 +1448,22 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl ryw->getSpecialKeySpaceWriteMap().containedRanges(specialKeys); state RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); while (iter != ranges.end()) { - Key bareKey = iter->begin().removePrefix(globalConfig->getKeyRange().begin); - Key systemKey = bareKey.withPrefix(globalConfigKeysPrefix); std::pair> entry = iter->value(); if (entry.first) { if (entry.second.present()) { + Key bareKey = iter->begin().removePrefix(globalConfig->getKeyRange().begin); mutations.emplace_back_deep(arena, MutationRef(MutationRef::SetValue, bareKey, entry.second.get())); + + Key systemKey = bareKey.withPrefix(globalConfigKeysPrefix); tr.set(systemKey, entry.second.get()); } else { - KeyRef clearRangeBegin = iter->range().begin.removePrefix(globalConfig->getKeyRange().begin); - KeyRef clearRangeEnd = iter->range().end.removePrefix(globalConfig->getKeyRange().begin); - mutations.emplace_back_deep(arena, MutationRef(MutationRef::ClearRange, clearRangeBegin, clearRangeEnd)); - tr.clear(systemKey); + KeyRef bareRangeBegin = iter->range().begin.removePrefix(globalConfig->getKeyRange().begin); + KeyRef bareRangeEnd = iter->range().end.removePrefix(globalConfig->getKeyRange().begin); + mutations.emplace_back_deep(arena, MutationRef(MutationRef::ClearRange, bareRangeBegin, bareRangeEnd)); + + Key systemRangeBegin = bareRangeBegin.withPrefix(globalConfigKeysPrefix); + Key systemRangeEnd = bareRangeEnd.withPrefix(globalConfigKeysPrefix); + tr.clear(KeyRangeRef(systemRangeBegin, systemRangeEnd)); } } ++iter; From e9e2ca54d68ef7f06ab3c2d97065b2dba8a29b4c Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Wed, 24 Feb 2021 11:50:13 -0800 Subject: [PATCH 37/60] Assert history contains data --- fdbclient/GlobalConfig.actor.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index 261680f4d4..4ae6c20f83 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -129,13 +129,8 @@ ACTOR Future GlobalConfig::updater(GlobalConfig* self, Referencerefresh(self)); - // TODO: This check is a temporary fix while old functionality - // for setting ClientDBInfo fields exist, but eventually it - // should be replaced with an assert that the size of `history` - // is greater than 0. - if (dbInfo->get().history.size() > 0) { - self->lastUpdate = dbInfo->get().history.back().contents().first; - } + ASSERT(dbInfo->get().history.size() > 0); + self->lastUpdate = dbInfo->get().history.back().contents().first; } else { // Apply history in order, from lowest version to highest // version. Mutation history should already be stored in From e5e48da5ceadae455fbd52af383cbd6387d8ee2f Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Wed, 24 Feb 2021 12:59:40 -0800 Subject: [PATCH 38/60] Revert removal of history size check --- fdbclient/GlobalConfig.actor.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index 4ae6c20f83..52df7c30e5 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -129,8 +129,9 @@ ACTOR Future GlobalConfig::updater(GlobalConfig* self, Referencerefresh(self)); - ASSERT(dbInfo->get().history.size() > 0); - self->lastUpdate = dbInfo->get().history.back().contents().first; + if (dbInfo->get().history.size() > 0) { + self->lastUpdate = dbInfo->get().history.back().contents().first; + } } else { // Apply history in order, from lowest version to highest // version. Mutation history should already be stored in From b7cd8175be269e4c12b8e08ebbc7ab549d8d4d02 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Wed, 24 Feb 2021 18:29:53 -0800 Subject: [PATCH 39/60] Add arena per object in global config --- fdbclient/GlobalConfig.actor.cpp | 36 +++++++++++++++-------------- fdbclient/GlobalConfig.actor.h | 33 ++++++++++++++++---------- fdbclient/SpecialKeySpace.actor.cpp | 22 +++++++++--------- 3 files changed, 51 insertions(+), 40 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index 52df7c30e5..6569943ff3 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -26,19 +26,21 @@ #include "flow/actorcompiler.h" // This must be the last #include. -const KeyRef fdbClientInfoTxnSampleRate = LiteralStringRef("fdbClientInfo/client_txn_sample_rate"); -const KeyRef fdbClientInfoTxnSizeLimit = LiteralStringRef("fdbClientInfo/client_txn_size_limit"); +const KeyRef fdbClientInfoTxnSampleRate = LiteralStringRef("config/fdbClientInfo/client_txn_sample_rate"); +const KeyRef fdbClientInfoTxnSizeLimit = LiteralStringRef("config/fdbClientInfo/client_txn_size_limit"); -const KeyRef transactionTagSampleRate = LiteralStringRef("transactionTagSampleRate"); -const KeyRef transactionTagSampleCost = LiteralStringRef("transactionTagSampleCost"); +const KeyRef transactionTagSampleRate = LiteralStringRef("config/transactionTagSampleRate"); +const KeyRef transactionTagSampleCost = LiteralStringRef("config/transactionTagSampleCost"); GlobalConfig::GlobalConfig() : lastUpdate(0) {} void GlobalConfig::create(DatabaseContext* cx, Reference> dbInfo) { - auto config = new GlobalConfig{}; - config->cx = Database(cx); - g_network->setGlobal(INetwork::enGlobalConfig, config); - config->_updater = updater(config, dbInfo); + if (g_network->global(INetwork::enGlobalConfig) == nullptr) { + auto config = new GlobalConfig{}; + config->cx = Database(cx); + g_network->setGlobal(INetwork::enGlobalConfig, config); + config->_updater = updater(config, dbInfo); + } } GlobalConfig& GlobalConfig::globalConfig() { @@ -47,16 +49,16 @@ GlobalConfig& GlobalConfig::globalConfig() { return *reinterpret_cast(res); } -const std::any GlobalConfig::get(KeyRef name) { +const ConfigValue GlobalConfig::get(KeyRef name) { auto it = data.find(name); if (it == data.end()) { - return std::any{}; + return ConfigValue{ Arena(), std::any{} }; } return it->second; } -const std::map GlobalConfig::get(KeyRangeRef range) { - std::map results; +const std::map GlobalConfig::get(KeyRangeRef range) { + std::map results; for (const auto& [key, value] : data) { if (range.contains(key)) { results[key] = value; @@ -70,17 +72,18 @@ Future GlobalConfig::onInitialized() { } void GlobalConfig::insert(KeyRef key, ValueRef value) { + Arena arena(1); KeyRef stableKey = KeyRef(arena, key); try { Tuple t = Tuple::unpack(value); if (t.getType(0) == Tuple::ElementType::UTF8) { - data[stableKey] = t.getString(0); + data[stableKey] = ConfigValue{ arena, StringRef(arena, t.getString(0).contents()) }; } else if (t.getType(0) == Tuple::ElementType::INT) { - data[stableKey] = t.getInt(0); + data[stableKey] = ConfigValue{ arena, t.getInt(0) }; } else if (t.getType(0) == Tuple::ElementType::FLOAT) { - data[stableKey] = t.getFloat(0); + data[stableKey] = ConfigValue{ arena, t.getFloat(0) }; } else if (t.getType(0) == Tuple::ElementType::DOUBLE) { - data[stableKey] = t.getDouble(0); + data[stableKey] = ConfigValue{ arena, t.getDouble(0) }; } else { ASSERT(false); } @@ -94,7 +97,6 @@ void GlobalConfig::erase(KeyRef key) { } void GlobalConfig::erase(KeyRangeRef range) { - // TODO: Memory leak -- memory for key remains allocated in arena auto it = data.begin(); while (it != data.end()) { if (range.contains(it->first)) { diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index 3902654112..a82d86cc8a 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -28,6 +28,7 @@ #include #include +#include #include #include "fdbclient/CommitProxyInterface.h" @@ -46,6 +47,11 @@ extern const KeyRef fdbClientInfoTxnSizeLimit; extern const KeyRef transactionTagSampleRate; extern const KeyRef transactionTagSampleCost; +struct ConfigValue { + Arena arena; + std::any value; +}; + class GlobalConfig { public: GlobalConfig(const GlobalConfig&) = delete; @@ -54,27 +60,31 @@ public: static void create(DatabaseContext* cx, Reference> dbInfo); static GlobalConfig& globalConfig(); - const std::any get(KeyRef name); - const std::map get(KeyRangeRef range); + const ConfigValue get(KeyRef name); + const std::map get(KeyRangeRef range); - template + template {}, bool>::type = true> const T get(KeyRef name) { try { - auto any = get(name); + auto any = get(name).value; return std::any_cast(any); } catch (Error& e) { throw; } } - template + template {}, bool>::type = true> const T get(KeyRef name, T defaultVal) { - auto any = get(name); - if (any.has_value()) { - return std::any_cast(any); - } + try { + auto any = get(name).value; + if (any.has_value()) { + return std::any_cast(any); + } - return defaultVal; + return defaultVal; + } catch (Error& e) { + throw; + } } // To write into the global configuration, submit a transaction to @@ -96,8 +106,7 @@ private: Database cx; Future _updater; Promise initialized; - Arena arena; - std::unordered_map data; + std::unordered_map data; Version lastUpdate; }; diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 395e791839..4c4a09456a 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1379,18 +1379,18 @@ Future> GlobalConfigImpl::getRange(ReadYourWritesTran auto& globalConfig = GlobalConfig::globalConfig(); KeyRangeRef modified = KeyRangeRef(kr.begin.removePrefix(getKeyRange().begin), kr.end.removePrefix(getKeyRange().begin)); - std::map values = globalConfig.get(modified); - for (const auto& [key, any] : values) { + std::map values = globalConfig.get(modified); + for (const auto& [key, config] : values) { Key prefixedKey = key.withPrefix(getKeyRange().begin); - if (any.has_value()) { - if (any.type() == typeid(Standalone)) { - result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::any_cast>(any).contents())); - } else if (any.type() == typeid(int64_t)) { - result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(any)))); - } else if (any.type() == typeid(float)) { - result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(any)))); - } else if (any.type() == typeid(double)) { - result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(any)))); + if (config.value.has_value()) { + if (config.value.type() == typeid(StringRef)) { + result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::any_cast(config.value).toString())); + } else if (config.value.type() == typeid(int64_t)) { + result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); + } else if (config.value.type() == typeid(float)) { + result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); + } else if (config.value.type() == typeid(double)) { + result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); } else { ASSERT(false); } From 388344c31e9c1f7bbb56470e78ad5b9f369f9a54 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 26 Feb 2021 09:27:55 -0800 Subject: [PATCH 40/60] Better estimation for arena size --- fdbclient/GlobalConfig.actor.cpp | 2 +- fdbclient/GlobalConfig.actor.h | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index 6569943ff3..cbb92d053e 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -72,7 +72,7 @@ Future GlobalConfig::onInitialized() { } void GlobalConfig::insert(KeyRef key, ValueRef value) { - Arena arena(1); + Arena arena(key.expectedSize() + value.expectedSize()); KeyRef stableKey = KeyRef(arena, key); try { Tuple t = Tuple::unpack(value); diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index a82d86cc8a..479e65d427 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -60,19 +60,17 @@ public: static void create(DatabaseContext* cx, Reference> dbInfo); static GlobalConfig& globalConfig(); + // Get a value from the framework. Values are returned in a ConfigValue + // struct which also contains a reference to the arena containing the + // memory for the object. As long as the caller keeps a reference to the + // returned ConfigValue, the value is guaranteed to be readable (if it + // exists). const ConfigValue get(KeyRef name); const std::map get(KeyRangeRef range); - template {}, bool>::type = true> - const T get(KeyRef name) { - try { - auto any = get(name).value; - return std::any_cast(any); - } catch (Error& e) { - throw; - } - } - + // For arithmetic value types, returns a copy of the value for the given + // key, or the supplied default value if the framework does not know about + // the key. template {}, bool>::type = true> const T get(KeyRef name, T defaultVal) { try { From fb9a929780a97d57ef8768c8afe3a03855894d35 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Tue, 2 Mar 2021 15:03:05 -0800 Subject: [PATCH 41/60] Fix issue with freed memory being accessed --- fdbserver/ClusterController.actor.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 4aca282d62..28bb8c4d4d 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -3206,7 +3206,6 @@ ACTOR Future monitorClientTxnInfoConfigs(ClusterControllerData::DBInfo* db state ProtocolVersion protocolVersion; versionReader >> commitVersion >> serializationOrder >> protocolVersion; - state Arena arena; if (protocolVersion == g_network->protocolVersion()) { Standalone globalConfigHistory = wait(tr.getRange(globalConfigHistoryKeys, CLIENT_KNOBS->TOO_MANY)); // If the global configuration version key has been @@ -3228,7 +3227,7 @@ ACTOR Future monitorClientTxnInfoConfigs(ClusterControllerData::DBInfo* db BinaryReader mutationReader = BinaryReader(kv.value, AssumeVersion(protocolVersion)); VectorRef mutations; mutationReader >> mutations; - data.second = VectorRef(arena, mutations); + data.second = VectorRef(data.arena(), mutations); clientInfo.history.push_back(data); } From 1c84c04ffc25106b71288f9b15421c628b8af837 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Thu, 11 Mar 2021 10:57:46 -0800 Subject: [PATCH 42/60] Add global configuration prefix function --- fdbcli/fdbcli.actor.cpp | 4 ++-- fdbclient/GlobalConfig.actor.cpp | 5 +++++ fdbclient/GlobalConfig.actor.h | 4 ++++ fdbclient/SpecialKeySpace.actor.cpp | 1 + .../workloads/ClientTransactionProfileCorrectness.actor.cpp | 4 ++-- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 35f7fdd884..e8167c4855 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -3891,8 +3891,8 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { Tuple rate = Tuple().appendDouble(sampleRate); Tuple size = Tuple().append(sizeLimit); tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - tr->set(fdbClientInfoTxnSampleRate.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG).begin), rate.pack()); - tr->set(fdbClientInfoTxnSizeLimit.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG).begin), size.pack()); + tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack()); + tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack()); if (!intrans) { wait(commitTransaction(tr)); } diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index cbb92d053e..dd9b2d8aff 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -19,6 +19,7 @@ */ #include "fdbclient/GlobalConfig.actor.h" +#include "fdbclient/SpecialKeySpace.actor.h" #include "fdbclient/SystemData.h" #include "fdbclient/Tuple.h" #include "flow/flow.h" @@ -49,6 +50,10 @@ GlobalConfig& GlobalConfig::globalConfig() { return *reinterpret_cast(res); } +Key GlobalConfig::prefixedKey(KeyRef key) { + return key.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG).begin); +} + const ConfigValue GlobalConfig::get(KeyRef name) { auto it = data.find(name); if (it == data.end()) { diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index 479e65d427..d699c804f0 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -60,6 +60,10 @@ public: static void create(DatabaseContext* cx, Reference> dbInfo); static GlobalConfig& globalConfig(); + // Use this function to turn a global configuration key defined above into + // the full path needed to set the value in the database. + static Key prefixedKey(KeyRef key); + // Get a value from the framework. Values are returned in a ConfigValue // struct which also contains a reference to the arena containing the // memory for the object. As long as the caller keeps a reference to the diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 4c4a09456a..356cdfc2c1 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1434,6 +1434,7 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl // clear_range(\xff/globalConfig/h, \xff/globalConfig/h/1000) results // in zero key-value pairs being deleted (999 is lexicographically // larger than 1000, and the range is exclusive). + // Delete the oldest key(s) in the history to make room for new data. for (int i = 0; i < keys.size() - (kGlobalConfigMaxHistorySize - 1); ++i) { tr.clear(keys[i]); } diff --git a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp b/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp index 9343d24694..d56061ca3a 100644 --- a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp +++ b/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp @@ -275,8 +275,8 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload { tr->setOption(FDBTransactionOptions::LOCK_AWARE); Tuple rate = Tuple().appendDouble(sampleProbability); Tuple size = Tuple().append(sizeLimit); - tr->set(fdbClientInfoTxnSampleRate.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG).begin), rate.pack()); - tr->set(fdbClientInfoTxnSizeLimit.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG).begin), size.pack()); + tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack()); + tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack()); return Void(); } )); From 12603859659108bfd092e50ae0edaaa65e8934a5 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Mon, 15 Mar 2021 18:03:54 -0700 Subject: [PATCH 43/60] Use object to wrap global configuration history --- fdbclient/CommitProxyInterface.h | 3 +- fdbclient/GlobalConfig.actor.cpp | 22 ++--- fdbclient/GlobalConfig.actor.h | 8 +- fdbclient/GlobalConfig.h | 45 ++++++++++ fdbclient/SpecialKeySpace.actor.cpp | 62 ++++---------- fdbserver/ClusterController.actor.cpp | 113 ++++++++++++-------------- 6 files changed, 131 insertions(+), 122 deletions(-) create mode 100644 fdbclient/GlobalConfig.h diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h index 4fffa116ac..794b88ceaa 100644 --- a/fdbclient/CommitProxyInterface.h +++ b/fdbclient/CommitProxyInterface.h @@ -31,6 +31,7 @@ #include "fdbclient/CommitTransaction.h" #include "fdbserver/RatekeeperInterface.h" #include "fdbclient/TagThrottle.h" +#include "fdbclient/GlobalConfig.h" #include "fdbrpc/Stats.h" #include "fdbrpc/TimedRequest.h" @@ -114,7 +115,7 @@ struct ClientDBInfo { Optional firstCommitProxy; // not serialized, used for commitOnFirstProxy when the commit proxies vector has been shrunk Optional forward; - vector>>> history; + vector history; ClientDBInfo() {} diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index dd9b2d8aff..5315a8f68a 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -112,6 +112,8 @@ void GlobalConfig::erase(KeyRangeRef range) { } } +// Updates local copy of global configuration by reading the entire key-range +// from storage. ACTOR Future GlobalConfig::refresh(GlobalConfig* self) { Transaction tr(self->cx); Standalone result = wait(tr.getRange(globalConfigDataKeys, CLIENT_KNOBS->TOO_MANY)); @@ -122,6 +124,8 @@ ACTOR Future GlobalConfig::refresh(GlobalConfig* self) { return Void(); } +// Applies updates to the local copy of the global configuration when this +// process receives an updated history. ACTOR Future GlobalConfig::updater(GlobalConfig* self, Reference> dbInfo) { wait(self->refresh(self)); self->initialized.send(Void()); @@ -131,28 +135,24 @@ ACTOR Future GlobalConfig::updater(GlobalConfig* self, ReferenceonChange()); auto& history = dbInfo->get().history; - if (history.size() == 0 || (self->lastUpdate < history[0].first && self->lastUpdate != 0)) { + if (history.size() == 0 || (self->lastUpdate < history[0].version && self->lastUpdate != 0)) { // This process missed too many global configuration // history updates or the protocol version changed, so it // must re-read the entire configuration range. wait(self->refresh(self)); if (dbInfo->get().history.size() > 0) { - self->lastUpdate = dbInfo->get().history.back().contents().first; + self->lastUpdate = dbInfo->get().history.back().version; } } else { // Apply history in order, from lowest version to highest // version. Mutation history should already be stored in // ascending version order. - for (int i = 0; i < history.size(); ++i) { - const std::pair>& pair = history[i].contents(); - - Version version = pair.first; - if (version <= self->lastUpdate) { + for (const auto& vh : history) { + if (vh.version <= self->lastUpdate) { continue; // already applied this mutation } - const VectorRef& mutations = pair.second; - for (const auto& mutation : mutations) { + for (const auto& mutation : vh.mutations.contents()) { if (mutation.type == MutationRef::SetValue) { self->insert(mutation.param1, mutation.param2); } else if (mutation.type == MutationRef::ClearRange) { @@ -162,8 +162,8 @@ ACTOR Future GlobalConfig::updater(GlobalConfig* self, Reference self->lastUpdate); - self->lastUpdate = version; + ASSERT(vh.version > self->lastUpdate); + self->lastUpdate = vh.version; } } } catch (Error& e) { diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index d699c804f0..11699633da 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -32,6 +32,7 @@ #include #include "fdbclient/CommitProxyInterface.h" +#include "fdbclient/GlobalConfig.h" #include "fdbclient/ReadYourWrites.h" #include "flow/actorcompiler.h" // has to be last include @@ -62,6 +63,8 @@ public: // Use this function to turn a global configuration key defined above into // the full path needed to set the value in the database. + // + // For example, given "config/a", returns "\xff\xff/global_config/config/a". static Key prefixedKey(KeyRef key); // Get a value from the framework. Values are returned in a ConfigValue @@ -91,8 +94,11 @@ public: // To write into the global configuration, submit a transaction to // \xff\xff/global_config/ with encoded using the - // FDB tuple typecodes. + // FDB tuple typecodes. Use the helper function `prefixedKey` to correctly + // prefix your global configuration key. + // Triggers the returned future when the global configuration singleton has + // been created and is ready. Future onInitialized(); private: diff --git a/fdbclient/GlobalConfig.h b/fdbclient/GlobalConfig.h new file mode 100644 index 0000000000..14e10f8635 --- /dev/null +++ b/fdbclient/GlobalConfig.h @@ -0,0 +1,45 @@ +/* + * GlobalConfig.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "fdbclient/CommitTransaction.h" +#include "fdbclient/FDBTypes.h" + +// Used to store a list of mutations made to the global configuration at a +// specific version. +struct VersionHistory { + constexpr static FileIdentifier file_identifier = 5863456; + + Version version; + Standalone> mutations; + + bool operator<(const VersionHistory& other) const { return version < other.version; } + + int expectedSize() const { return sizeof(version) + mutations.expectedSize(); } + + template + void serialize(Ar& ar) { + // The version is not serialized because this object is only sent over + // the network during a write. In this case, the version is included in + // the key, while this object will be written to the value. + serializer(ar, mutations); + } +}; diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 356cdfc2c1..6bec7665d1 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1409,42 +1409,19 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl // History should only contain three most recent updates. If it currently // has three items, remove the oldest to make room for a new item. - Standalone history = wait(tr.getRange(globalConfigHistoryKeys, CLIENT_KNOBS->TOO_MANY, false, true)); + Standalone history = wait(tr.getRange(globalConfigHistoryKeys, CLIENT_KNOBS->TOO_MANY)); constexpr int kGlobalConfigMaxHistorySize = 3; if (history.size() > kGlobalConfigMaxHistorySize - 1) { - std::vector keys; - for (const auto& kv : history) { - keys.push_back(kv.key); - } - // Fix ordering of returned keys. This will ensure versions are ordered - // numerically; for example \xff/globalConfig/h/1000 should come after - // \xff/globalConfig/h/999. - std::sort(keys.begin(), keys.end(), [](const KeyRef& lhs, const KeyRef& rhs) { - if (lhs.size() != rhs.size()) { - return lhs.size() < rhs.size(); - } - return lhs.compare(rhs) < 0; - }); - - // Cannot use a range clear because of how keys are ordered in FDB. - // \xff/globalConfig/h/999 -> ... - // \xff/globalConfig/h/1000 -> ... - // \xff/globalConfig/h/1001 -> ... - // - // clear_range(\xff/globalConfig/h, \xff/globalConfig/h/1000) results - // in zero key-value pairs being deleted (999 is lexicographically - // larger than 1000, and the range is exclusive). - // Delete the oldest key(s) in the history to make room for new data. - for (int i = 0; i < keys.size() - (kGlobalConfigMaxHistorySize - 1); ++i) { - tr.clear(keys[i]); + for (int i = 0; i < history.size() - (kGlobalConfigMaxHistorySize - 1); ++i) { + tr.clear(history[i].key); } } - Arena arena; - VectorRef mutations; + VersionHistory vh; - // Transform writes from special-key-space (\xff\xff/global_config/) to - // system key space (\xff/globalConfig/). + // Transform writes from the special-key-space (\xff\xff/global_config/) to + // the system key space (\xff/globalConfig/), and writes mutations to + // latest version history. state RangeMap>, KeyRangeRef>::Ranges ranges = ryw->getSpecialKeySpaceWriteMap().containedRanges(specialKeys); state RangeMap>, KeyRangeRef>::iterator iter = ranges.begin(); @@ -1453,14 +1430,16 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl if (entry.first) { if (entry.second.present()) { Key bareKey = iter->begin().removePrefix(globalConfig->getKeyRange().begin); - mutations.emplace_back_deep(arena, MutationRef(MutationRef::SetValue, bareKey, entry.second.get())); + vh.mutations.emplace_back_deep(vh.mutations.arena(), + MutationRef(MutationRef::SetValue, bareKey, entry.second.get())); Key systemKey = bareKey.withPrefix(globalConfigKeysPrefix); tr.set(systemKey, entry.second.get()); } else { KeyRef bareRangeBegin = iter->range().begin.removePrefix(globalConfig->getKeyRange().begin); KeyRef bareRangeEnd = iter->range().end.removePrefix(globalConfig->getKeyRange().begin); - mutations.emplace_back_deep(arena, MutationRef(MutationRef::ClearRange, bareRangeBegin, bareRangeEnd)); + vh.mutations.emplace_back_deep(vh.mutations.arena(), + MutationRef(MutationRef::ClearRange, bareRangeBegin, bareRangeEnd)); Key systemRangeBegin = bareRangeBegin.withPrefix(globalConfigKeysPrefix); Key systemRangeEnd = bareRangeEnd.withPrefix(globalConfigKeysPrefix); @@ -1470,27 +1449,18 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl ++iter; } - ProtocolVersion protocolVersion = g_network->protocolVersion(); - // Record the mutations in this commit into the global configuration history. - BinaryWriter historyKeyWriter(AssumeVersion(protocolVersion)); - historyKeyWriter.serializeBytes(globalConfigHistoryPrefix); - Key historyKey = addVersionStampAtEnd(historyKeyWriter.toValue()); - - BinaryWriter historyMutationsWriter(AssumeVersion(protocolVersion)); - historyMutationsWriter << mutations; - - tr.atomicOp(historyKey, historyMutationsWriter.toValue(), MutationRef::SetVersionstampedKey); + Key historyKey = addVersionStampAtEnd(globalConfigHistoryPrefix); + ObjectWriter historyWriter(IncludeVersion()); + historyWriter.serialize(vh); + tr.atomicOp(historyKey, historyWriter.toStringRef(), MutationRef::SetVersionstampedKey); // Write version key to trigger update in cluster controller. tr.atomicOp(globalConfigVersionKey, - BinaryWriter::toValue(protocolVersion, AssumeVersion(protocolVersion)) - .withPrefix(LiteralStringRef("0123456789")) // placeholder for versionstamp - .withSuffix(LiteralStringRef("\x00\x00\x00\x00")), + LiteralStringRef("0123456789\x00\x00\x00\x00"), // versionstamp MutationRef::SetVersionstampedValue); return Optional(); - } Future> GlobalConfigImpl::commit(ReadYourWritesTransaction* ryw) { diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 28bb8c4d4d..93632650e1 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -41,6 +41,7 @@ #include "fdbserver/Status.h" #include "fdbserver/LatencyBandConfig.h" #include "fdbclient/DatabaseContext.h" +#include "fdbclient/GlobalConfig.actor.h" #include "fdbserver/RecoveryState.h" #include "fdbclient/ReadYourWrites.h" #include "fdbrpc/Replication.h" @@ -3189,7 +3190,7 @@ ACTOR Future monitorServerInfoConfig(ClusterControllerData::DBInfo* db) { } } -ACTOR Future monitorClientTxnInfoConfigs(ClusterControllerData::DBInfo* db) { +ACTOR Future monitorGlobalConfig(ClusterControllerData::DBInfo* db) { loop { state ReadYourWritesTransaction tr(db->db); loop { @@ -3200,49 +3201,39 @@ ACTOR Future monitorClientTxnInfoConfigs(ClusterControllerData::DBInfo* db state ClientDBInfo clientInfo = db->clientInfo->get(); if (globalConfigVersion.present()) { - BinaryReader versionReader = BinaryReader(globalConfigVersion.get(), AssumeVersion(g_network->protocolVersion())); - int64_t commitVersion; // Currently unused. Convert to little endian if you want to use it - int16_t serializationOrder; - state ProtocolVersion protocolVersion; - versionReader >> commitVersion >> serializationOrder >> protocolVersion; + // Since the history keys end with versionstamps, they + // should be sorted correctly (versionstamps are stored in + // big-endian order). + Standalone globalConfigHistory = + wait(tr.getRange(globalConfigHistoryKeys, CLIENT_KNOBS->TOO_MANY)); + // If the global configuration version key has been set, + // the history should contain at least one item. + ASSERT(globalConfigHistory.size() > 0); + clientInfo.history.clear(); - if (protocolVersion == g_network->protocolVersion()) { - Standalone globalConfigHistory = wait(tr.getRange(globalConfigHistoryKeys, CLIENT_KNOBS->TOO_MANY)); - // If the global configuration version key has been - // set, the history should contain at least one item. - ASSERT(globalConfigHistory.size() > 0); - clientInfo.history.clear(); - - for (const auto& kv : globalConfigHistory) { - Standalone>> data; - - // Read commit version out of versionstamp at end of key. - BinaryReader versionReader = BinaryReader(kv.key.removePrefix(globalConfigHistoryPrefix), AssumeVersion(protocolVersion)); - Version historyCommitVersion; - versionReader >> historyCommitVersion; - historyCommitVersion = bigEndian64(historyCommitVersion); - data.first = historyCommitVersion; - - // Read the list of mutations that occurred at this version. - BinaryReader mutationReader = BinaryReader(kv.value, AssumeVersion(protocolVersion)); - VectorRef mutations; - mutationReader >> mutations; - data.second = VectorRef(data.arena(), mutations); - - clientInfo.history.push_back(data); + for (const auto& kv : globalConfigHistory) { + VersionHistory vh; + ObjectReader reader(kv.value.begin(), IncludeVersion()); + if (reader.protocolVersion() != g_network->protocolVersion()) { + // If the protocol version has changed, the + // GlobalConfig actor should refresh its view by + // reading the entire global configuration key + // range. An empty mutation list will signal the + // actor to refresh. + clientInfo.history.clear(); + break; } + reader.deserialize(vh); - // History should be ordered by version, ascending. - std::sort(clientInfo.history.begin(), clientInfo.history.end(), [](const auto& lhs, const auto& rhs) { - return lhs.first < rhs.first; - }); - } else { - // If the protocol version has changed, the - // GlobalConfig actor should refresh its view by - // reading the entire global configuration key range. - // An empty mutation list will signal the actor to - // refresh. - clientInfo.history.clear(); + // Read commit version out of versionstamp at end of key. + BinaryReader versionReader = + BinaryReader(kv.key.removePrefix(globalConfigHistoryPrefix), Unversioned()); + Version historyCommitVersion; + versionReader >> historyCommitVersion; + historyCommitVersion = bigEndian64(historyCommitVersion); + vh.version = historyCommitVersion; + + clientInfo.history.push_back(vh); } clientInfo.id = deterministicRandom()->randomUniqueID(); @@ -3711,29 +3702,25 @@ ACTOR Future clusterControllerCore(ClusterControllerFullInterface interf, state ClusterControllerData self(interf, locality); state Future coordinationPingDelay = delay(SERVER_KNOBS->WORKER_COORDINATION_PING_DELAY); state uint64_t step = 0; - state Future> error = errorOr(actorCollection(self.addActor.getFuture())); + state Future> error = errorOr( actorCollection( self.addActor.getFuture() ) ); - self.addActor.send(clusterWatchDatabase(&self, &self.db)); // Start the master database - self.addActor.send(self.updateWorkerList.init(self.db.db)); - self.addActor.send(statusServer(interf.clientInterface.databaseStatus.getFuture(), &self, coordinators)); - self.addActor.send(timeKeeper(&self)); - self.addActor.send(monitorProcessClasses(&self)); - self.addActor.send(monitorServerInfoConfig(&self.db)); - self.addActor.send(monitorClientTxnInfoConfigs(&self.db)); - self.addActor.send(updatedChangingDatacenters(&self)); - self.addActor.send(updatedChangedDatacenters(&self)); - self.addActor.send(updateDatacenterVersionDifference(&self)); - self.addActor.send(handleForcedRecoveries(&self, interf)); - self.addActor.send(monitorDataDistributor(&self)); - self.addActor.send(monitorRatekeeper(&self)); - self.addActor.send(dbInfoUpdater(&self)); - self.addActor.send(traceCounters("ClusterControllerMetrics", - self.id, - SERVER_KNOBS->STORAGE_LOGGING_DELAY, - &self.clusterControllerMetrics, - self.id.toString() + "/ClusterControllerMetrics")); - self.addActor.send(traceRole(Role::CLUSTER_CONTROLLER, interf.id())); - // printf("%s: I am the cluster controller\n", g_network->getLocalAddress().toString().c_str()); + self.addActor.send( clusterWatchDatabase( &self, &self.db ) ); // Start the master database + self.addActor.send( self.updateWorkerList.init( self.db.db ) ); + self.addActor.send( statusServer( interf.clientInterface.databaseStatus.getFuture(), &self, coordinators)); + self.addActor.send( timeKeeper(&self) ); + self.addActor.send( monitorProcessClasses(&self) ); + self.addActor.send( monitorServerInfoConfig(&self.db) ); + self.addActor.send(monitorGlobalConfig(&self.db)); + self.addActor.send( updatedChangingDatacenters(&self) ); + self.addActor.send( updatedChangedDatacenters(&self) ); + self.addActor.send( updateDatacenterVersionDifference(&self) ); + self.addActor.send( handleForcedRecoveries(&self, interf) ); + self.addActor.send( monitorDataDistributor(&self) ); + self.addActor.send( monitorRatekeeper(&self) ); + self.addActor.send( dbInfoUpdater(&self) ); + self.addActor.send( traceCounters("ClusterControllerMetrics", self.id, SERVER_KNOBS->STORAGE_LOGGING_DELAY, &self.clusterControllerMetrics, self.id.toString() + "/ClusterControllerMetrics") ); + self.addActor.send( traceRole(Role::CLUSTER_CONTROLLER, interf.id()) ); + //printf("%s: I am the cluster controller\n", g_network->getLocalAddress().toString().c_str()); loop choose { when(ErrorOr err = wait(error)) { From 6de28dd916df282d025044ad590ee94f69742806 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Tue, 16 Mar 2021 17:20:25 -0700 Subject: [PATCH 44/60] clang-format --- fdbcli/fdbcli.actor.cpp | 6 +- fdbclient/GlobalConfig.actor.cpp | 2 +- fdbclient/NativeAPI.actor.cpp | 23 +++++--- fdbclient/SpecialKeySpace.actor.cpp | 15 +++-- fdbclient/SystemData.cpp | 3 +- fdbclient/Tuple.cpp | 57 +++++++++---------- fdbserver/ClusterController.actor.cpp | 40 +++++++------ ...entTransactionProfileCorrectness.actor.cpp | 20 +++---- 8 files changed, 89 insertions(+), 77 deletions(-) diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index e8167c4855..d655601e22 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -3842,8 +3842,10 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { is_error = true; continue; } - const double sampleRateDbl = GlobalConfig::globalConfig().get(fdbClientInfoTxnSampleRate, std::numeric_limits::infinity()); - const int64_t sizeLimit = GlobalConfig::globalConfig().get(fdbClientInfoTxnSizeLimit, -1); + const double sampleRateDbl = GlobalConfig::globalConfig().get( + fdbClientInfoTxnSampleRate, std::numeric_limits::infinity()); + const int64_t sizeLimit = + GlobalConfig::globalConfig().get(fdbClientInfoTxnSizeLimit, -1); std::string sampleRateStr = "default", sizeLimitStr = "default"; if (!std::isinf(sampleRateDbl)) { sampleRateStr = boost::lexical_cast(sampleRateDbl); diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index 5315a8f68a..a482a8e3ea 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -149,7 +149,7 @@ ACTOR Future GlobalConfig::updater(GlobalConfig* self, ReferencelastUpdate) { - continue; // already applied this mutation + continue; // already applied this mutation } for (const auto& mutation : vh.mutations.contents()) { diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 92d6d6711e..2e460d55d2 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -507,8 +507,10 @@ ACTOR static Future clientStatusUpdateActor(DatabaseContext* cx) { } cx->clientStatusUpdater.outStatusQ.clear(); wait(GlobalConfig::globalConfig().onInitialized()); - double sampleRate = GlobalConfig::globalConfig().get(fdbClientInfoTxnSampleRate, std::numeric_limits::infinity()); - double clientSamplingProbability = std::isinf(sampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : sampleRate; + double sampleRate = GlobalConfig::globalConfig().get(fdbClientInfoTxnSampleRate, + std::numeric_limits::infinity()); + double clientSamplingProbability = + std::isinf(sampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : sampleRate; int64_t sizeLimit = GlobalConfig::globalConfig().get(fdbClientInfoTxnSizeLimit, -1); int64_t clientTxnInfoSizeLimit = sizeLimit == -1 ? CLIENT_KNOBS->CSI_SIZE_LIMIT : sizeLimit; if (!trChunksQ.empty() && deterministicRandom()->random01() < clientSamplingProbability) @@ -1277,8 +1279,10 @@ bool DatabaseContext::sampleReadTags() const { } bool DatabaseContext::sampleOnCost(uint64_t cost) const { - double sampleCost = GlobalConfig::globalConfig().get(transactionTagSampleCost, CLIENT_KNOBS->COMMIT_SAMPLE_COST); - if (sampleCost <= 0) return false; + double sampleCost = + GlobalConfig::globalConfig().get(transactionTagSampleCost, CLIENT_KNOBS->COMMIT_SAMPLE_COST); + if (sampleCost <= 0) + return false; return deterministicRandom()->random01() <= (double)cost / sampleCost; } @@ -5374,11 +5378,14 @@ void Transaction::checkDeferredError() { cx->checkDeferredError(); } -Reference Transaction::createTrLogInfoProbabilistically(const Database &cx) { - if(!cx->isError()) { - double sampleRate = GlobalConfig::globalConfig().get(fdbClientInfoTxnSampleRate, std::numeric_limits::infinity()); +Reference Transaction::createTrLogInfoProbabilistically(const Database& cx) { + if (!cx->isError()) { + double sampleRate = GlobalConfig::globalConfig().get(fdbClientInfoTxnSampleRate, + std::numeric_limits::infinity()); double clientSamplingProbability = std::isinf(sampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : sampleRate; - if (((networkOptions.logClientInfo.present() && networkOptions.logClientInfo.get()) || BUGGIFY) && deterministicRandom()->random01() < clientSamplingProbability && (!g_network->isSimulated() || !g_simulator.speedUpSimulation)) { + if (((networkOptions.logClientInfo.present() && networkOptions.logClientInfo.get()) || BUGGIFY) && + deterministicRandom()->random01() < clientSamplingProbability && + (!g_network->isSimulated() || !g_simulator.speedUpSimulation)) { return makeReference(TransactionLogInfo::DATABASE); } } diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 6bec7665d1..bff44b03ba 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1378,19 +1378,24 @@ Future> GlobalConfigImpl::getRange(ReadYourWritesTran Standalone result; auto& globalConfig = GlobalConfig::globalConfig(); - KeyRangeRef modified = KeyRangeRef(kr.begin.removePrefix(getKeyRange().begin), kr.end.removePrefix(getKeyRange().begin)); + KeyRangeRef modified = + KeyRangeRef(kr.begin.removePrefix(getKeyRange().begin), kr.end.removePrefix(getKeyRange().begin)); std::map values = globalConfig.get(modified); for (const auto& [key, config] : values) { Key prefixedKey = key.withPrefix(getKeyRange().begin); if (config.value.has_value()) { if (config.value.type() == typeid(StringRef)) { - result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::any_cast(config.value).toString())); + result.push_back_deep(result.arena(), + KeyValueRef(prefixedKey, std::any_cast(config.value).toString())); } else if (config.value.type() == typeid(int64_t)) { - result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); + result.push_back_deep(result.arena(), + KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); } else if (config.value.type() == typeid(float)) { - result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); + result.push_back_deep(result.arena(), + KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); } else if (config.value.type() == typeid(double)) { - result.push_back_deep(result.arena(), KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); + result.push_back_deep(result.arena(), + KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); } else { ASSERT(false); } diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 7c12a69059..42fec5f9f2 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -757,7 +757,8 @@ const KeyRef tagThrottleLimitKey = LiteralStringRef("\xff\x02/throttledTags/manu const KeyRef tagThrottleCountKey = LiteralStringRef("\xff\x02/throttledTags/manualThrottleCount"); // Client status info prefix -const KeyRangeRef fdbClientInfoPrefixRange(LiteralStringRef("\xff\x02/fdbClientInfo/"), LiteralStringRef("\xff\x02/fdbClientInfo0")); +const KeyRangeRef fdbClientInfoPrefixRange(LiteralStringRef("\xff\x02/fdbClientInfo/"), + LiteralStringRef("\xff\x02/fdbClientInfo0")); // See remaining fields in GlobalConfig.actor.h // ConsistencyCheck settings diff --git a/fdbclient/Tuple.cpp b/fdbclient/Tuple.cpp index 96f806c791..9d81281b14 100644 --- a/fdbclient/Tuple.cpp +++ b/fdbclient/Tuple.cpp @@ -44,9 +44,10 @@ static size_t find_string_terminator(const StringRef data, size_t offset) { // If encoding and the sign bit is 1 (the number is negative), flip all the bits. // If decoding and the sign bit is 0 (the number is negative), flip all the bits. // Otherwise, the number is positive, so flip the sign bit. -static void adjust_floating_point(uint8_t *bytes, size_t size, bool encode) { - if((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) { - for(size_t i = 0; i < size; i++) { +static void adjust_floating_point(uint8_t* bytes, size_t size, bool encode) { + if ((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || + (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) { + for (size_t i = 0; i < size; i++) { bytes[i] ^= (uint8_t)0xff; } } else { @@ -65,14 +66,11 @@ Tuple::Tuple(StringRef const& str, bool exclude_incomplete) { i = find_string_terminator(str, i + 1) + 1; } else if (data[i] >= '\x0c' && data[i] <= '\x1c') { i += abs(data[i] - '\x14') + 1; - } - else if(data[i] == 0x20) { + } else if (data[i] == 0x20) { i += sizeof(float) + 1; - } - else if(data[i] == 0x21) { + } else if (data[i] == 0x21) { i += sizeof(double) + 1; - } - else if(data[i] == '\x00') { + } else if (data[i] == '\x00') { i += 1; } else { throw invalid_tuple_data_type(); @@ -145,26 +143,26 @@ Tuple& Tuple::append(int64_t value) { return *this; } -Tuple& Tuple::appendFloat( float value ) { - offsets.push_back( data.size() ); +Tuple& Tuple::appendFloat(float value) { + offsets.push_back(data.size()); float swap = bigEndianFloat(value); - uint8_t *bytes = (uint8_t*)&swap; + uint8_t* bytes = (uint8_t*)&swap; adjust_floating_point(bytes, sizeof(float), true); - data.push_back( data.arena(), 0x20 ); - data.append( data.arena(), bytes, sizeof(float) ); + data.push_back(data.arena(), 0x20); + data.append(data.arena(), bytes, sizeof(float)); return *this; } -Tuple& Tuple::appendDouble( double value ) { - offsets.push_back( data.size() ); +Tuple& Tuple::appendDouble(double value) { + offsets.push_back(data.size()); double swap = value; swap = bigEndianDouble(swap); - uint8_t *bytes = (uint8_t*)&swap; + uint8_t* bytes = (uint8_t*)&swap; adjust_floating_point(bytes, sizeof(double), true); - data.push_back( data.arena(), 0x21 ); - data.append( data.arena(), bytes, sizeof(double) ); + data.push_back(data.arena(), 0x21); + data.append(data.arena(), bytes, sizeof(double)); return *this; } @@ -189,14 +187,11 @@ Tuple::ElementType Tuple::getType(size_t index) const { return ElementType::UTF8; } else if (code >= '\x0c' && code <= '\x1c') { return ElementType::INT; - } - else if(code == 0x20) { + } else if (code == 0x20) { return ElementType::FLOAT; - } - else if(code == 0x21) { + } else if (code == 0x21) { return ElementType::DOUBLE; - } - else { + } else { throw invalid_tuple_data_type(); } } @@ -292,12 +287,12 @@ int64_t Tuple::getInt(size_t index, bool allow_incomplete) const { // TODO: Combine with bindings/flow/Tuple.*. This code is copied from there. float Tuple::getFloat(size_t index) const { - if(index >= offsets.size()) { + if (index >= offsets.size()) { throw invalid_tuple_index(); } ASSERT_LT(offsets[index], data.size()); uint8_t code = data[offsets[index]]; - if(code != 0x20) { + if (code != 0x20) { throw invalid_tuple_data_type(); } @@ -305,18 +300,18 @@ float Tuple::getFloat(size_t index) const { uint8_t* bytes = (uint8_t*)&swap; ASSERT_LE(offsets[index] + 1 + sizeof(float), data.size()); swap = *(float*)(data.begin() + offsets[index] + 1); - adjust_floating_point( bytes, sizeof(float), false ); + adjust_floating_point(bytes, sizeof(float), false); return bigEndianFloat(swap); } double Tuple::getDouble(size_t index) const { - if(index >= offsets.size()) { + if (index >= offsets.size()) { throw invalid_tuple_index(); } ASSERT_LT(offsets[index], data.size()); uint8_t code = data[offsets[index]]; - if(code != 0x21) { + if (code != 0x21) { throw invalid_tuple_data_type(); } @@ -324,7 +319,7 @@ double Tuple::getDouble(size_t index) const { uint8_t* bytes = (uint8_t*)&swap; ASSERT_LE(offsets[index] + 1 + sizeof(double), data.size()); swap = *(double*)(data.begin() + offsets[index] + 1); - adjust_floating_point( bytes, sizeof(double), false ); + adjust_floating_point(bytes, sizeof(double), false); return bigEndianDouble(swap); } diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 93632650e1..0f5b7ebb94 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -2716,7 +2716,7 @@ void clusterRegisterMaster(ClusterControllerData* self, RegisterMasterRequest co clientInfo.id = deterministicRandom()->randomUniqueID(); clientInfo.commitProxies = req.commitProxies; clientInfo.grvProxies = req.grvProxies; - db->clientInfo->set( clientInfo ); + db->clientInfo->set(clientInfo); dbInfo.client = db->clientInfo->get(); } @@ -3702,25 +3702,29 @@ ACTOR Future clusterControllerCore(ClusterControllerFullInterface interf, state ClusterControllerData self(interf, locality); state Future coordinationPingDelay = delay(SERVER_KNOBS->WORKER_COORDINATION_PING_DELAY); state uint64_t step = 0; - state Future> error = errorOr( actorCollection( self.addActor.getFuture() ) ); + state Future> error = errorOr(actorCollection(self.addActor.getFuture())); - self.addActor.send( clusterWatchDatabase( &self, &self.db ) ); // Start the master database - self.addActor.send( self.updateWorkerList.init( self.db.db ) ); - self.addActor.send( statusServer( interf.clientInterface.databaseStatus.getFuture(), &self, coordinators)); - self.addActor.send( timeKeeper(&self) ); - self.addActor.send( monitorProcessClasses(&self) ); - self.addActor.send( monitorServerInfoConfig(&self.db) ); + self.addActor.send(clusterWatchDatabase(&self, &self.db)); // Start the master database + self.addActor.send(self.updateWorkerList.init(self.db.db)); + self.addActor.send(statusServer(interf.clientInterface.databaseStatus.getFuture(), &self, coordinators)); + self.addActor.send(timeKeeper(&self)); + self.addActor.send(monitorProcessClasses(&self)); + self.addActor.send(monitorServerInfoConfig(&self.db)); self.addActor.send(monitorGlobalConfig(&self.db)); - self.addActor.send( updatedChangingDatacenters(&self) ); - self.addActor.send( updatedChangedDatacenters(&self) ); - self.addActor.send( updateDatacenterVersionDifference(&self) ); - self.addActor.send( handleForcedRecoveries(&self, interf) ); - self.addActor.send( monitorDataDistributor(&self) ); - self.addActor.send( monitorRatekeeper(&self) ); - self.addActor.send( dbInfoUpdater(&self) ); - self.addActor.send( traceCounters("ClusterControllerMetrics", self.id, SERVER_KNOBS->STORAGE_LOGGING_DELAY, &self.clusterControllerMetrics, self.id.toString() + "/ClusterControllerMetrics") ); - self.addActor.send( traceRole(Role::CLUSTER_CONTROLLER, interf.id()) ); - //printf("%s: I am the cluster controller\n", g_network->getLocalAddress().toString().c_str()); + self.addActor.send(updatedChangingDatacenters(&self)); + self.addActor.send(updatedChangedDatacenters(&self)); + self.addActor.send(updateDatacenterVersionDifference(&self)); + self.addActor.send(handleForcedRecoveries(&self, interf)); + self.addActor.send(monitorDataDistributor(&self)); + self.addActor.send(monitorRatekeeper(&self)); + self.addActor.send(dbInfoUpdater(&self)); + self.addActor.send(traceCounters("ClusterControllerMetrics", + self.id, + SERVER_KNOBS->STORAGE_LOGGING_DELAY, + &self.clusterControllerMetrics, + self.id.toString() + "/ClusterControllerMetrics")); + self.addActor.send(traceRole(Role::CLUSTER_CONTROLLER, interf.id())); + // printf("%s: I am the cluster controller\n", g_network->getLocalAddress().toString().c_str()); loop choose { when(ErrorOr err = wait(error)) { diff --git a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp b/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp index d56061ca3a..5c99263f58 100644 --- a/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp +++ b/fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp @@ -269,17 +269,15 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload { ACTOR Future changeProfilingParameters(Database cx, int64_t sizeLimit, double sampleProbability) { - wait(runRYWTransaction(cx, [=](Reference tr) -> Future - { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - Tuple rate = Tuple().appendDouble(sampleProbability); - Tuple size = Tuple().append(sizeLimit); - tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack()); - tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack()); - return Void(); - } - )); + wait(runRYWTransaction(cx, [=](Reference tr) -> Future { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + Tuple rate = Tuple().appendDouble(sampleProbability); + Tuple size = Tuple().append(sizeLimit); + tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack()); + tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack()); + return Void(); + })); return Void(); } From aa0014ab6e672d59df2ba2372b86d72c841229f1 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Wed, 17 Mar 2021 20:22:38 -0700 Subject: [PATCH 45/60] Fix version serialization --- fdbclient/GlobalConfig.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fdbclient/GlobalConfig.h b/fdbclient/GlobalConfig.h index 14e10f8635..4973a86499 100644 --- a/fdbclient/GlobalConfig.h +++ b/fdbclient/GlobalConfig.h @@ -37,9 +37,6 @@ struct VersionHistory { template void serialize(Ar& ar) { - // The version is not serialized because this object is only sent over - // the network during a write. In this case, the version is included in - // the key, while this object will be written to the value. - serializer(ar, mutations); + serializer(ar, mutations, version); } }; From 1c60653c2acfc3b938c07dbbe17e60c2983445a2 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Wed, 17 Mar 2021 20:41:46 -0700 Subject: [PATCH 46/60] Add fix to conditionally set global config history --- fdbclient/CMakeLists.txt | 1 + fdbclient/GlobalConfig.actor.cpp | 5 +++++ fdbserver/ClusterController.actor.cpp | 4 ++-- fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp | 1 + 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt index e733259611..bd14ef7b52 100644 --- a/fdbclient/CMakeLists.txt +++ b/fdbclient/CMakeLists.txt @@ -28,6 +28,7 @@ set(FDBCLIENT_SRCS FDBOptions.h FDBTypes.h FileBackupAgent.actor.cpp + GlobalConfig.h GlobalConfig.actor.h GlobalConfig.actor.cpp GrvProxyInterface.h diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index a482a8e3ea..f504ec2db2 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -18,6 +18,7 @@ * limitations under the License. */ +#include "fdbclient/DatabaseContext.h" #include "fdbclient/GlobalConfig.actor.h" #include "fdbclient/SpecialKeySpace.actor.h" #include "fdbclient/SystemData.h" @@ -134,6 +135,10 @@ ACTOR Future GlobalConfig::updater(GlobalConfig* self, ReferenceonChange()); + if (dbInfo->get().id.second() != 123456789) { + continue; + } + auto& history = dbInfo->get().history; if (history.size() == 0 || (self->lastUpdate < history[0].version && self->lastUpdate != 0)) { // This process missed too many global configuration diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 0f5b7ebb94..47b19a932a 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -3233,10 +3233,10 @@ ACTOR Future monitorGlobalConfig(ClusterControllerData::DBInfo* db) { historyCommitVersion = bigEndian64(historyCommitVersion); vh.version = historyCommitVersion; - clientInfo.history.push_back(vh); + clientInfo.history.push_back(std::move(vh)); } - clientInfo.id = deterministicRandom()->randomUniqueID(); + clientInfo.id = UID(deterministicRandom()->randomUniqueID().first(), 123456789); db->clientInfo->set(clientInfo); } diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index 34c1f32cf4..a6c98910d2 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -21,6 +21,7 @@ #include "boost/lexical_cast.hpp" #include "boost/algorithm/string.hpp" +#include "fdbclient/GlobalConfig.actor.h" #include "fdbclient/ManagementAPI.actor.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" From 7ba7257cd2812e76fb992e06c544a82320140f73 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 19 Mar 2021 13:28:03 -0700 Subject: [PATCH 47/60] Store global config data on heap --- fdbclient/GlobalConfig.actor.cpp | 26 ++++++++++++++----------- fdbclient/GlobalConfig.actor.h | 28 +++++++++++++++------------ fdbclient/GlobalConfig.h | 3 +++ fdbclient/SpecialKeySpace.actor.cpp | 20 +++++++++---------- fdbserver/ClusterController.actor.cpp | 12 ++++++++---- 5 files changed, 52 insertions(+), 37 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index f504ec2db2..f8f97916f8 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -55,16 +55,16 @@ Key GlobalConfig::prefixedKey(KeyRef key) { return key.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::GLOBALCONFIG).begin); } -const ConfigValue GlobalConfig::get(KeyRef name) { +const Reference GlobalConfig::get(KeyRef name) { auto it = data.find(name); if (it == data.end()) { - return ConfigValue{ Arena(), std::any{} }; + return Reference(); } return it->second; } -const std::map GlobalConfig::get(KeyRangeRef range) { - std::map results; +const std::map> GlobalConfig::get(KeyRangeRef range) { + std::map> results; for (const auto& [key, value] : data) { if (range.contains(key)) { results[key] = value; @@ -78,21 +78,25 @@ Future GlobalConfig::onInitialized() { } void GlobalConfig::insert(KeyRef key, ValueRef value) { + data.erase(key); + Arena arena(key.expectedSize() + value.expectedSize()); KeyRef stableKey = KeyRef(arena, key); try { + std::any any; Tuple t = Tuple::unpack(value); if (t.getType(0) == Tuple::ElementType::UTF8) { - data[stableKey] = ConfigValue{ arena, StringRef(arena, t.getString(0).contents()) }; + any = StringRef(arena, t.getString(0).contents()); } else if (t.getType(0) == Tuple::ElementType::INT) { - data[stableKey] = ConfigValue{ arena, t.getInt(0) }; + any = t.getInt(0); } else if (t.getType(0) == Tuple::ElementType::FLOAT) { - data[stableKey] = ConfigValue{ arena, t.getFloat(0) }; + any = t.getFloat(0); } else if (t.getType(0) == Tuple::ElementType::DOUBLE) { - data[stableKey] = ConfigValue{ arena, t.getDouble(0) }; + any = t.getDouble(0); } else { ASSERT(false); } + data[stableKey] = makeReference(std::move(arena), std::move(any)); } catch (Error& e) { TraceEvent("GlobalConfigTupleError").detail("What", e.what()); } @@ -135,12 +139,12 @@ ACTOR Future GlobalConfig::updater(GlobalConfig* self, ReferenceonChange()); - if (dbInfo->get().id.second() != 123456789) { + auto& history = dbInfo->get().history; + if (history.size() == 0) { continue; } - auto& history = dbInfo->get().history; - if (history.size() == 0 || (self->lastUpdate < history[0].version && self->lastUpdate != 0)) { + if (self->lastUpdate < history[0].version) { // This process missed too many global configuration // history updates or the protocol version changed, so it // must re-read the entire configuration range. diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index 11699633da..45a3e469e9 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -48,9 +48,12 @@ extern const KeyRef fdbClientInfoTxnSizeLimit; extern const KeyRef transactionTagSampleRate; extern const KeyRef transactionTagSampleCost; -struct ConfigValue { +struct ConfigValue : ReferenceCounted { Arena arena; std::any value; + + ConfigValue() {} + ConfigValue(Arena&& a, std::any&& v) : arena(a), value(v) {} }; class GlobalConfig { @@ -67,13 +70,12 @@ public: // For example, given "config/a", returns "\xff\xff/global_config/config/a". static Key prefixedKey(KeyRef key); - // Get a value from the framework. Values are returned in a ConfigValue - // struct which also contains a reference to the arena containing the - // memory for the object. As long as the caller keeps a reference to the - // returned ConfigValue, the value is guaranteed to be readable (if it - // exists). - const ConfigValue get(KeyRef name); - const std::map get(KeyRangeRef range); + // Get a value from the framework. Values are returned as a ConfigValue + // reference which also contains the arena holding the object. As long as + // the caller keeps the ConfigValue reference, the value is guaranteed to + // be readable. An empty reference is returned if the value does not exist. + const Reference get(KeyRef name); + const std::map> get(KeyRangeRef range); // For arithmetic value types, returns a copy of the value for the given // key, or the supplied default value if the framework does not know about @@ -81,9 +83,11 @@ public: template {}, bool>::type = true> const T get(KeyRef name, T defaultVal) { try { - auto any = get(name).value; - if (any.has_value()) { - return std::any_cast(any); + auto configValue = get(name); + if (configValue.isValid()) { + if (configValue->value.has_value()) { + return std::any_cast(configValue->value); + } } return defaultVal; @@ -114,7 +118,7 @@ private: Database cx; Future _updater; Promise initialized; - std::unordered_map data; + std::unordered_map> data; Version lastUpdate; }; diff --git a/fdbclient/GlobalConfig.h b/fdbclient/GlobalConfig.h index 4973a86499..f68ea2361e 100644 --- a/fdbclient/GlobalConfig.h +++ b/fdbclient/GlobalConfig.h @@ -28,6 +28,9 @@ struct VersionHistory { constexpr static FileIdentifier file_identifier = 5863456; + VersionHistory() {} + VersionHistory(Version v) : version(v) {} + Version version; Standalone> mutations; diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index bff44b03ba..4b76a14255 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1380,22 +1380,22 @@ Future> GlobalConfigImpl::getRange(ReadYourWritesTran auto& globalConfig = GlobalConfig::globalConfig(); KeyRangeRef modified = KeyRangeRef(kr.begin.removePrefix(getKeyRange().begin), kr.end.removePrefix(getKeyRange().begin)); - std::map values = globalConfig.get(modified); + std::map> values = globalConfig.get(modified); for (const auto& [key, config] : values) { Key prefixedKey = key.withPrefix(getKeyRange().begin); - if (config.value.has_value()) { - if (config.value.type() == typeid(StringRef)) { + if (config.isValid() && config->value.has_value()) { + if (config->value.type() == typeid(StringRef)) { result.push_back_deep(result.arena(), - KeyValueRef(prefixedKey, std::any_cast(config.value).toString())); - } else if (config.value.type() == typeid(int64_t)) { + KeyValueRef(prefixedKey, std::any_cast(config->value).toString())); + } else if (config->value.type() == typeid(int64_t)) { result.push_back_deep(result.arena(), - KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); - } else if (config.value.type() == typeid(float)) { + KeyValueRef(prefixedKey, std::to_string(std::any_cast(config->value)))); + } else if (config->value.type() == typeid(float)) { result.push_back_deep(result.arena(), - KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); - } else if (config.value.type() == typeid(double)) { + KeyValueRef(prefixedKey, std::to_string(std::any_cast(config->value)))); + } else if (config->value.type() == typeid(double)) { result.push_back_deep(result.arena(), - KeyValueRef(prefixedKey, std::to_string(std::any_cast(config.value)))); + KeyValueRef(prefixedKey, std::to_string(std::any_cast(config->value)))); } else { ASSERT(false); } diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 47b19a932a..4b7b26fb95 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -3212,17 +3212,21 @@ ACTOR Future monitorGlobalConfig(ClusterControllerData::DBInfo* db) { clientInfo.history.clear(); for (const auto& kv : globalConfigHistory) { - VersionHistory vh; ObjectReader reader(kv.value.begin(), IncludeVersion()); if (reader.protocolVersion() != g_network->protocolVersion()) { // If the protocol version has changed, the // GlobalConfig actor should refresh its view by // reading the entire global configuration key - // range. An empty mutation list will signal the - // actor to refresh. + // range. Setting the version to the max int64_t + // will always cause the global configuration + // updater to refresh its view of the configuration + // keyspace. clientInfo.history.clear(); + clientInfo.history.emplace_back(std::numeric_limits::max()); break; } + + VersionHistory vh; reader.deserialize(vh); // Read commit version out of versionstamp at end of key. @@ -3236,7 +3240,7 @@ ACTOR Future monitorGlobalConfig(ClusterControllerData::DBInfo* db) { clientInfo.history.push_back(std::move(vh)); } - clientInfo.id = UID(deterministicRandom()->randomUniqueID().first(), 123456789); + clientInfo.id = deterministicRandom()->randomUniqueID(); db->clientInfo->set(clientInfo); } From c38ddf5eb72a82abc9f6e4245ce275e9caead350 Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 19 Mar 2021 17:37:01 -0700 Subject: [PATCH 48/60] Add comments --- fdbclient/GlobalConfig.actor.h | 21 +++++++++++++++++++++ fdbclient/SpecialKeySpace.actor.cpp | 11 +++++++++++ fdbserver/ClusterController.actor.cpp | 5 +++++ 3 files changed, 37 insertions(+) diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index 45a3e469e9..dc62203746 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -48,6 +48,9 @@ extern const KeyRef fdbClientInfoTxnSizeLimit; extern const KeyRef transactionTagSampleRate; extern const KeyRef transactionTagSampleCost; +// Structure used to hold the values stored by global configuration. The arena +// is used as memory to store both the key and the value (the value is only +// stored in the arena if it is an object; primitives are just copied). struct ConfigValue : ReferenceCounted { Arena arena; std::any value; @@ -61,7 +64,14 @@ public: GlobalConfig(const GlobalConfig&) = delete; GlobalConfig& operator=(const GlobalConfig&) = delete; + // Creates a GlobalConfig singleton, accessed by calling GlobalConfig(). + // This function should only be called once by each process (however, it is + // idempotent and calling it multiple times will have no effect). static void create(DatabaseContext* cx, Reference> dbInfo); + + // Returns a reference to the global GlobalConfig object. Clients should + // call this function whenever they need to read a value out of the global + // configuration. static GlobalConfig& globalConfig(); // Use this function to turn a global configuration key defined above into @@ -108,8 +118,19 @@ public: private: GlobalConfig(); + // The functions below only affect the local copy of the global + // configuration keyspace! To insert or remove values across all nodes you + // must use a transaction (see the note above). + + // Inserts the given key-value pair into the local copy of the global + // configuration keyspace, overwriting the old key-value pair if it exists. + // `value` must be encoded using the FDB tuple typecodes. void insert(KeyRef key, ValueRef value); + // Removes the given key (and associated value) from the local copy of the + // global configuration keyspace. void erase(KeyRef key); + // Removes the given key range (and associated values) from the local copy + // of the global configuration keyspace. void erase(KeyRangeRef range); ACTOR static Future refresh(GlobalConfig* self); diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 4b76a14255..da53a91f93 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1373,6 +1373,10 @@ Future> ConsistencyCheckImpl::commit(ReadYourWritesTransac GlobalConfigImpl::GlobalConfigImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} +// Returns key-value pairs for each value stored in the global configuration +// framework within the range specified. The special-key-space getrange +// function should only be used for informational purposes. All values are +// returned as strings regardless of their true type. Future> GlobalConfigImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { Standalone result; @@ -1405,10 +1409,14 @@ Future> GlobalConfigImpl::getRange(ReadYourWritesTran return result; } +// Marks the key for insertion into global configuration. void GlobalConfigImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) { ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional(value))); } +// Writes global configuration changes to durable memory. Also writes the +// changes made in the transaction to a recent history set, and updates the +// latest version which the global configuration was updated at. ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* globalConfig, ReadYourWritesTransaction* ryw) { state Transaction& tr = ryw->getTransaction(); @@ -1468,14 +1476,17 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl return Optional(); } +// Called when a transaction includes keys in the global configuration special-key-space range. Future> GlobalConfigImpl::commit(ReadYourWritesTransaction* ryw) { return globalConfigCommitActor(this, ryw); } +// Marks the range for deletion from global configuration. void GlobalConfigImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { ryw->getSpecialKeySpaceWriteMap().insert(range, std::make_pair(true, Optional())); } +// Marks the key for deletion from global configuration. void GlobalConfigImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional())); } diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 4b7b26fb95..1d1d05a930 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -3190,6 +3190,11 @@ ACTOR Future monitorServerInfoConfig(ClusterControllerData::DBInfo* db) { } } +// Monitors the global configuration version key for changes. When changes are +// made, the global configuration history is read and any updates are sent to +// all processes in the system by updating the ClientDBInfo object. The +// GlobalConfig actor class contains the functionality to read the latest +// history and update the processes local view. ACTOR Future monitorGlobalConfig(ClusterControllerData::DBInfo* db) { loop { state ReadYourWritesTransaction tr(db->db); From 7de23918c0d14bfcd29e83b893704eb12301854d Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Tue, 23 Mar 2021 16:22:39 -0700 Subject: [PATCH 49/60] Add comments, fix erase bug, make optimizations --- fdbclient/GlobalConfig.actor.cpp | 6 ++++-- fdbclient/GlobalConfig.actor.h | 17 ++++++++--------- fdbclient/NativeAPI.actor.cpp | 5 ++--- fdbclient/SpecialKeySpace.actor.cpp | 5 +++-- fdbclient/SystemData.h | 6 +++--- fdbclient/Tuple.cpp | 15 ++++++++------- fdbserver/ClusterController.actor.cpp | 20 +++++++++++++++++--- 7 files changed, 45 insertions(+), 29 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index f8f97916f8..1071a08a28 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -98,12 +98,12 @@ void GlobalConfig::insert(KeyRef key, ValueRef value) { } data[stableKey] = makeReference(std::move(arena), std::move(any)); } catch (Error& e) { - TraceEvent("GlobalConfigTupleError").detail("What", e.what()); + TraceEvent("GlobalConfigTupleParseError").detail("What", e.what()); } } void GlobalConfig::erase(KeyRef key) { - erase(KeyRangeRef(key, keyAfter(key))); + data.erase(key); } void GlobalConfig::erase(KeyRangeRef range) { @@ -120,6 +120,8 @@ void GlobalConfig::erase(KeyRangeRef range) { // Updates local copy of global configuration by reading the entire key-range // from storage. ACTOR Future GlobalConfig::refresh(GlobalConfig* self) { + self->data.clear(); + Transaction tr(self->cx); Standalone result = wait(tr.getRange(globalConfigDataKeys, CLIENT_KNOBS->TOO_MANY)); for (const auto& kv : result) { diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index dc62203746..ded1bc32c6 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -39,7 +39,8 @@ // The global configuration is a series of typed key-value pairs synced to all // nodes (server and client) in an FDB cluster in an eventually consistent -// manner. +// manner. Only small key-value pairs should be stored in global configuration; +// an excessive amount of data can cause synchronization slowness. // Keys extern const KeyRef fdbClientInfoTxnSampleRate; @@ -59,11 +60,8 @@ struct ConfigValue : ReferenceCounted { ConfigValue(Arena&& a, std::any&& v) : arena(a), value(v) {} }; -class GlobalConfig { +class GlobalConfig : NonCopyable { public: - GlobalConfig(const GlobalConfig&) = delete; - GlobalConfig& operator=(const GlobalConfig&) = delete; - // Creates a GlobalConfig singleton, accessed by calling GlobalConfig(). // This function should only be called once by each process (however, it is // idempotent and calling it multiple times will have no effect). @@ -106,10 +104,11 @@ public: } } - // To write into the global configuration, submit a transaction to - // \xff\xff/global_config/ with encoded using the - // FDB tuple typecodes. Use the helper function `prefixedKey` to correctly - // prefix your global configuration key. + // Trying to write into the global configuration keyspace? To write data, + // submit a transaction to \xff\xff/global_config/ with + // encoded using the FDB tuple typecodes. Use the helper + // function `prefixedKey` to correctly prefix your global configuration + // key. // Triggers the returned future when the global configuration singleton has // been created and is ready. diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 2e460d55d2..be581a08ea 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -5380,9 +5380,8 @@ void Transaction::checkDeferredError() { Reference Transaction::createTrLogInfoProbabilistically(const Database& cx) { if (!cx->isError()) { - double sampleRate = GlobalConfig::globalConfig().get(fdbClientInfoTxnSampleRate, - std::numeric_limits::infinity()); - double clientSamplingProbability = std::isinf(sampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : sampleRate; + double clientSamplingProbability = GlobalConfig::globalConfig().get( + fdbClientInfoTxnSampleRate, CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY); if (((networkOptions.logClientInfo.present() && networkOptions.logClientInfo.get()) || BUGGIFY) && deterministicRandom()->random01() < clientSamplingProbability && (!g_network->isSimulated() || !g_simulator.speedUpSimulation)) { diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index da53a91f93..cb347f7fcd 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1441,14 +1441,15 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl while (iter != ranges.end()) { std::pair> entry = iter->value(); if (entry.first) { - if (entry.second.present()) { + if (entry.second.present() && iter->begin().startsWith(globalConfig->getKeyRange().begin)) { Key bareKey = iter->begin().removePrefix(globalConfig->getKeyRange().begin); vh.mutations.emplace_back_deep(vh.mutations.arena(), MutationRef(MutationRef::SetValue, bareKey, entry.second.get())); Key systemKey = bareKey.withPrefix(globalConfigKeysPrefix); tr.set(systemKey, entry.second.get()); - } else { + } else if (!entry.second.present() && iter->range().begin.startsWith(globalConfig->getKeyRange().begin) && + iter->range().end.startsWith(globalConfig->getKeyRange().begin)) { KeyRef bareRangeBegin = iter->range().begin.removePrefix(globalConfig->getKeyRange().begin); KeyRef bareRangeEnd = iter->range().end.removePrefix(globalConfig->getKeyRange().begin); vh.mutations.emplace_back_deep(vh.mutations.arena(), diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 5cf56ef7ec..952e8fcf00 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -249,9 +249,9 @@ extern const KeyRef globalConfigKeysPrefix; extern const KeyRangeRef globalConfigHistoryKeys; extern const KeyRef globalConfigHistoryPrefix; -// "\xff/globalConfig/v" := "version,protocol" -// Read-only key which returns the version and protocol of the most recent -// data written to the global configuration keyspace. +// "\xff/globalConfig/v" := "version" +// Read-only key which returns the commit version of the most recent mutation +// made to the global configuration keyspace. extern const KeyRef globalConfigVersionKey; // "\xff/workers/[[processID]]" := "" diff --git a/fdbclient/Tuple.cpp b/fdbclient/Tuple.cpp index 9d81281b14..367a7b80fb 100644 --- a/fdbclient/Tuple.cpp +++ b/fdbclient/Tuple.cpp @@ -20,6 +20,7 @@ #include "fdbclient/Tuple.h" +// TODO: Many functions copied from bindings/flow/Tuple.cpp. Merge at some point. static float bigEndianFloat(float orig) { int32_t big = *(int32_t*)&orig; big = bigEndian32(big); @@ -32,7 +33,7 @@ static double bigEndianDouble(double orig) { return *(double*)&big; } -static size_t find_string_terminator(const StringRef data, size_t offset) { +static size_t findStringTerminator(const StringRef data, size_t offset) { size_t i = offset; while (i < data.size() - 1 && !(data[i] == '\x00' && data[i + 1] != (uint8_t)'\xff')) { i += (data[i] == '\x00' ? 2 : 1); @@ -44,7 +45,7 @@ static size_t find_string_terminator(const StringRef data, size_t offset) { // If encoding and the sign bit is 1 (the number is negative), flip all the bits. // If decoding and the sign bit is 0 (the number is negative), flip all the bits. // Otherwise, the number is positive, so flip the sign bit. -static void adjust_floating_point(uint8_t* bytes, size_t size, bool encode) { +static void adjustFloatingPoint(uint8_t* bytes, size_t size, bool encode) { if ((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) { for (size_t i = 0; i < size; i++) { @@ -63,7 +64,7 @@ Tuple::Tuple(StringRef const& str, bool exclude_incomplete) { offsets.push_back(i); if (data[i] == '\x01' || data[i] == '\x02') { - i = find_string_terminator(str, i + 1) + 1; + i = findStringTerminator(str, i + 1) + 1; } else if (data[i] >= '\x0c' && data[i] <= '\x1c') { i += abs(data[i] - '\x14') + 1; } else if (data[i] == 0x20) { @@ -147,7 +148,7 @@ Tuple& Tuple::appendFloat(float value) { offsets.push_back(data.size()); float swap = bigEndianFloat(value); uint8_t* bytes = (uint8_t*)&swap; - adjust_floating_point(bytes, sizeof(float), true); + adjustFloatingPoint(bytes, sizeof(float), true); data.push_back(data.arena(), 0x20); data.append(data.arena(), bytes, sizeof(float)); @@ -159,7 +160,7 @@ Tuple& Tuple::appendDouble(double value) { double swap = value; swap = bigEndianDouble(swap); uint8_t* bytes = (uint8_t*)&swap; - adjust_floating_point(bytes, sizeof(double), true); + adjustFloatingPoint(bytes, sizeof(double), true); data.push_back(data.arena(), 0x21); data.append(data.arena(), bytes, sizeof(double)); @@ -300,7 +301,7 @@ float Tuple::getFloat(size_t index) const { uint8_t* bytes = (uint8_t*)&swap; ASSERT_LE(offsets[index] + 1 + sizeof(float), data.size()); swap = *(float*)(data.begin() + offsets[index] + 1); - adjust_floating_point(bytes, sizeof(float), false); + adjustFloatingPoint(bytes, sizeof(float), false); return bigEndianFloat(swap); } @@ -319,7 +320,7 @@ double Tuple::getDouble(size_t index) const { uint8_t* bytes = (uint8_t*)&swap; ASSERT_LE(offsets[index] + 1 + sizeof(double), data.size()); swap = *(double*)(data.begin() + offsets[index] + 1); - adjust_floating_point(bytes, sizeof(double), false); + adjustFloatingPoint(bytes, sizeof(double), false); return bigEndianDouble(swap); } diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 1d1d05a930..750d6643ed 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -3245,15 +3245,29 @@ ACTOR Future monitorGlobalConfig(ClusterControllerData::DBInfo* db) { clientInfo.history.push_back(std::move(vh)); } + if (clientInfo.history.size() > 0) { + // The first item in the historical list of mutations + // is only used to: + // a) Recognize that some historical changes may have + // been missed, and the entire global + // configuration keyspace needs to be read, or.. + // b) Check which historical updates have already + // been applied. If this is the case, the first + // history item must have a version greater than + // or equal to whatever version the global + // configuration was last updated at, and + // therefore won't need to be applied again. + clientInfo.history[0].mutations = Standalone>(); + } + clientInfo.id = deterministicRandom()->randomUniqueID(); db->clientInfo->set(clientInfo); } state Future globalConfigFuture = tr.watch(globalConfigVersionKey); wait(tr.commit()); - choose { - when (wait(globalConfigFuture)) { break; } - } + wait(globalConfigFuture); + break; } catch (Error& e) { wait(tr.onError(e)); } From 2594d91f113711b1593a515abbef8a335e57525e Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Mon, 12 Apr 2021 10:27:41 -0700 Subject: [PATCH 50/60] Update casing --- fdbclient/GlobalConfig.actor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index 1071a08a28..1aa8c84992 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -28,11 +28,11 @@ #include "flow/actorcompiler.h" // This must be the last #include. -const KeyRef fdbClientInfoTxnSampleRate = LiteralStringRef("config/fdbClientInfo/client_txn_sample_rate"); -const KeyRef fdbClientInfoTxnSizeLimit = LiteralStringRef("config/fdbClientInfo/client_txn_size_limit"); +const KeyRef fdbClientInfoTxnSampleRate = LiteralStringRef("config/fdb_client_info/client_txn_sample_rate"); +const KeyRef fdbClientInfoTxnSizeLimit = LiteralStringRef("config/fdb_client_info/client_txn_size_limit"); -const KeyRef transactionTagSampleRate = LiteralStringRef("config/transactionTagSampleRate"); -const KeyRef transactionTagSampleCost = LiteralStringRef("config/transactionTagSampleCost"); +const KeyRef transactionTagSampleRate = LiteralStringRef("config/transaction_tag_sample_rate"); +const KeyRef transactionTagSampleCost = LiteralStringRef("config/transaction_tag_sample_cost"); GlobalConfig::GlobalConfig() : lastUpdate(0) {} From 51e4c19675d99c6504df03c27ac396649cbfbd1d Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Tue, 13 Apr 2021 12:50:18 -0700 Subject: [PATCH 51/60] Add migration for client profiling keys --- fdbclient/GlobalConfig.actor.cpp | 47 ++++++++++++++++++++++++++++- fdbclient/GlobalConfig.actor.h | 1 + fdbclient/SpecialKeySpace.actor.cpp | 9 +++--- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/fdbclient/GlobalConfig.actor.cpp b/fdbclient/GlobalConfig.actor.cpp index 1aa8c84992..58e032d363 100644 --- a/fdbclient/GlobalConfig.actor.cpp +++ b/fdbclient/GlobalConfig.actor.cpp @@ -26,7 +26,7 @@ #include "flow/flow.h" #include "flow/genericactors.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. +#include "flow/actorcompiler.h" // This must be the last #include. const KeyRef fdbClientInfoTxnSampleRate = LiteralStringRef("config/fdb_client_info/client_txn_sample_rate"); const KeyRef fdbClientInfoTxnSizeLimit = LiteralStringRef("config/fdb_client_info/client_txn_size_limit"); @@ -117,6 +117,49 @@ void GlobalConfig::erase(KeyRangeRef range) { } } +// Older FDB versions used different keys for client profiling data. This +// function performs a one-time migration of data in these keys to the new +// global configuration key space. +ACTOR Future GlobalConfig::migrate(GlobalConfig* self) { + state Reference tr = makeReference(self->cx); + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + + state Key migratedKey("\xff\x02/fdbClientInfo/migrated/"_sr); + state Optional migrated = wait(tr->get(migratedKey)); + if (migrated.present()) { + // Already performed migration. + return Void(); + } + + state Optional sampleRate = wait(tr->get(Key("\xff\x02/fdbClientInfo/client_txn_sample_rate/"_sr))); + state Optional sizeLimit = wait(tr->get(Key("\xff\x02/fdbClientInfo/client_txn_size_limit/"_sr))); + + loop { + try { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + // The value doesn't matter too much, as long as the key is set. + tr->set(migratedKey.contents(), "1"_sr); + if (sampleRate.present()) { + const double sampleRateDbl = + BinaryReader::fromStringRef(sampleRate.get().contents(), Unversioned()); + Tuple rate = Tuple().appendDouble(sampleRateDbl); + tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack()); + } + if (sizeLimit.present()) { + const int64_t sizeLimitInt = + BinaryReader::fromStringRef(sizeLimit.get().contents(), Unversioned()); + Tuple size = Tuple().append(sizeLimitInt); + tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack()); + } + + wait(tr->commit()); + return Void(); + } catch (Error& e) { + throw; + } + } +} + // Updates local copy of global configuration by reading the entire key-range // from storage. ACTOR Future GlobalConfig::refresh(GlobalConfig* self) { @@ -134,6 +177,8 @@ ACTOR Future GlobalConfig::refresh(GlobalConfig* self) { // Applies updates to the local copy of the global configuration when this // process receives an updated history. ACTOR Future GlobalConfig::updater(GlobalConfig* self, Reference> dbInfo) { + wait(self->migrate(self)); + wait(self->refresh(self)); self->initialized.send(Void()); diff --git a/fdbclient/GlobalConfig.actor.h b/fdbclient/GlobalConfig.actor.h index ded1bc32c6..5c3693f450 100644 --- a/fdbclient/GlobalConfig.actor.h +++ b/fdbclient/GlobalConfig.actor.h @@ -132,6 +132,7 @@ private: // of the global configuration keyspace. void erase(KeyRangeRef range); + ACTOR static Future migrate(GlobalConfig* self); ACTOR static Future refresh(GlobalConfig* self); ACTOR static Future updater(GlobalConfig* self, Reference> dbInfo); diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index cb347f7fcd..2bbafbd451 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1377,8 +1377,7 @@ GlobalConfigImpl::GlobalConfigImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) { // framework within the range specified. The special-key-space getrange // function should only be used for informational purposes. All values are // returned as strings regardless of their true type. -Future> GlobalConfigImpl::getRange(ReadYourWritesTransaction* ryw, - KeyRangeRef kr) const { +Future> GlobalConfigImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { Standalone result; auto& globalConfig = GlobalConfig::globalConfig(); @@ -1417,7 +1416,8 @@ void GlobalConfigImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, co // Writes global configuration changes to durable memory. Also writes the // changes made in the transaction to a recent history set, and updates the // latest version which the global configuration was updated at. -ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* globalConfig, ReadYourWritesTransaction* ryw) { +ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* globalConfig, + ReadYourWritesTransaction* ryw) { state Transaction& tr = ryw->getTransaction(); // History should only contain three most recent updates. If it currently @@ -1494,8 +1494,7 @@ void GlobalConfigImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) TracingOptionsImpl::TracingOptionsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} -Future> TracingOptionsImpl::getRange(ReadYourWritesTransaction* ryw, - KeyRangeRef kr) const { +Future> TracingOptionsImpl::getRange(ReadYourWritesTransaction* ryw, KeyRangeRef kr) const { Standalone result; for (const auto& option : SpecialKeySpace::getTracingOptions()) { auto key = getKeyRange().begin.withSuffix(option); From 3ed0d614d29e5734ba03c5492386fddb5ee629cf Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 14 Apr 2021 12:50:30 -0700 Subject: [PATCH 52/60] Move fdb_get_server_protocol to be a function on the database object. Add an argument for expected_version that can be used to signal that the function shouldn't return unless the protocol version is different. --- bindings/c/fdb_c.cpp | 15 +++++-- bindings/c/foundationdb/fdb_c.h | 4 +- bindings/c/test/unit/unit_tests.cpp | 4 +- fdbclient/IClientApi.h | 12 ++++- fdbclient/MultiVersionTransaction.actor.cpp | 49 ++++++++++++++++----- fdbclient/MultiVersionTransaction.h | 33 ++++++++++++-- fdbclient/NativeAPI.actor.cpp | 15 +++++-- fdbclient/NativeAPI.actor.h | 5 ++- fdbclient/ThreadSafeTransaction.cpp | 19 ++++---- fdbclient/ThreadSafeTransaction.h | 18 ++++++-- 10 files changed, 133 insertions(+), 41 deletions(-) diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index bf6af3aab7..907f8058b6 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -364,6 +364,17 @@ extern "C" DLLEXPORT double fdb_database_get_main_thread_busyness(FDBDatabase* d return DB(d)->getMainThreadBusyness(); } +// Returns the protocol version reported by a quorum of coordinators +// If an expected version is non-zero, the future won't return until the protocol version is different than expected +extern "C" DLLEXPORT FDBFuture* fdb_database_get_server_protocol(FDBDatabase* db, uint64_t expected_version) { + Optional expected; + if (expected_version > 0) { + expected = ProtocolVersion(expected_version); + } + + return (FDBFuture*)(DB(db)->getServerProtocol(expected).extractPtr()); +} + extern "C" DLLEXPORT void fdb_transaction_destroy(FDBTransaction* tr) { try { TXN(tr)->delref(); @@ -583,10 +594,6 @@ extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_approximate_size(FDBTransact return (FDBFuture*)TXN(tr)->getApproximateSize().extractPtr(); } -extern "C" DLLEXPORT FDBFuture* fdb_get_server_protocol(const char* clusterFilePath) { - return (FDBFuture*)(API->getServerProtocol(clusterFilePath ? clusterFilePath : "").extractPtr()); -} - extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_versionstamp(FDBTransaction* tr) { return (FDBFuture*)(TXN(tr)->getVersionstamp().extractPtr()); } diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index 2086cbd775..4ea59ac11e 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -189,6 +189,8 @@ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_create_snapshot(FDBDatabase DLLEXPORT WARN_UNUSED_RESULT double fdb_database_get_main_thread_busyness(FDBDatabase* db); +DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_get_server_protocol(FDBDatabase* db, uint64_t expected_version); + DLLEXPORT void fdb_transaction_destroy(FDBTransaction* tr); DLLEXPORT void fdb_transaction_cancel(FDBTransaction* tr); @@ -281,8 +283,6 @@ DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_transaction_get_committed_version(F */ DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr); -DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_get_server_protocol(const char* clusterFilePath); - DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_versionstamp(FDBTransaction* tr); DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_on_error(FDBTransaction* tr, fdb_error_t error); diff --git a/bindings/c/test/unit/unit_tests.cpp b/bindings/c/test/unit/unit_tests.cpp index f3f97476c2..a87e483ef3 100644 --- a/bindings/c/test/unit/unit_tests.cpp +++ b/bindings/c/test/unit/unit_tests.cpp @@ -1515,7 +1515,7 @@ TEST_CASE("fdb_transaction_get_approximate_size") { TEST_CASE("fdb_get_server_protocol") { // We don't really have any expectations other than "don't crash" here - FDBFuture* protocolFuture = fdb_get_server_protocol(clusterFilePath.c_str()); + FDBFuture* protocolFuture = fdb_database_get_server_protocol(db, 0); uint64_t out; fdb_check(fdb_future_block_until_ready(protocolFuture)); @@ -1523,7 +1523,7 @@ TEST_CASE("fdb_get_server_protocol") { fdb_future_destroy(protocolFuture); // "Default" cluster file version - protocolFuture = fdb_get_server_protocol(nullptr); + protocolFuture = fdb_database_get_server_protocol(nullptr, 0x0FDB00A200090000LL); fdb_check(fdb_future_block_until_ready(protocolFuture)); fdb_check(fdb_future_get_uint64(protocolFuture, &out)); fdb_future_destroy(protocolFuture); diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h index 6f3ad07cd1..4496eff732 100644 --- a/fdbclient/IClientApi.h +++ b/fdbclient/IClientApi.h @@ -28,6 +28,7 @@ #include "flow/ThreadHelper.actor.h" +// An interface that represents a transaction created by a client class ITransaction { public: virtual ~ITransaction() {} @@ -90,6 +91,7 @@ public: virtual void delref() = 0; }; +// An interface that represents a connection to a cluster made by a client class IDatabase { public: virtual ~IDatabase() {} @@ -98,6 +100,11 @@ public: virtual void setOption(FDBDatabaseOptions::Option option, Optional value = Optional()) = 0; virtual double getMainThreadBusyness() = 0; + // Returns the protocol version reported by a quorum of coordinators + // If an expected version is given, the future won't return until the protocol version is different than expected + virtual ThreadFuture getServerProtocol( + Optional expectedVersion = Optional()) = 0; + virtual void addref() = 0; virtual void delref() = 0; @@ -110,13 +117,16 @@ public: virtual ThreadFuture createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0; }; +// An interface that presents the top-level FDB client API as exposed through the C bindings +// +// This interface and its associated objects are intended to live outside the network thread, so its asynchronous +// operations use ThreadFutures and implementations should be thread safe. class IClientApi { public: virtual ~IClientApi() {} virtual void selectApiVersion(int apiVersion) = 0; virtual const char* getClientVersion() = 0; - virtual ThreadFuture getServerProtocol(const char* clusterFilePath) = 0; virtual void setNetworkOption(FDBNetworkOptions::Option option, Optional value = Optional()) = 0; diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index ac1855c811..4b6ba0c27c 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -356,7 +356,32 @@ double DLDatabase::getMainThreadBusyness() { return 0; } +// Returns the protocol version reported by a quorum of coordinators +// If an expected version is given, the future won't return until the protocol version is different than expected +ThreadFuture DLDatabase::getServerProtocol(Optional expectedVersion) { + ASSERT(api->databaseGetServerProtocol != nullptr); + + uint64_t expected = + expectedVersion.map([](const ProtocolVersion& v) { return v.version(); }).orDefault(0); + FdbCApi::FDBFuture* f = api->databaseGetServerProtocol(db, expected); + return toThreadFuture(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { + uint64_t pv; + FdbCApi::fdb_error_t error = api->futureGetUInt64(f, &pv); + ASSERT(!error); + return ProtocolVersion(pv); + }); +} + // DLApi + +// Loads the specified function from a dynamic library +// +// fp - The function pointer where the loaded function will be stored +// lib - The dynamic library where the function is loaded from +// libPath - The path of the dynamic library (used for logging) +// functionName - The function to load +// requireFunction - Determines the behavior if the function is not present. If true, an error is thrown. If false, +// the function pointer will be set to nullptr. template void loadClientFunction(T* fp, void* lib, std::string libPath, const char* functionName, bool requireFunction = true) { *(void**)(fp) = loadFunction(lib, functionName); @@ -403,6 +428,8 @@ void DLApi::init() { fdbCPath, "fdb_database_get_main_thread_busyness", headerVersion >= 700); + loadClientFunction( + &api->databaseGetServerProtocol, lib, fdbCPath, "fdb_database_get_server_protocol", headerVersion >= 700); loadClientFunction(&api->databaseDestroy, lib, fdbCPath, "fdb_database_destroy"); loadClientFunction(&api->databaseRebootWorker, lib, fdbCPath, "fdb_database_reboot_worker", headerVersion >= 700); loadClientFunction(&api->databaseForceRecoveryWithDataLoss, @@ -452,7 +479,7 @@ void DLApi::init() { loadClientFunction( &api->futureGetInt64, lib, fdbCPath, headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version"); - loadClientFunction(&api->futureGetUInt64, lib, fdbCPath, "fdb_future_get_uint64"); + loadClientFunction(&api->futureGetUInt64, lib, fdbCPath, "fdb_future_get_uint64", headerVersion >= 700); loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error"); loadClientFunction(&api->futureGetKey, lib, fdbCPath, "fdb_future_get_key"); loadClientFunction(&api->futureGetValue, lib, fdbCPath, "fdb_future_get_value"); @@ -488,11 +515,6 @@ const char* DLApi::getClientVersion() { return api->getClientVersion(); } -ThreadFuture DLApi::getServerProtocol(const char* clusterFilePath) { - ASSERT(false); - return ThreadFuture(); -} - void DLApi::setNetworkOption(FDBNetworkOptions::Option option, Optional value) { throwIfError(api->setNetworkOption( option, value.present() ? value.get().begin() : nullptr, value.present() ? value.get().size() : 0)); @@ -856,7 +878,7 @@ MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api, std::string clusterFilePath, Reference db, bool openConnectors) - : dbState(new DatabaseState()) { + : dbState(new DatabaseState()), clusterFilePath(clusterFilePath) { dbState->db = db; dbState->dbVar->set(db); @@ -941,6 +963,15 @@ double MultiVersionDatabase::getMainThreadBusyness() { return 0; } +// Returns the protocol version reported by a quorum of coordinators +// If an expected version is given, the future won't return until the protocol version is different than expected +ThreadFuture MultiVersionDatabase::getServerProtocol(Optional expectedVersion) { + // TODO: send this out through the active database + return MultiVersionApi::api->getLocalClient() + ->api->createDatabase(clusterFilePath.c_str()) + ->getServerProtocol(expectedVersion); +} + void MultiVersionDatabase::Connector::connect() { addref(); onMainThreadVoid( @@ -1181,10 +1212,6 @@ const char* MultiVersionApi::getClientVersion() { return localClient->api->getClientVersion(); } -ThreadFuture MultiVersionApi::getServerProtocol(const char* clusterFilePath) { - return api->localClient->api->getServerProtocol(clusterFilePath); -} - void validateOption(Optional value, bool canBePresent, bool canBeAbsent, bool canBeEmpty = true) { ASSERT(canBePresent || canBeAbsent); diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index ea16f4f35e..badb848334 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -28,6 +28,8 @@ #include "flow/ThreadHelper.actor.h" +// FdbCApi is used as a wrapper around the FoundationDB C API that gets loaded from an external client library. +// All of the required functions loaded from that external library are stored in function pointers in this struct. struct FdbCApi : public ThreadSafeReferenceCounted { typedef struct future FDBFuture; typedef struct cluster FDBCluster; @@ -55,7 +57,6 @@ struct FdbCApi : public ThreadSafeReferenceCounted { // Network fdb_error_t (*selectApiVersion)(int runtimeVersion, int headerVersion); const char* (*getClientVersion)(); - FDBFuture* (*getServerProtocol)(const char* clusterFilePath); fdb_error_t (*setNetworkOption)(FDBNetworkOptions::Option option, uint8_t const* value, int valueLength); fdb_error_t (*setupNetwork)(); fdb_error_t (*runNetwork)(); @@ -81,6 +82,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { uint8_t const* snapshotCommmand, int snapshotCommandLength); double (*databaseGetMainThreadBusyness)(FDBDatabase* database); + FDBFuture* (*databaseGetServerProtocol)(FDBDatabase* database, uint64_t expectedVersion); // Transaction fdb_error_t (*transactionSetOption)(FDBTransaction* tr, @@ -185,6 +187,8 @@ struct FdbCApi : public ThreadSafeReferenceCounted { fdb_error_t (*futureGetCluster)(FDBFuture* f, FDBCluster** outCluster); }; +// An implementation of ITransaction that wraps a transaction object created on an externally loaded client library. +// All API calls to that transaction are routed through the external library. class DLTransaction : public ITransaction, ThreadSafeReferenceCounted { public: DLTransaction(Reference api, FdbCApi::FDBTransaction* tr) : api(api), tr(tr) {} @@ -249,6 +253,8 @@ private: FdbCApi::FDBTransaction* const tr; }; +// An implementation of IDatabase that wraps a database object created on an externally loaded client library. +// All API calls to that database are routed through the external library. class DLDatabase : public IDatabase, ThreadSafeReferenceCounted { public: DLDatabase(Reference api, FdbCApi::FDBDatabase* db) : api(api), db(db), ready(Void()) {} @@ -265,6 +271,11 @@ public: void setOption(FDBDatabaseOptions::Option option, Optional value = Optional()) override; double getMainThreadBusyness() override; + // Returns the protocol version reported by a quorum of coordinators + // If an expected version is given, the future won't return until the protocol version is different than expected + ThreadFuture getServerProtocol( + Optional expectedVersion = Optional()) override; + void addref() override { ThreadSafeReferenceCounted::addref(); } void delref() override { ThreadSafeReferenceCounted::delref(); } @@ -279,13 +290,14 @@ private: ThreadFuture ready; }; +// An implementation of IClientApi that re-issues API calls to the C API of an externally loaded client library. +// The DL prefix stands for "dynamic library". class DLApi : public IClientApi { public: DLApi(std::string fdbCPath, bool unlinkOnLoad = false); void selectApiVersion(int apiVersion) override; const char* getClientVersion() override; - ThreadFuture getServerProtocol(const char* clusterFilePath) override; void setNetworkOption(FDBNetworkOptions::Option option, Optional value = Optional()) override; void setupNetwork() override; @@ -312,6 +324,9 @@ private: class MultiVersionDatabase; +// An implementation of ITransaction that wraps a transaction created either locally or through a dynamically loaded +// external client. When needed (e.g on cluster version change), the MultiVersionTransaction can automatically replace +// its wrapped transaction with one from another client. class MultiVersionTransaction : public ITransaction, ThreadSafeReferenceCounted { public: MultiVersionTransaction(Reference db, @@ -413,6 +428,9 @@ struct ClientInfo : ClientDesc, ThreadSafeReferenceCounted { class MultiVersionApi; +// An implementation of IDatabase that wraps a database created either locally or through a dynamically loaded +// external client. The MultiVersionDatabase monitors the protocol version of the cluster and automatically +// replaces the wrapped database when the protocol version changes. class MultiVersionDatabase final : public IDatabase, ThreadSafeReferenceCounted { public: MultiVersionDatabase(MultiVersionApi* api, @@ -426,6 +444,11 @@ public: void setOption(FDBDatabaseOptions::Option option, Optional value = Optional()) override; double getMainThreadBusyness() override; + // Returns the protocol version reported by a quorum of coordinators + // If an expected version is given, the future won't return until the protocol version is different than expected + ThreadFuture getServerProtocol( + Optional expectedVersion = Optional()) override; + void addref() override { ThreadSafeReferenceCounted::addref(); } void delref() override { ThreadSafeReferenceCounted::delref(); } @@ -487,15 +510,19 @@ private: Mutex optionLock; }; + std::string clusterFilePath; const Reference dbState; friend class MultiVersionTransaction; }; +// An implementation of IClientApi that can choose between multiple different client implementations either provided +// locally within the primary loaded fdb_c client or through any number of dynamically loaded clients. +// +// This functionality is used to provide support for multiple protocol versions simultaneously. class MultiVersionApi : public IClientApi { public: void selectApiVersion(int apiVersion) override; const char* getClientVersion() override; - ThreadFuture getServerProtocol(const char* clusterFilePath) override; void setNetworkOption(FDBNetworkOptions::Option option, Optional value = Optional()) override; void setupNetwork() override; diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 9f6784e279..6615e973dd 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -4900,9 +4900,18 @@ ACTOR Future coordinatorProtocolsFetcher(Reference getCoordinatorProtocols(Reference f) { - ProtocolVersion protocolVersion = wait(coordinatorProtocolsFetcher(f)); - return protocolVersion.version(); +// Returns the protocol version reported by a quorum of coordinators +// If an expected version is given, the future won't return until the protocol version is different than expected +ACTOR Future getClusterProtocol(Reference f, + Optional expectedVersion) { + loop { + ProtocolVersion protocolVersion = wait(coordinatorProtocolsFetcher(f)); + if (!expectedVersion.present() || protocolVersion != expectedVersion.get()) { + return protocolVersion; + } else { + wait(delay(2.0)); // TODO: this is temporary, so not making into a knob yet + } + } } uint32_t Transaction::getSize() { diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index ac31967d83..51411ae0a2 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -400,7 +400,10 @@ ACTOR Future snapCreate(Database cx, Standalone snapCmd, UID sn // Checks with Data Distributor that it is safe to mark all servers in exclusions as failed ACTOR Future checkSafeExclusions(Database cx, vector exclusions); -ACTOR Future getCoordinatorProtocols(Reference f); +// Returns the protocol version reported by a quorum of coordinators +// If an expected version is given, the future won't return until the protocol version is different than expected +ACTOR Future getClusterProtocol(Reference f, + Optional expectedVersion); inline uint64_t getWriteOperationCost(uint64_t bytes) { return bytes / std::max(1, CLIENT_KNOBS->WRITE_COST_BYTE_FACTOR) + 1; diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp index 0e0877f9af..c5bf2dce87 100644 --- a/fdbclient/ThreadSafeTransaction.cpp +++ b/fdbclient/ThreadSafeTransaction.cpp @@ -97,6 +97,15 @@ double ThreadSafeDatabase::getMainThreadBusyness() { return g_network->networkInfo.metrics.networkBusyness; } +// Returns the protocol version reported by a quorum of coordinators +// If an expected version is given, the future won't return until the protocol version is different than expected +ThreadFuture ThreadSafeDatabase::getServerProtocol(Optional expectedVersion) { + DatabaseContext* db = this->db; + return onMainThread([db, expectedVersion]() -> Future { + return getClusterProtocol(db->getConnectionFile(), expectedVersion); + }); +} + ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) { ClusterConnectionFile* connFile = new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFilename).first); @@ -407,16 +416,6 @@ const char* ThreadSafeApi::getClientVersion() { return clientVersion.c_str(); } -// Wait until a quorum of coordinators with the same protocol version are available, and then return that protocol -// version. -ThreadFuture ThreadSafeApi::getServerProtocol(const char* clusterFilePath) { - return onMainThread([clusterFilePath = std::string(clusterFilePath)]() -> Future { - auto [clusterFile, isDefault] = ClusterConnectionFile::lookupClusterFileName(clusterFilePath); - Reference f = Reference(new ClusterConnectionFile(clusterFile)); - return getCoordinatorProtocols(f); - }); -} - void ThreadSafeApi::setNetworkOption(FDBNetworkOptions::Option option, Optional value) { if (option == FDBNetworkOptions::EXTERNAL_CLIENT_TRANSPORT_ID) { if (value.present()) { diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h index a62e503c11..e6360c2a6d 100644 --- a/fdbclient/ThreadSafeTransaction.h +++ b/fdbclient/ThreadSafeTransaction.h @@ -27,6 +27,8 @@ #include "fdbclient/ClusterInterface.h" #include "fdbclient/IClientApi.h" +// An implementation of IDatabase that serializes operations onto the network thread and interacts with the lower-level +// client APIs exposed by NativeAPI and ReadYourWrites. class ThreadSafeDatabase : public IDatabase, public ThreadSafeReferenceCounted { public: ~ThreadSafeDatabase() override; @@ -37,9 +39,14 @@ public: void setOption(FDBDatabaseOptions::Option option, Optional value = Optional()) override; double getMainThreadBusyness() override; - ThreadFuture - onConnected(); // Returns after a majority of coordination servers are available and have reported a leader. The - // cluster file therefore is valid, but the database might be unavailable. + // Returns the protocol version reported by a quorum of coordinators + // If an expected version is given, the future won't return until the protocol version is different than expected + ThreadFuture getServerProtocol( + Optional expectedVersion = Optional()) override; + + // Returns after a majority of coordination servers are available and have reported a leader. The + // cluster file therefore is valid, but the database might be unavailable. + ThreadFuture onConnected(); void addref() override { ThreadSafeReferenceCounted::addref(); } void delref() override { ThreadSafeReferenceCounted::delref(); } @@ -58,6 +65,8 @@ public: // Internal use only DatabaseContext* unsafeGetPtr() const { return db; } }; +// An implementation of ITransaction that serializes operations onto the network thread and interacts with the +// lower-level client APIs exposed by NativeAPI and ReadYourWrites. class ThreadSafeTransaction : public ITransaction, ThreadSafeReferenceCounted, NonCopyable { public: explicit ThreadSafeTransaction(DatabaseContext* cx); @@ -135,11 +144,12 @@ private: ReadYourWritesTransaction* tr; }; +// An implementation of IClientApi that serializes operations onto the network thread and interacts with the lower-level +// client APIs exposed by NativeAPI and ReadYourWrites. class ThreadSafeApi : public IClientApi, ThreadSafeReferenceCounted { public: void selectApiVersion(int apiVersion) override; const char* getClientVersion() override; - ThreadFuture getServerProtocol(const char* clusterFilePath) override; void setNetworkOption(FDBNetworkOptions::Option option, Optional value = Optional()) override; void setupNetwork() override; From bc8568d4bbdccf6a90200965c04dfe2ecd6641ae Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 14 Apr 2021 12:58:59 -0700 Subject: [PATCH 53/60] Use the correct pointer in the unit test for fdb_database_get_server_protocol --- bindings/c/test/unit/unit_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/c/test/unit/unit_tests.cpp b/bindings/c/test/unit/unit_tests.cpp index a87e483ef3..c5c40b88c1 100644 --- a/bindings/c/test/unit/unit_tests.cpp +++ b/bindings/c/test/unit/unit_tests.cpp @@ -1523,7 +1523,7 @@ TEST_CASE("fdb_get_server_protocol") { fdb_future_destroy(protocolFuture); // "Default" cluster file version - protocolFuture = fdb_database_get_server_protocol(nullptr, 0x0FDB00A200090000LL); + protocolFuture = fdb_database_get_server_protocol(db, 0x0FDB00A200090000LL); fdb_check(fdb_future_block_until_ready(protocolFuture)); fdb_check(fdb_future_get_uint64(protocolFuture, &out)); fdb_future_destroy(protocolFuture); From d3b6a543ab29bbdd9becaf0bf205b1b6c6ac1cb9 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 14 Apr 2021 13:23:06 -0700 Subject: [PATCH 54/60] Update comment in unit test --- bindings/c/test/unit/unit_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/c/test/unit/unit_tests.cpp b/bindings/c/test/unit/unit_tests.cpp index c5c40b88c1..64898f6ede 100644 --- a/bindings/c/test/unit/unit_tests.cpp +++ b/bindings/c/test/unit/unit_tests.cpp @@ -1522,7 +1522,7 @@ TEST_CASE("fdb_get_server_protocol") { fdb_check(fdb_future_get_uint64(protocolFuture, &out)); fdb_future_destroy(protocolFuture); - // "Default" cluster file version + // Passing in an expected version that's different than the cluster version protocolFuture = fdb_database_get_server_protocol(db, 0x0FDB00A200090000LL); fdb_check(fdb_future_block_until_ready(protocolFuture)); fdb_check(fdb_future_get_uint64(protocolFuture, &out)); From 028c02c7b0fa16c8ddadb664b67b5a5df6f3eae8 Mon Sep 17 00:00:00 2001 From: Vishesh Yadav Date: Thu, 15 Apr 2021 22:26:32 +0000 Subject: [PATCH 55/60] doc: Link FDB Commit Process doc in CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e599780e37..525e80a9d9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,7 @@ Members of the Apple FoundationDB team are part of the core committers helping r ## Contributing ### Opening a Pull Request -We love pull requests! For minor changes, feel free to open up a PR directly. For larger feature development and any changes that may require community discussion, we ask that you discuss your ideas on the [community forums](https://forums.foundationdb.org) prior to opening a PR, and then reference that thread within your PR comment. +We love pull requests! For minor changes, feel free to open up a PR directly. For larger feature development and any changes that may require community discussion, we ask that you discuss your ideas on the [community forums](https://forums.foundationdb.org) prior to opening a PR, and then reference that thread within your PR comment. Please refer to [FoundationDB Commit Process](https://github.com/apple/foundationdb/wiki/FoundationDB-Commit-Process) for more detailed guidelines. CI will be run automatically for core committers, and for community PRs it will be initiated by the request of a core committer. Tests can also be run locally via `ctest`, and core committers can run additional validation on pull requests prior to merging them. From e7d4a452e46e2c41b64f7ac978be4a5dcf6ee1fe Mon Sep 17 00:00:00 2001 From: Vishesh Yadav Date: Tue, 13 Apr 2021 18:09:48 -0700 Subject: [PATCH 57/60] docker: Load custom bashrc if available This will check if there is a `.bashrc.local` avialable in the synced directory, and load it. - This is useful so that any changes usrs make to our bashrc in Okteto containers, doesn't get lost between re-deployement of containers. - Can also used to automate setting up environment, e.g. copy various dotfiles etc from the synced directory to $HOME folder during first run. --- build/docker/centos6/devel/Dockerfile | 7 ++++++- build/docker/centos7/devel/Dockerfile | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/build/docker/centos6/devel/Dockerfile b/build/docker/centos6/devel/Dockerfile index 82c99d4464..c5c9db2914 100644 --- a/build/docker/centos6/devel/Dockerfile +++ b/build/docker/centos6/devel/Dockerfile @@ -76,4 +76,9 @@ RUN rm -f /root/anaconda-ks.cfg && \ ' j start --tarball $(find ${HOME}/build_output/packages -name correctness\*.tar.gz) "${@}"' \ '}' \ '' \ - >> .bashrc \ No newline at end of file + 'USER_BASHRC="$HOME/src/.bashrc.local"' \ + 'if test -f "$USER_BASHRC"; then' \ + ' source $USER_BASHRC' \ + 'fi' \ + '' \ + >> .bashrc diff --git a/build/docker/centos7/devel/Dockerfile b/build/docker/centos7/devel/Dockerfile index ea60da54e7..98f1923c17 100644 --- a/build/docker/centos7/devel/Dockerfile +++ b/build/docker/centos7/devel/Dockerfile @@ -104,5 +104,10 @@ RUN rm -f /root/anaconda-ks.cfg && \ ' j start --tarball $(find ${HOME}/build_output/packages -name correctness\*.tar.gz) "${@}"' \ '}' \ '' \ + 'USER_BASHRC="$HOME/src/.bashrc.local"' \ + 'if test -f "$USER_BASHRC"; then' \ + ' source $USER_BASHRC' \ + 'fi' \ + '' \ 'bash ${HOME}/docker_proxy.sh' \ - >> .bashrc \ No newline at end of file + >> .bashrc From 4a1a55f27063a16ea7f793b9da55c6fee6228117 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Fri, 16 Apr 2021 13:48:44 -0700 Subject: [PATCH 58/60] Remove fdb_get_server_protocol from the Python bindings. This C function this was using recently moved and changed signature, so it no longer works in Python. --- bindings/c/test/unit/unit_tests.cpp | 2 +- bindings/python/fdb/__init__.py | 1 - bindings/python/fdb/impl.py | 10 ---------- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/bindings/c/test/unit/unit_tests.cpp b/bindings/c/test/unit/unit_tests.cpp index 64898f6ede..54f763fb5c 100644 --- a/bindings/c/test/unit/unit_tests.cpp +++ b/bindings/c/test/unit/unit_tests.cpp @@ -1513,7 +1513,7 @@ TEST_CASE("fdb_transaction_get_approximate_size") { } } -TEST_CASE("fdb_get_server_protocol") { +TEST_CASE("fdb_database_get_server_protocol") { // We don't really have any expectations other than "don't crash" here FDBFuture* protocolFuture = fdb_database_get_server_protocol(db, 0); uint64_t out; diff --git a/bindings/python/fdb/__init__.py b/bindings/python/fdb/__init__.py index 17f697797d..c969b6c70c 100644 --- a/bindings/python/fdb/__init__.py +++ b/bindings/python/fdb/__init__.py @@ -95,7 +95,6 @@ def api_version(ver): 'transactional', 'options', 'StreamingMode', - 'get_server_protocol' ) _add_symbols(fdb.impl, list) diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index 6e7803777a..e8cc2a79b8 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -1531,9 +1531,6 @@ def init_c_api(): _capi.fdb_transaction_get_approximate_size.argtypes = [ctypes.c_void_p] _capi.fdb_transaction_get_approximate_size.restype = ctypes.c_void_p - _capi.fdb_get_server_protocol.argtypes = [ctypes.c_char_p] - _capi.fdb_get_server_protocol.restype = ctypes.c_void_p - _capi.fdb_transaction_get_versionstamp.argtypes = [ctypes.c_void_p] _capi.fdb_transaction_get_versionstamp.restype = ctypes.c_void_p @@ -1733,13 +1730,6 @@ open_databases = {} cacheLock = threading.Lock() -def get_server_protocol(clusterFilePath=None): - with _network_thread_reentrant_lock: - if not _network_thread: - init() - - return FutureUInt64(_capi.fdb_get_server_protocol(optionalParamToBytes(clusterFilePath)[0])) - def open(cluster_file=None, event_model=None): """Opens the given database (or the default database of the cluster indicated by the fdb.cluster file in a platform-specific location, if no cluster_file From bb5539bb70a27d38cdeeb8c1fab362fe1d8317fa Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Fri, 16 Apr 2021 13:47:41 -0700 Subject: [PATCH 59/60] Initialize version field --- fdbclient/SpecialKeySpace.actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 2bbafbd451..9ac802a9ca 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -1430,7 +1430,7 @@ ACTOR Future> globalConfigCommitActor(GlobalConfigImpl* gl } } - VersionHistory vh; + VersionHistory vh{ 0 }; // Transform writes from the special-key-space (\xff\xff/global_config/) to // the system key space (\xff/globalConfig/), and writes mutations to From db610355cf4f59f8b097ff7a61964395dcdfb05d Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Fri, 16 Apr 2021 14:19:37 -0700 Subject: [PATCH 60/60] Keep simulated disk write delay high until speedUp is set. --- fdbrpc/AsyncFileNonDurable.actor.h | 2 +- flow/Knobs.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h index fe3d3a4137..848d755fb1 100644 --- a/fdbrpc/AsyncFileNonDurable.actor.h +++ b/fdbrpc/AsyncFileNonDurable.actor.h @@ -197,7 +197,7 @@ private: this->file = file; this->filename = filename; this->diskParameters = diskParameters; - maxWriteDelay = deterministicRandom()->random01() * FLOW_KNOBS->NON_DURABLE_MAX_WRITE_DELAY; + maxWriteDelay = FLOW_KNOBS->NON_DURABLE_MAX_WRITE_DELAY; hasBeenSynced = false; killMode = (KillMode)deterministicRandom()->randomInt(1, 3); diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 6dc77e2fb2..4a3eb4e2d7 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -135,7 +135,7 @@ void FlowKnobs::initialize(bool randomize, bool isSimulated) { init( DISABLE_POSIX_KERNEL_AIO, 0 ); //AsyncFileNonDurable - init( NON_DURABLE_MAX_WRITE_DELAY, 0.0001 ); if( randomize && BUGGIFY ) NON_DURABLE_MAX_WRITE_DELAY = 5.0; + init( NON_DURABLE_MAX_WRITE_DELAY, 5.0 ); init( MAX_PRIOR_MODIFICATION_DELAY, 1.0 ); if( randomize && BUGGIFY ) MAX_PRIOR_MODIFICATION_DELAY = 10.0; //GenericActors