From 0131f0dc48a12f2804066fbbb027f2f64d818fa6 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Sun, 12 Jul 2020 14:42:43 -0700 Subject: [PATCH] Make simulation able to load and run .toml files --- fdbserver/fdbserver.actor.cpp | 6 +- fdbserver/tester.actor.cpp | 113 ++++++++++++++++++++++++++++++---- 2 files changed, 104 insertions(+), 15 deletions(-) diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 6becec0224..73614bd7fd 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -697,10 +697,12 @@ Optional checkBuggifyOverride(const char *testFile) { std::string value = removeWhitespace(line.substr(found + 1)); if (attrib == "buggify") { - if( !strcmp( value.c_str(), "on" ) ) { + // Testspec uses `on` or `off` (without quotes). + // TOML uses literal `true` and `false`. + if( !strcmp( value.c_str(), "on" ) || !strcmp( value.c_str(), "true" ) ) { ifs.close(); return true; - } else if( !strcmp( value.c_str(), "off" ) ) { + } else if( !strcmp( value.c_str(), "off" ) || !strcmp( value.c_str(), "false" )) { ifs.close(); return false; } else { diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index a1c7945c55..311d5d3012 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "flow/ActorCollection.h" #include "fdbrpc/sim_validation.h" @@ -875,29 +876,29 @@ std::vector readTOMLTests( ifstream& ifs ) { return {}; } -std::map> testSpecGlobalKeys = { +std::map> testSpecGlobalKeys = { // These are read by SimulatedCluster and used before testers exist. Thus, they must // be recognized and accepted, but there's no point in placing them into a testSpec. - {"extraDB", [](const std::string& value, TestSpec* spec) { + {"extraDB", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedExtraDB", ""); }}, - {"configureLocked", [](const std::string& value, TestSpec* spec) { + {"configureLocked", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedConfigureLocked", ""); }}, - {"minimumReplication", [](const std::string& value, TestSpec* spec) { + {"minimumReplication", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedMinimumReplication", ""); }}, - {"minimumRegions", [](const std::string& value, TestSpec* spec) { + {"minimumRegions", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedMinimumRegions", ""); }}, - {"buggify", [](const std::string& value, TestSpec* spec) { + {"buggify", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedBuggify", ""); }}, // The test harness handles NewSeverity events specially. - {"StderrSeverity", [](const std::string& value, TestSpec* spec) { + {"StderrSeverity", [](const std::string& value) { TraceEvent("StderrSeverity").detail("NewSeverity", value); }}, - {"ClientInfoLogging", [](const std::string& value, TestSpec* spec) { + {"ClientInfoLogging", [](const std::string& value) { if (value == "false") { setNetworkOption(FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING); } @@ -907,6 +908,10 @@ std::map> testSp }; std::map> testSpecTestKeys = { + { "testTitle", [](const std::string& value, TestSpec* spec) { + spec->title = value; + TraceEvent("TestParserTest").detail("ParsedTest", spec->title ); + }}, { "timeout", [](const std::string& value, TestSpec* spec) { sscanf( value.c_str(), "%d", &(spec->timeout) ); ASSERT( spec->timeout > 0 ); @@ -1034,14 +1039,13 @@ vector readTests( ifstream& ifs ) { spec = TestSpec(); } - spec.title = StringRef( value ); - TraceEvent("TestParserTest").detail("ParsedTest", spec.title ); + testSpecTestKeys[attrib](value, &spec); } else if ( testSpecTestKeys.find(attrib) != testSpecTestKeys.end() ) { if (parsingWorkloads) TraceEvent(SevError, "TestSpecTestParamInWorkload").detail("Attrib", attrib).detail("Value", value); testSpecTestKeys[attrib](value, &spec); } else if ( testSpecGlobalKeys.find(attrib) != testSpecGlobalKeys.end() ) { if (!beforeFirstTest) TraceEvent(SevError, "TestSpecGlobalParamInTest").detail("Attrib", attrib).detail("Value", value); - testSpecGlobalKeys[attrib](value, &spec); + testSpecGlobalKeys[attrib](value); } else { if( attrib == "testName" ) { @@ -1067,6 +1071,85 @@ vector readTests( ifstream& ifs ) { return result; } +template +std::string toml_to_string(const T& value) { + // TOML formatting converts numbers to strings exactly how they're in the file + // and thus, is equivalent to testspec. However, strings are quoted, so we + // must remove the quotes. + if (value.type() == toml::value_t::string) { + const std::string& formatted = toml::format(value); + return formatted.substr(1, formatted.size()-2); + } else { + return toml::format(value); + } +} + + +std::vector readTOMLTests_( std::string fileName ) { + TestSpec spec; + Standalone< VectorRef< KeyValueRef > > workloadOptions; + std::vector result; + + const toml::value& conf = toml::parse(fileName); + + // Handle all global settings + for (const auto& [k, v] : conf.as_table()) { + if (k == "test") { + continue; + } + if (testSpecGlobalKeys.find(k) != testSpecGlobalKeys.end()) { + testSpecGlobalKeys[k](toml_to_string(v)); + } + } + + // Then parse each test + const toml::array& tests = toml::find(conf, "test").as_array(); + for (const toml::value& test : tests) { + spec = TestSpec(); + + // First handle all test-level settings + for (const auto& [k, v] : test.as_table()) { + if (k == "workload") { + continue; + } + if (testSpecTestKeys.find(k) != testSpecTestKeys.end()) { + testSpecTestKeys[k](toml_to_string(v), &spec); + } + } + + // And then copy the workload attributes to spec.options + const toml::array& workloads = toml::find(test, "workload").as_array(); + for (const toml::value& workload : workloads) { + workloadOptions = Standalone< VectorRef< KeyValueRef > >(); + TraceEvent("TestParserFlush").detail("Reason", "new (compound) test"); + for (const auto& [attrib, v] : workload.as_table()) { + const std::string& value = toml_to_string(v); + workloadOptions.push_back_deep( workloadOptions.arena(), + KeyValueRef( StringRef( attrib ), StringRef( value ) ) ); + TraceEvent("TestParserOption").detail("ParsedKey", attrib).detail("ParsedValue", value); + } + spec.options.push_back_deep( spec.options.arena(), workloadOptions ); + } + + result.push_back(spec); + } + + return result; +} + +// A hack to catch and log std::exception, because TOML11 has very useful +// error messages, but the actor framework can't handle std::exception. +std::vector readTOMLTests( std::string fileName ) { + try { + return readTOMLTests_( fileName ); + } catch (std::exception &e) { + std::cerr << e.what() << std::endl; + TraceEvent("TOMLParseError").detail("Error", printable(e.what())); + // TODO: replace with toml_parse_error(); + throw unknown_error(); + } +} + ACTOR Future monitorServerDBInfo(Reference>> ccInterface, LocalityData locality, Reference> dbInfo) { @@ -1259,11 +1342,15 @@ ACTOR Future runTests( Reference connFile, test_typ ifs.open( fileName.c_str(), ifstream::in ); if( !ifs.good() ) { TraceEvent(SevError, "TestHarnessFail").detail("Reason", "file open failed").detail("File", fileName.c_str()); - fprintf(stderr, "ERROR: Could not open test spec file `%s'\n", fileName.c_str()); + fprintf(stderr, "ERROR: Could not open file `%s'\n", fileName.c_str()); return Void(); } enableClientInfoLogging(); // Enable Client Info logging by default for tester - testSpecs = readTests( ifs ); + if ( boost::algorithm::ends_with(fileName, ".txt") ) { + testSpecs = readTests( ifs ); + } else if ( boost::algorithm::ends_with(fileName, ".toml") ) { + testSpecs = readTOMLTests( fileName ); + } ifs.close(); }