From b70ff34a663c1985e0bef98f8fb0f7ecd133b39d Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Tue, 25 Apr 2023 10:48:54 -0700 Subject: [PATCH] Move custom shard test setup to a separate function. Add JSON utf-8 escaped bytes to fdbcli token parsing. --- fdbcli/fdbcli.actor.cpp | 22 +++ .../fdbserver/workloads/workloads.actor.h | 4 + fdbserver/tester.actor.cpp | 110 +-------------- fdbserver/workloads/workloads.actor.cpp | 133 ++++++++++++++++++ 4 files changed, 163 insertions(+), 106 deletions(-) create mode 100644 fdbserver/workloads/workloads.actor.cpp diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index cd79c56e28..57dee8f780 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -406,6 +406,28 @@ static std::vector> parseLine(std::string& line, bool& er case ';': line.erase(i, 1); break; + // Handle \uNNNN utf-8 characters from JSON strings but only as a single byte + // Return an error for a sequence out of range for a single byte + case 'u': { + if (i + 6 > line.length()) { + err = true; + ret.push_back(std::move(buf)); + return ret; + } + char* pEnd; + save = line[i + 6]; + line[i + 6] = 0; + unsigned long val = strtoul(line.data() + i + 2, &pEnd, 16); + ent = char(val); + if (*pEnd || val > std::numeric_limits::max()) { + err = true; + ret.push_back(std::move(buf)); + return ret; + } + line[i + 6] = save; + line.replace(i, 6, 1, ent); + break; + } case 'x': if (i + 4 > line.length()) { err = true; diff --git a/fdbserver/include/fdbserver/workloads/workloads.actor.h b/fdbserver/include/fdbserver/workloads/workloads.actor.h index 9b252dfaf4..330cabf9a8 100644 --- a/fdbserver/include/fdbserver/workloads/workloads.actor.h +++ b/fdbserver/include/fdbserver/workloads/workloads.actor.h @@ -389,6 +389,10 @@ Future testExpectedError(Future test, std::string getTestEncryptionFileName(); +// This should become a BehaviorInjectionWorkload or perhaps ConfigInjectionWorkload which should be a new class that +// should represent non-failure behaviors that can be randomly injected into any test run. +ACTOR Future customShardConfigWorkload(Database cx); + #include "flow/unactorcompiler.h" #endif diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index 246d93488a..2cd18834ac 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -2054,113 +2054,11 @@ ACTOR Future runTests(ReferencestoragePolicy && g_simulator->tLogPolicy); ASSERT(!g_simulator->hasSatelliteReplication || g_simulator->satelliteTLogPolicy); - // This block will randomly add custom configured key ranges to the test. - // TODO: Move this to a workload which is randomly added to tests similar to FailureInjectionWorkload classes - // but the injected behavior should not be considered a failure for workload selection purposes. + // Randomly inject custom shard configuration + // TOOO: Move this to a workload representing non-failure behaviors which can be randomly added to any test + // run. if (deterministicRandom()->random01() < 0.25) { - state ReadYourWritesTransaction tr(cx); - state bool verbose = (KEYBACKEDTYPES_DEBUG != 0); - - // This has to be optional because state vars need default constructors and VersionOptions types can't be - // default constructed and RangeConfigMap uses ObjectWriter which needs VersionOptions. - state Optional rangeConfig = DDConfiguration().userRangeConfig(); - - loop { - try { - tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - - // Map logic should work with or without allKeys endpoints initialized - if (deterministicRandom()->coinflip()) { - TraceEvent("KeyRangeConfigSetDefault").log(); - CODE_PROBE(true, "Set default shard range"); - wait(rangeConfig->updateRange(&tr, allKeys.begin, allKeys.end, DDRangeConfig())); - } - - if (deterministicRandom()->coinflip()) { - CODE_PROBE(true, "Set range config test cases"); - TraceEvent("KeyRangeConfigSetTestRanges").log(); - - wait(rangeConfig->updateRange(&tr, "\xff\x03"_sr, "\xff\x04"_sr, DDRangeConfig(3))); - wait(rangeConfig->updateRange(&tr, "\xff\x06"_sr, "\xff\x07"_sr, DDRangeConfig(6))); - wait(rangeConfig->updateRange(&tr, "\xff\x04"_sr, "\xff\x05"_sr, DDRangeConfig(4))); - - wait(rangeConfig->updateRange(&tr, "\xff\x03k"_sr, "\xff\x03z"_sr, DDRangeConfig(3))); - wait(rangeConfig->updateRange(&tr, "\xff\x04t"_sr, "\xff\x05h"_sr, DDRangeConfig(3, 1), true)); - wait(rangeConfig->updateRange(&tr, "\xff\x06u"_sr, "\xff\x07m"_sr, DDRangeConfig({}, 2))); - - wait(rangeConfig->updateRange(&tr, "a"_sr, "b"_sr, DDRangeConfig(3, 20), true)); - wait(rangeConfig->updateRange(&tr, "a"_sr, "a10"_sr, DDRangeConfig(3, 20), true)); - - // Key, entryRequiredInDB, expectedRangeConfig - state std::vector> rangeTests = { - { "\x01"_sr, false, DDRangeConfig() }, - { "\xff\x10"_sr, false, DDRangeConfig() }, - { "\xff\x03x"_sr, true, DDRangeConfig(3) }, - { "\xff\x04z"_sr, true, DDRangeConfig(3, 1) }, - { "\xff\x05"_sr, true, DDRangeConfig(3, 1) }, - { "\xff\x06"_sr, true, DDRangeConfig(6) }, - { "\xff\x06u"_sr, true, DDRangeConfig(6, 2) }, - { "\xff\x06v"_sr, true, DDRangeConfig(6, 2) }, - { "\xff\x07m"_sr, false, DDRangeConfig() }, - { "\xff\x07k"_sr, true, DDRangeConfig({}, 2) }, - { "a"_sr, true, DDRangeConfig(3, 20) }, - { "a10"_sr, true, DDRangeConfig(3, 20) }, - { "a11"_sr, true, DDRangeConfig(3, 20) }, - { "b"_sr, false, DDRangeConfig() }, - { "b1"_sr, false, DDRangeConfig() } - }; - - state Reference snapshot = - wait(rangeConfig->getSnapshot(&tr, allKeys.begin, allKeys.end)); - - if (verbose) { - fmt::print("DD User Range Config:\n{}\n", - json_spirit::write_string(DDConfiguration::toJSON(*snapshot, true), - json_spirit::pretty_print)); - } - - state int i; - for (i = 0; i < rangeTests.size(); ++i) { - state Key query = std::get<0>(rangeTests[i]); - Optional verify = - wait(rangeConfig->getRangeForKey(&tr, query)); - - if (verbose) { - if (verify.present()) { - fmt::print("'{}' is in '{}' to '{}' with config {}\n", - query.printable(), - verify->range.begin, - verify->range.end, - verify->value.toString()); - } else { - fmt::print("'{}' is not in a range in the config\n", query.printable()); - } - } - - // Range value is either present or not required by test - ASSERT(verify.present() || !std::get<1>(rangeTests[i])); - DDRangeConfig rc = std::get<2>(rangeTests[i]); - ASSERT(!verify.present() || verify->value == rc); - - auto snapshotRange = snapshot->rangeContaining(query); - ASSERT(snapshotRange.value() == rc); - // The snapshot has all ranges covered but the db may not - if (verify.present()) { - ASSERT(snapshotRange.range().begin == verify->range.begin); - ASSERT(snapshotRange.range().end == verify->range.end); - } - } - } - - wait(tr.commit()); - TraceEvent("KeyRangeConfigCommitted").log(); - break; - } catch (Error& e) { - TraceEvent("KeyRangeConfigCommitError").error(e); - wait(tr.onError(e)); - } - } + wait(customShardConfigWorkload(cx)); } } diff --git a/fdbserver/workloads/workloads.actor.cpp b/fdbserver/workloads/workloads.actor.cpp new file mode 100644 index 0000000000..db8eb76167 --- /dev/null +++ b/fdbserver/workloads/workloads.actor.cpp @@ -0,0 +1,133 @@ +/* + * workloads.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2023 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "flow/flow.h" +#include "fdbclient/FDBTypes.h" +#include "fdbclient/DataDistributionConfig.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +ACTOR Future customShardConfigWorkload(Database cx) { + state ReadYourWritesTransaction tr(cx); + state bool verbose = (KEYBACKEDTYPES_DEBUG != 0); + + // This has to be optional because state vars need default constructors and VersionOptions types can't be + // default constructed and RangeConfigMap uses ObjectWriter which needs VersionOptions. + state Optional rangeConfig = DDConfiguration().userRangeConfig(); + + loop { + try { + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + + // Map logic should work with or without allKeys endpoints initialized + if (deterministicRandom()->coinflip()) { + TraceEvent("KeyRangeConfigSetDefault").log(); + CODE_PROBE(true, "Set default shard range"); + wait(rangeConfig->updateRange(&tr, allKeys.begin, allKeys.end, DDRangeConfig())); + } + + if (deterministicRandom()->coinflip()) { + CODE_PROBE(true, "Set range config test cases"); + TraceEvent("KeyRangeConfigSetTestRanges").log(); + + wait(rangeConfig->updateRange(&tr, "\xff\x03"_sr, "\xff\x04"_sr, DDRangeConfig(3))); + wait(rangeConfig->updateRange(&tr, "\xff\x06"_sr, "\xff\x07"_sr, DDRangeConfig(6))); + wait(rangeConfig->updateRange(&tr, "\xff\x04"_sr, "\xff\x05"_sr, DDRangeConfig(4))); + + wait(rangeConfig->updateRange(&tr, "\xff\x03k"_sr, "\xff\x03z"_sr, DDRangeConfig(3))); + wait(rangeConfig->updateRange(&tr, "\xff\x04t"_sr, "\xff\x05h"_sr, DDRangeConfig(3, 1), true)); + wait(rangeConfig->updateRange(&tr, "\xff\x06u"_sr, "\xff\x07m"_sr, DDRangeConfig({}, 2))); + + wait(rangeConfig->updateRange(&tr, "a"_sr, "b"_sr, DDRangeConfig(3, 20), true)); + wait(rangeConfig->updateRange(&tr, "a"_sr, "a10"_sr, DDRangeConfig(3, 20), true)); + + // Key, entryRequiredInDB, expectedRangeConfig + state std::vector> rangeTests = { + { "\x01"_sr, false, DDRangeConfig() }, + { "\xff\x10"_sr, false, DDRangeConfig() }, + { "\xff\x03x"_sr, true, DDRangeConfig(3) }, + { "\xff\x04z"_sr, true, DDRangeConfig(3, 1) }, + { "\xff\x05"_sr, true, DDRangeConfig(3, 1) }, + { "\xff\x06"_sr, true, DDRangeConfig(6) }, + { "\xff\x06u"_sr, true, DDRangeConfig(6, 2) }, + { "\xff\x06v"_sr, true, DDRangeConfig(6, 2) }, + { "\xff\x07m"_sr, false, DDRangeConfig() }, + { "\xff\x07k"_sr, true, DDRangeConfig({}, 2) }, + { "a"_sr, true, DDRangeConfig(3, 20) }, + { "a10"_sr, true, DDRangeConfig(3, 20) }, + { "a11"_sr, true, DDRangeConfig(3, 20) }, + { "b"_sr, false, DDRangeConfig() }, + { "b1"_sr, false, DDRangeConfig() } + }; + + state Reference snapshot = + wait(rangeConfig->getSnapshot(&tr, allKeys.begin, allKeys.end)); + + if (verbose) { + fmt::print( + "DD User Range Config:\n{}\n", + json_spirit::write_string(DDConfiguration::toJSON(*snapshot, true), json_spirit::pretty_print)); + } + + state int i; + for (i = 0; i < rangeTests.size(); ++i) { + state Key query = std::get<0>(rangeTests[i]); + Optional verify = + wait(rangeConfig->getRangeForKey(&tr, query)); + + if (verbose) { + if (verify.present()) { + fmt::print("'{}' is in '{}' to '{}' with config {}\n", + query.printable(), + verify->range.begin, + verify->range.end, + verify->value.toString()); + } else { + fmt::print("'{}' is not in a range in the config\n", query.printable()); + } + } + + // Range value is either present or not required by test + ASSERT(verify.present() || !std::get<1>(rangeTests[i])); + DDRangeConfig rc = std::get<2>(rangeTests[i]); + ASSERT(!verify.present() || verify->value == rc); + + auto snapshotRange = snapshot->rangeContaining(query); + ASSERT(snapshotRange.value() == rc); + // The snapshot has all ranges covered but the db may not + if (verify.present()) { + ASSERT(snapshotRange.range().begin == verify->range.begin); + ASSERT(snapshotRange.range().end == verify->range.end); + } + } + } + + wait(tr.commit()); + TraceEvent("KeyRangeConfigCommitted").log(); + break; + } catch (Error& e) { + TraceEvent("KeyRangeConfigCommitError").error(e); + wait(tr.onError(e)); + } + } + return Void(); +}