diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index c1e5c6f718..aa1ef9dd31 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -3441,6 +3441,8 @@ ACTOR Future> getValue(Reference trState, } trState->cx->getValueCompleted->latency = timer_int() - startTime; trState->cx->getValueCompleted->log(); + trState->totalCost += + getReadOperationCost(key.size() + (reply.value.present() ? reply.value.get().size() : 0)); if (getValueID.present()) { g_traceBatch.addEvent("GetValueDebug", @@ -5751,6 +5753,7 @@ void Transaction::set(const KeyRef& key, const ValueRef& value, AddConflictRange auto r = singleKeyRange(key, req.arena); auto v = ValueRef(req.arena, value); t.mutations.emplace_back(req.arena, MutationRef::SetValue, r.begin, v); + trState->totalCost += getWriteOperationCost(key.expectedSize() + value.expectedSize()); if (addConflictRange) { t.write_conflict_ranges.push_back(req.arena, r); @@ -5780,6 +5783,7 @@ void Transaction::atomicOp(const KeyRef& key, auto v = ValueRef(req.arena, operand); t.mutations.emplace_back(req.arena, operationType, r.begin, v); + trState->totalCost += getWriteOperationCost(key.expectedSize()); if (addConflictRange && operationType != MutationRef::SetVersionstampedKey) t.write_conflict_ranges.push_back(req.arena, r); @@ -6215,14 +6219,14 @@ ACTOR Future> estimateCommitCosts(Referen state int i = 0; for (; i < transaction->mutations.size(); ++i) { - auto* it = &transaction->mutations[i]; + auto const& mutation = transaction->mutations[i]; - if (it->type == MutationRef::Type::SetValue || it->isAtomicOp()) { + if (mutation.type == MutationRef::Type::SetValue || mutation.isAtomicOp()) { trCommitCosts.opsCount++; - trCommitCosts.writeCosts += getWriteOperationCost(it->expectedSize()); - } else if (it->type == MutationRef::Type::ClearRange) { + trCommitCosts.writeCosts += getWriteOperationCost(mutation.expectedSize()); + } else if (mutation.type == MutationRef::Type::ClearRange) { trCommitCosts.opsCount++; - keyRange = KeyRangeRef(it->param1, it->param2); + keyRange = KeyRangeRef(mutation.param1, mutation.param2); if (trState->options.expensiveClearCostEstimation) { StorageMetrics m = wait(trState->cx->getStorageMetrics(keyRange, CLIENT_KNOBS->TOO_MANY)); trCommitCosts.clearIdxCosts.emplace_back(i, getWriteOperationCost(m.bytes)); diff --git a/fdbclient/include/fdbclient/NativeAPI.actor.h b/fdbclient/include/fdbclient/NativeAPI.actor.h index 3931182ab0..8f6d244606 100644 --- a/fdbclient/include/fdbclient/NativeAPI.actor.h +++ b/fdbclient/include/fdbclient/NativeAPI.actor.h @@ -249,6 +249,9 @@ struct TransactionState : ReferenceCounted { SpanContext spanContext; UseProvisionalProxies useProvisionalProxies = UseProvisionalProxies::False; bool readVersionObtainedFromGrvProxy; + // Measured by summing the bytes accessed by each read and write operations, + // after rounding up to the nearest page size and applying a write penalty + uint64_t totalCost = 0; // Special flag to skip prepending tenant prefix to mutations and conflict ranges // when a dummy, internal transaction gets commited. The sole purpose of commitDummyTransaction() is to @@ -445,6 +448,8 @@ public: // May be called only after commit() returns success Version getCommittedVersion() const { return trState->committedVersion; } + uint64_t getTotalCost() const { return trState->totalCost; } + // Will be fulfilled only after commit() returns success [[nodiscard]] Future> getVersionstamp(); @@ -568,6 +573,10 @@ inline uint64_t getWriteOperationCost(uint64_t bytes) { return (bytes - 1) / CLIENT_KNOBS->WRITE_COST_BYTE_FACTOR + 1; } +inline uint64_t getReadOperationCost(uint64_t bytes) { + return (bytes - 1) / CLIENT_KNOBS->READ_COST_BYTE_FACTOR + 1; +} + // Create a transaction to set the value of system key \xff/conf/perpetual_storage_wiggle. If enable == true, the value // will be 1. Otherwise, the value will be 0. // Returns the FDB version at which the transaction was committed. diff --git a/fdbserver/TransactionTagCounter.cpp b/fdbserver/TransactionTagCounter.cpp index 17e7539bad..89377b1b99 100644 --- a/fdbserver/TransactionTagCounter.cpp +++ b/fdbserver/TransactionTagCounter.cpp @@ -18,6 +18,7 @@ * limitations under the License. */ +#include "fdbclient/NativeAPI.actor.h" #include "fdbserver/Knobs.h" #include "fdbserver/TransactionTagCounter.h" #include "flow/Trace.h" @@ -90,9 +91,6 @@ class TransactionTagCounterImpl { std::vector previousBusiestTags; Reference busiestReadTagEventHolder; - // Round up to the nearest page size - static int64_t costFunction(int64_t bytes) { return (bytes - 1) / CLIENT_KNOBS->READ_COST_BYTE_FACTOR + 1; } - public: TransactionTagCounterImpl(UID thisServerID) : thisServerID(thisServerID), topTags(SERVER_KNOBS->SS_THROTTLE_TAGS_TRACKED), @@ -101,7 +99,7 @@ public: void addRequest(Optional const& tags, int64_t bytes) { if (tags.present()) { CODE_PROBE(true, "Tracking transaction tag in counter"); - double cost = costFunction(bytes); + auto const cost = getReadOperationCost(bytes); for (auto& tag : tags.get()) { int64_t& count = intervalCounts[TransactionTag(tag, tags.get().getArena())]; topTags.incrementCount(tag, count, cost); diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index b7fa9ad323..adf158c818 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -1984,7 +1984,7 @@ ACTOR Future getValueQ(StorageServer* data, GetValueRequest req) { data->sendErrorWithPenalty(req.reply, e, data->getPenalty()); } - data->transactionTagCounter.addRequest(req.tags, resultSize); + data->transactionTagCounter.addRequest(req.tags, req.key.size() + resultSize); ++data->counters.finishedQueries; diff --git a/fdbserver/workloads/TransactionCost.actor.cpp b/fdbserver/workloads/TransactionCost.actor.cpp new file mode 100644 index 0000000000..c4965bb0ab --- /dev/null +++ b/fdbserver/workloads/TransactionCost.actor.cpp @@ -0,0 +1,124 @@ +/* + * TransactionCost.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbserver/workloads/workloads.actor.h" +#include "flow/actorcompiler.h" + +class TransactionCostWorkload : public TestWorkload { + int iterations{ 1000 }; + Key prefix; + bool debugTransactions{ false }; + + static constexpr auto transactionTypes = 3; + + ACTOR static Future read(Database cx, Optional debugID) { + state Transaction tr(cx); + if (debugID.present()) { + tr.debugTransaction(debugID.get()); + } + loop { + try { + ASSERT_EQ(tr.getTotalCost(), 0); + wait(success(tr.get("foo"_sr))); + ASSERT_EQ(tr.getTotalCost(), 1); + return Void(); + } catch (Error& e) { + wait(tr.onError(e)); + } + } + } + + ACTOR static Future write(Database cx, Optional debugID) { + state Transaction tr(cx); + if (debugID.present()) { + tr.debugTransaction(debugID.get()); + } + loop { + try { + ASSERT_EQ(tr.getTotalCost(), 0); + tr.set("foo"_sr, "bar"_sr); + ASSERT_EQ(tr.getTotalCost(), 1); + wait(tr.commit()); + ASSERT_EQ(tr.getTotalCost(), 1); + return Void(); + } catch (Error& e) { + TraceEvent("TransactionCost_Error").error(e); + wait(tr.onError(e)); + } + } + } + + ACTOR static Future clear(Database cx, Optional debugID) { + state Transaction tr(cx); + if (debugID.present()) { + tr.debugTransaction(debugID.get()); + } + loop { + try { + ASSERT_EQ(tr.getTotalCost(), 0); + tr.clear(singleKeyRange("foo"_sr)); + wait(tr.commit()); + ASSERT_EQ(tr.getTotalCost(), 0); + return Void(); + } catch (Error& e) { + wait(tr.onError(e)); + } + } + } + + ACTOR static Future start(TransactionCostWorkload* self, Database cx) { + state uint64_t i = 0; + state Future f; + for (; i < self->iterations; ++i) { + int rand = deterministicRandom()->randomInt(0, transactionTypes); + auto const debugID = self->debugTransactions ? UID(i << 32, i << 32) : Optional(); + if (rand == 0) { + f = read(cx, debugID); + } else if (rand == 1) { + f = write(cx, debugID); + } else if (rand == 2) { + f = clear(cx, debugID); + } + wait(f); + } + return Void(); + } + +public: + TransactionCostWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { + iterations = getOption(options, "iterations"_sr, 1000); + prefix = getOption(options, "prefix"_sr, "transactionCost/"_sr); + debugTransactions = getOption(options, "debug"_sr, false); + } + + static constexpr auto NAME = "TransactionCost"; + + std::string description() const override { return NAME; } + + Future setup(Database const& cx) override { return Void(); } + + Future start(Database const& cx) override { return start(this, cx); } + + Future check(Database const& cx) override { return true; } + + void getMetrics(std::vector& m) override {} +}; + +WorkloadFactory Transaction("TransactionCost"); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3de24af735..d05d828fc4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -240,6 +240,7 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES rare/RedwoodCorrectnessBTree.toml) add_fdb_test(TEST_FILES rare/RedwoodDeltaTree.toml) add_fdb_test(TEST_FILES rare/Throttling.toml) + add_fdb_test(TEST_FILES rare/TransactionCost.toml) add_fdb_test(TEST_FILES rare/TransactionTagApiCorrectness.toml) add_fdb_test(TEST_FILES rare/TransactionTagSwizzledApiCorrectness.toml) add_fdb_test(TEST_FILES rare/WriteTagThrottling.toml) diff --git a/tests/rare/TransactionCost.toml b/tests/rare/TransactionCost.toml new file mode 100644 index 0000000000..2c0cb9f88b --- /dev/null +++ b/tests/rare/TransactionCost.toml @@ -0,0 +1,6 @@ +[[test]] +testTitle = 'TransactionCostTest' + + [[test.workload]] + testName = 'TransactionCost' + iterations = 1000