wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
foundationdb/fdbserver/TLogServer.actor.cpp

3434 lines
139 KiB
C++
Raw Normal View History

Initial repository commit
2017-05-26 04:48:44 +08:00
/*
* TLogServer.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
remove trailing whitespace from our copyright headers ; fixed formatting of python setup.py
2018-02-22 02:25:11 +08:00
*
Initial repository commit
2017-05-26 04:48:44 +08:00
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
remove trailing whitespace from our copyright headers ; fixed formatting of python setup.py
2018-02-22 02:25:11 +08:00
*
Initial repository commit
2017-05-26 04:48:44 +08:00
* http://www.apache.org/licenses/LICENSE-2.0
remove trailing whitespace from our copyright headers ; fixed formatting of python setup.py
2018-02-22 02:25:11 +08:00
*
Initial repository commit
2017-05-26 04:48:44 +08:00
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "flow/Hash3.h"
#include "flow/UnitTest.h"
Use proper fwd decl in NativeAPI Also NativeAPI.h -> NativeAPI.actor.h
2019-02-18 07:41:16 +08:00
#include "fdbclient/NativeAPI.actor.h"
moved Notified.h from flow -> fdbclient ; flow bindings package does better job when excluding testers
2017-07-15 06:49:30 +08:00
#include "fdbclient/Notified.h"
Initial repository commit
2017-05-26 04:48:44 +08:00
#include "fdbclient/KeyRangeMap.h"
Avoid using trackLatest for TLog pop test cases
2019-04-24 07:17:54 +08:00
#include "fdbclient/RunTransaction.actor.h"
make spawnProcess predictable in the simulator
2019-04-25 03:45:28 +08:00
#include "fdbclient/SystemData.h"
Address review comments for PR #1625
2019-08-15 05:19:50 +08:00
#include "fdbclient/FDBTypes.h"
Use fwd decls in WorkerInterface Also WorkerInterface.h -> WorkerInterface.actor.h
2019-02-18 11:13:26 +08:00
#include "fdbserver/WorkerInterface.actor.h"
Fixed a bug in cache server while peeking at a version lower than popped version and added some logging.
2020-03-27 03:39:07 +08:00
#include "fdbserver/LogProtocolMessage.h"
Add span metadata message
2020-08-28 06:11:16 +08:00
#include "fdbserver/SpanContextMessage.h"
Adjust all includes to be relative to the root. Remove the use of relative paths. A header at foo/bar.h could be included by files under foo/ with "bar.h", but would be included everywhere else as "foo/bar.h". Adjust so that every include references such a header with the latter form. Signed-off-by: Robert Escriva <rescriva@dropbox.com>
2018-10-20 01:30:13 +08:00
#include "fdbserver/TLogInterface.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/IKeyValueStore.h"
Add debugMessagesAndTags, and track mutations in more places. Like: * Leaving the proxy * Entering the TLog * Leaving the TLog * Being read on a cursor All of this brought to you by TagsAndMessage! This also slides in a minor optimization as to how mutations are serialized per target log.
2020-03-27 18:31:04 +08:00
#include "fdbserver/MutationTracking.h"
Initial repository commit
2017-05-26 04:48:44 +08:00
#include "flow/ActorCollection.h"
#include "fdbrpc/FailureMonitor.h"
Adjust all includes to be relative to the root. Remove the use of relative paths. A header at foo/bar.h could be included by files under foo/ with "bar.h", but would be included everywhere else as "foo/bar.h". Adjust so that every include references such a header with the latter form. Signed-off-by: Robert Escriva <rescriva@dropbox.com>
2018-10-20 01:30:13 +08:00
#include "fdbserver/IDiskQueue.h"
Initial repository commit
2017-05-26 04:48:44 +08:00
#include "fdbrpc/sim_validation.h"
tlog commits very rarely take an additional 6 seconds
2019-03-12 03:11:17 +08:00
#include "fdbrpc/simulator.h"
Track statistics about server-side request latency on each process, to include min, max, mean, and various percentiles.
2020-07-10 07:39:15 +08:00
#include "fdbrpc/Stats.h"
Adjust all includes to be relative to the root. Remove the use of relative paths. A header at foo/bar.h could be included by files under foo/ with "bar.h", but would be included everywhere else as "foo/bar.h". Adjust so that every include references such a header with the latter form. Signed-off-by: Robert Escriva <rescriva@dropbox.com>
2018-10-20 01:30:13 +08:00
#include "fdbserver/ServerDBInfo.h"
#include "fdbserver/LogSystem.h"
#include "fdbserver/WaitFailure.h"
#include "fdbserver/RecoveryState.h"
code re-orgnaization and address comments
2019-04-21 03:58:24 +08:00
#include "fdbserver/FDBExecHelper.actor.h"
tLog: Track tlog commit latencies in histogram
2020-11-13 07:54:43 +08:00
#include "flow/Histogram.h"
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
#include "flow/actorcompiler.h" // This must be the last #include.
Initial repository commit
2017-05-26 04:48:44 +08:00
using std::make_pair;
using std::max;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
using std::min;
using std::pair;
Initial repository commit
2017-05-26 04:48:44 +08:00
struct TLogQueueEntryRef {
UID id;
Version version;
Version knownCommittedVersion;
StringRef messages;
TLogQueueEntryRef() : version(0), knownCommittedVersion(0) {}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TLogQueueEntryRef(Arena& a, TLogQueueEntryRef const& from)
: version(from.version), knownCommittedVersion(from.knownCommittedVersion), id(from.id),
messages(a, from.messages) {}
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// To change this serialization, ProtocolVersion::TLogQueueEntryRef must be updated, and downgrades need to be
// considered
Initial repository commit
2017-05-26 04:48:44 +08:00
template <class Ar>
void serialize(Ar& ar) {
Replace & operator with variadic function
2018-12-29 02:49:26 +08:00
serializer(ar, version, messages, knownCommittedVersion, id);
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
size_t expectedSize() const { return messages.expectedSize(); }
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
};
struct AlternativeTLogQueueEntryRef {
UID id;
Version version;
Version knownCommittedVersion;
std::vector<TagsAndMessage>* alternativeMessages;
Replace NULL with nullptr in fdbserver
2020-08-19 05:18:50 +08:00
AlternativeTLogQueueEntryRef() : version(0), knownCommittedVersion(0), alternativeMessages(nullptr) {}
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
template <class Ar>
void serialize(Ar& ar) {
ASSERT(!ar.isDeserializing && alternativeMessages);
uint32_t msgSize = expectedSize();
Replace & operator with variadic function
2018-12-29 02:49:26 +08:00
serializer(ar, version, msgSize);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& msg : *alternativeMessages) {
ar.serializeBytes(msg.message);
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
}
Replace & operator with variadic function
2018-12-29 02:49:26 +08:00
serializer(ar, knownCommittedVersion, id);
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
}
uint32_t expectedSize() const {
uint32_t msgSize = 0;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& msg : *alternativeMessages) {
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
msgSize += msg.message.size();
}
return msgSize;
Initial repository commit
2017-05-26 04:48:44 +08:00
}
};
typedef Standalone<TLogQueueEntryRef> TLogQueueEntry;
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
struct LogData;
struct TLogData;
Initial repository commit
2017-05-26 04:48:44 +08:00
Explicitly seal classes that inherit but aren't inherited from
2020-10-08 12:58:24 +08:00
struct TLogQueue final : public IClosable {
Initial repository commit
2017-05-26 04:48:44 +08:00
public:
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TLogQueue(IDiskQueue* queue, UID dbgid) : queue(queue), dbgid(dbgid) {}
Initial repository commit
2017-05-26 04:48:44 +08:00
// Each packet in the queue is
// uint32_t payloadSize
// uint8_t payload[payloadSize] (begins with uint64_t protocolVersion via IncludeVersion)
// uint8_t validFlag
// TLogQueue is a durable queue of TLogQueueEntry objects with an interface similar to IDiskQueue
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// TLogQueue pushes (but not commits) are atomic - after commit fails to return, a prefix of entire calls to push
// are durable. This is
// implemented on top of the weaker guarantee of IDiskQueue::commit (that a prefix of bytes is durable) using
// validFlag and by padding any incomplete packet with zeros after recovery.
Initial repository commit
2017-05-26 04:48:44 +08:00
// Before calling push, pop, or commit, the user must call readNext() until it throws
// end_of_stream(). It may not be called again thereafter.
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Future<TLogQueueEntry> readNext(TLogData* tLog) { return readNext(this, tLog); }
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Future<bool> initializeRecovery(IDiskQueue::location recoverAt) { return queue->initializeRecovery(recoverAt); }
Implement limiting how many bytes recovery will read. This time, track what location in the DiskQueue has been spilled in persistent state, and then feed it back into the disk queue before recovery. This also introduces an ASSERT that recovery only reads exactly the bytes that it needs to have in memory.
2019-03-16 12:01:22 +08:00
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
template <class T>
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void push(T const& qe, Reference<LogData> logData);
void forgetBefore(Version upToVersion, Reference<LogData> logData);
void pop(IDiskQueue::location upToLocation);
Initial repository commit
2017-05-26 04:48:44 +08:00
Future<Void> commit() { return queue->commit(); }
// Implements IClosable
Use override where applicable in fdbserver
2020-10-08 09:41:19 +08:00
Future<Void> getError() override { return queue->getError(); }
Future<Void> onClosed() override { return queue->onClosed(); }
void dispose() override {
queue->dispose();
delete this;
}
void close() override {
queue->close();
delete this;
}
Initial repository commit
2017-05-26 04:48:44 +08:00
private:
IDiskQueue* queue;
UID dbgid;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void updateVersionSizes(const TLogQueueEntry& result,
TLogData* tLog,
IDiskQueue::location start,
IDiskQueue::location end);
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR static Future<TLogQueueEntry> readNext(TLogQueue* self, TLogData* tLog) {
Initial repository commit
2017-05-26 04:48:44 +08:00
state TLogQueueEntry result;
state int zeroFillSize = 0;
loop {
Track both start and end versions in versionLocation.
2019-02-08 09:02:31 +08:00
state IDiskQueue::location startloc = self->queue->getNextReadLocation();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Standalone<StringRef> h = wait(self->queue->readNext(sizeof(uint32_t)));
Initial repository commit
2017-05-26 04:48:44 +08:00
if (h.size() != sizeof(uint32_t)) {
if (h.size()) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TEST(true); // Zero fill within size field
Initial repository commit
2017-05-26 04:48:44 +08:00
int payloadSize = 0;
memcpy(&payloadSize, h.begin(), h.size());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
zeroFillSize = sizeof(uint32_t) - h.size(); // zero fill the size itself
zeroFillSize += payloadSize + 1; // and then the contents and valid flag
Initial repository commit
2017-05-26 04:48:44 +08:00
}
break;
}
state uint32_t payloadSize = *(uint32_t*)h.begin();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ASSERT(payloadSize < (100 << 20));
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Standalone<StringRef> e = wait(self->queue->readNext(payloadSize + 1));
if (e.size() != payloadSize + 1) {
TEST(true); // Zero fill within payload
zeroFillSize = payloadSize + 1 - e.size();
Initial repository commit
2017-05-26 04:48:44 +08:00
break;
}
if (e[payloadSize]) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ASSERT(e[payloadSize] == 1);
Initial repository commit
2017-05-26 04:48:44 +08:00
Arena a = e.arena();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ArenaReader ar(a, e.substr(0, payloadSize), IncludeVersion());
Initial repository commit
2017-05-26 04:48:44 +08:00
ar >> result;
Track both start and end versions in versionLocation.
2019-02-08 09:02:31 +08:00
const IDiskQueue::location endloc = self->queue->getNextReadLocation();
self->updateVersionSizes(result, tLog, startloc, endloc);
Initial repository commit
2017-05-26 04:48:44 +08:00
return result;
}
}
if (zeroFillSize) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TEST(true); // Fixing a partial commit at the end of the tlog queue
for (int i = 0; i < zeroFillSize; i++)
self->queue->push(StringRef((const uint8_t*)"", 1));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
throw end_of_stream();
}
};
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
////// Persistence format (for self->persistentData)
// Immutable keys
Change many things about log_version. * log_version in the database (`/conf/log_version`) is now a hint that gets rounded to the nearest supported version. * fdbcli and FDB enforce that only a valid log_version can be configured to * TLogVersion is persisted in CoreTLogSet (and LogSet and TLogSet) * Some comments here and there * Add an assert on filename length to make sure KV-pairs in filename don't exceed a maximum length.
2019-02-27 08:47:04 +08:00
// persistFormat has been mostly invalidated by TLogVersion, and can probably be removed when
// 4.6's TLog code is removed.
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
static const KeyValueRef persistFormat(LiteralStringRef("Format"), LiteralStringRef("FoundationDB/LogServer/3/0"));
static const KeyRangeRef persistFormatReadableRange(LiteralStringRef("FoundationDB/LogServer/3/0"),
LiteralStringRef("FoundationDB/LogServer/4/0"));
static const KeyRangeRef persistProtocolVersionKeys(LiteralStringRef("ProtocolVersion/"),
LiteralStringRef("ProtocolVersion0"));
static const KeyRangeRef persistTLogSpillTypeKeys(LiteralStringRef("TLogSpillType/"),
LiteralStringRef("TLogSpillType0"));
static const KeyRangeRef persistRecoveryCountKeys =
KeyRangeRef(LiteralStringRef("DbRecoveryCount/"), LiteralStringRef("DbRecoveryCount0"));
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
// Updated on updatePersistentData()
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
static const KeyRangeRef persistCurrentVersionKeys =
KeyRangeRef(LiteralStringRef("version/"), LiteralStringRef("version0"));
static const KeyRangeRef persistKnownCommittedVersionKeys =
KeyRangeRef(LiteralStringRef("knownCommitted/"), LiteralStringRef("knownCommitted0"));
static const KeyRef persistRecoveryLocationKey = KeyRef(LiteralStringRef("recoveryLocation"));
static const KeyRangeRef persistLocalityKeys =
KeyRangeRef(LiteralStringRef("Locality/"), LiteralStringRef("Locality0"));
static const KeyRangeRef persistLogRouterTagsKeys =
KeyRangeRef(LiteralStringRef("LogRouterTags/"), LiteralStringRef("LogRouterTags0"));
static const KeyRangeRef persistTxsTagsKeys = KeyRangeRef(LiteralStringRef("TxsTags/"), LiteralStringRef("TxsTags0"));
Spill txsTag by value under TagMsg/ and not TagMsgRef/ There's not a tremendous reason as to why this matters now, but I feel like I might regret sometime later not keeping the same schema under the same key.
2019-03-04 09:01:52 +08:00
static const KeyRange persistTagMessagesKeys = prefixRange(LiteralStringRef("TagMsg/"));
Persist start,end of message when spilling tags.
2019-02-08 09:02:33 +08:00
static const KeyRange persistTagMessageRefsKeys = prefixRange(LiteralStringRef("TagMsgRef/"));
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
static const KeyRange persistTagPoppedKeys = prefixRange(LiteralStringRef("TagPop/"));
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
static Key persistTagMessagesKey(UID id, Tag tag, Version version) {
BinaryWriter wr(Unversioned());
Spill txsTag by value under TagMsg/ and not TagMsgRef/ There's not a tremendous reason as to why this matters now, but I feel like I might regret sometime later not keeping the same schema under the same key.
2019-03-04 09:01:52 +08:00
wr.serializeBytes(persistTagMessagesKeys.begin);
wr << id;
wr << tag;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wr << bigEndian64(version);
renamed BinaryWriter.toStringRef() to .toValue(), because the function now returns a Standalone<StringRef>() eliminated an unnecessary copy from the proxy commit path eliminated an unnecessary copy from buffered peek cursor
2019-03-29 02:52:50 +08:00
return wr.toValue();
Spill txsTag by value under TagMsg/ and not TagMsgRef/ There's not a tremendous reason as to why this matters now, but I feel like I might regret sometime later not keeping the same schema under the same key.
2019-03-04 09:01:52 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
static Key persistTagMessageRefsKey(UID id, Tag tag, Version version) {
BinaryWriter wr(Unversioned());
Persist start,end of message when spilling tags.
2019-02-08 09:02:33 +08:00
wr.serializeBytes(persistTagMessageRefsKeys.begin);
wr << id;
wr << tag;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wr << bigEndian64(version);
renamed BinaryWriter.toStringRef() to .toValue(), because the function now returns a Standalone<StringRef>() eliminated an unnecessary copy from the proxy commit path eliminated an unnecessary copy from buffered peek cursor
2019-03-29 02:52:50 +08:00
return wr.toValue();
Persist start,end of message when spilling tags.
2019-02-08 09:02:33 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
static Key persistTagPoppedKey(UID id, Tag tag) {
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
BinaryWriter wr(Unversioned());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wr.serializeBytes(persistTagPoppedKeys.begin);
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
wr << id;
wr << tag;
renamed BinaryWriter.toStringRef() to .toValue(), because the function now returns a Standalone<StringRef>() eliminated an unnecessary copy from the proxy commit path eliminated an unnecessary copy from buffered peek cursor
2019-03-29 02:52:50 +08:00
return wr.toValue();
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
static Value persistTagPoppedValue(Version popped) {
return BinaryWriter::toValue(popped, Unversioned());
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
static Tag decodeTagPoppedKey(KeyRef id, KeyRef key) {
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
Tag s;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
BinaryReader rd(key.removePrefix(persistTagPoppedKeys.begin).removePrefix(id), Unversioned());
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
rd >> s;
return s;
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
static Version decodeTagPoppedValue(ValueRef value) {
return BinaryReader::fromStringRef<Version>(value, Unversioned());
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
static StringRef stripTagMessagesKey(StringRef key) {
return key.substr(sizeof(UID) + sizeof(Tag) + persistTagMessagesKeys.begin.size());
Spill txsTag by value under TagMsg/ and not TagMsgRef/ There's not a tremendous reason as to why this matters now, but I feel like I might regret sometime later not keeping the same schema under the same key.
2019-03-04 09:01:52 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
static Version decodeTagMessagesKey(StringRef key) {
return bigEndian64(BinaryReader::fromStringRef<Version>(stripTagMessagesKey(key), Unversioned()));
Spill txsTag by value under TagMsg/ and not TagMsgRef/ There's not a tremendous reason as to why this matters now, but I feel like I might regret sometime later not keeping the same schema under the same key.
2019-03-04 09:01:52 +08:00
}
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
struct SpilledData {
SpilledData() = default;
SpilledData(Version version, IDiskQueue::location start, uint32_t length, uint32_t mutationBytes)
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
: version(version), start(start), length(length), mutationBytes(mutationBytes) {}
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, version, start, length, mutationBytes);
}
Version version = 0;
IDiskQueue::location start = 0;
uint32_t length = 0;
uint32_t mutationBytes = 0;
};
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
struct TLogData : NonCopyable {
AsyncTrigger newLogData;
Comments:Add comments to DiskQueue No functional change.
2019-07-30 10:17:10 +08:00
// A process has only 1 SharedTLog, which holds data for multiple logs, so that it obeys its assigned memory limit.
// A process has only 1 active log and multiple non-active log from old generations.
// In the figure below, TLog [1-4] are logs from old generations.
// Because SS may need to pull data from old generation log, we keep Tlog [1-4].
//
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
// We always pop the disk queue from the oldest TLog, spill from the oldest TLog that still has
// data in memory, and commits to the disk queue come from the most recent TLog.
//
// SharedTLog
// +--------+--------+--------+--------+--------+
// | TLog 1 | TLog 2 | TLog 3 | TLog 4 | TLog 5 |
// +--------+--------+--------+--------+--------+
// ^ popOrder ^spillOrder ^committing
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
//
Comments:Add comments to DiskQueue No functional change.
2019-07-30 10:17:10 +08:00
// ^popOrder is the location where SS reads the to-be-read data from tlog.
// ^committing is the location where the active TLog accepts the pushed data.
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
Deque<UID> popOrder;
Deque<UID> spillOrder;
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
std::map<UID, Reference<struct LogData>> id_data;
UID dbgid;
Improve TLog "Role" event by adding the worker ID, the TLog version, and under what circumstances the TLog is being started (Restored, Recruited, or Recovered). The SharedTLog role was being started and stopped twice, so remove one instance of it.
2020-02-13 07:11:38 +08:00
UID workerID;
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
Comments:Add comments to DiskQueue No functional change.
2019-07-30 10:17:10 +08:00
IKeyValueStore* persistentData; // Durable data on disk that were spilled.
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
IDiskQueue* rawPersistentQueue; // The physical queue the persistentQueue below stores its data. Ideally, log
// interface should work without directly accessing rawPersistentQueue
TLogQueue* persistentQueue; // Logical queue the log operates on and persist its data.
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
int64_t diskQueueCommitBytes;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
AsyncVar<bool>
largeDiskQueueCommitBytes; // becomes true when diskQueueCommitBytes is greater than MAX_QUEUE_COMMIT_BYTES
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
Prevent tLog from modifying ServerDBInfo object
2021-07-12 13:25:40 +08:00
Reference<AsyncVar<ServerDBInfo> const> dbInfo;
Avoid using trackLatest for TLog pop test cases
2019-04-24 07:17:54 +08:00
Database cx;
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
NotifiedVersion queueCommitEnd;
Version queueCommitBegin;
int64_t instanceID;
int64_t bytesInput;
int64_t bytesDurable;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
int64_t targetVolatileBytes; // The number of bytes of mutations this TLog should hold in memory before spilling.
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
int64_t overheadBytesInput;
added overhead bytes durable to complement overhead bytes input
2018-08-22 13:35:04 +08:00
int64_t overheadBytesDurable;
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
WorkerCache<TLogInterface> tlogCache;
Replace ResourceLimiter with FlowLock.
2019-03-04 08:49:42 +08:00
FlowLock peekMemoryLimiter;
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
PromiseStream<Future<Void>> sharedActors;
fix: during destruction logData could attempt to dereference tLogData after it has been deleted
2018-08-10 03:38:35 +08:00
Promise<Void> terminated;
tlogs serve reads to log routers at a low priority, to prevent them from using all their resources catching up a remote dc that has been down for a long time increase the amount of memory ratekeeper budgets for tlogs so that there is a gap after the spill threshold to prevent temporarily overshooting the budget
2018-08-05 01:31:30 +08:00
FlowLock concurrentLogRouterReads;
Merge release-6.0 into master
2018-09-22 07:14:39 +08:00
FlowLock persistentDataCommitLock;
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
Add Alex comment on tLog
2020-11-13 05:29:11 +08:00
// Beginning of fields used by snapshot based backup and restore
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
bool ignorePopRequest; // ignore pop request from storage servers
trace cleanup and other indentation changes
2019-04-03 20:47:19 +08:00
double ignorePopDeadline; // time until which the ignorePopRequest will be
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// honored
trace cleanup and other indentation changes
2019-04-03 20:47:19 +08:00
std::string ignorePopUid; // callers that set ignorePopRequest will set this
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// extra state, used to validate the ownership of
// the set and for callers that unset will
// be able to match it up
Snapshot based backup and resotre implementation
2019-02-28 07:40:33 +08:00
std::string dataFolder; // folder where data is stored
tlog: when pops are disabled, store them & replay In Tlogs, disable pop is done whlie taking snapshots. Earlier, tlogs were ignoring the pops if it got pop requests when pops were disabled. In this change, instead of ignoring the pop - it remembers the list of pops in-memory and plays them once the popping is enabled.
2019-03-20 02:36:07 +08:00
std::map<Tag, Version> toBePopped; // map of Tag->Version for all the pops
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// that came when ignorePopRequest was set
Fix few compilation and bugs in rebase
2019-03-20 06:12:47 +08:00
Reference<AsyncVar<bool>> degraded;
Add Alex comment on tLog
2020-11-13 05:29:11 +08:00
// End of fields used by snapshot based backup and restore
Add comments and questions to HA and tLog code reading The comments' correctness need to be confirmed by reviewers.
2020-10-28 00:11:56 +08:00
optimized memory allocations by using VectorRef<Tag> instead of std::vector<Tag>
2019-11-06 10:07:30 +08:00
std::vector<TagsAndMessage> tempTagMessages;
when a tlog queue commit takes longer than 5 seconds, its process is marked as degraded
2019-03-09 00:46:34 +08:00
tLog: Track tlog commit latencies in histogram
2020-11-13 07:54:43 +08:00
Reference<Histogram> commitLatencyDist;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TLogData(UID dbgid,
UID workerID,
IKeyValueStore* persistentData,
IDiskQueue* persistentQueue,
Prevent tLog from modifying ServerDBInfo object
2021-07-12 13:25:40 +08:00
Reference<AsyncVar<ServerDBInfo> const> dbInfo,
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Reference<AsyncVar<bool>> degraded,
std::string folder)
tLog: Track tlog commit latencies in histogram
2020-11-13 07:54:43 +08:00
: dbgid(dbgid), workerID(workerID), instanceID(deterministicRandom()->randomUniqueID().first()),
persistentData(persistentData), rawPersistentQueue(persistentQueue),
persistentQueue(new TLogQueue(persistentQueue, dbgid)), dbInfo(dbInfo), degraded(degraded), queueCommitBegin(0),
queueCommitEnd(0), diskQueueCommitBytes(0), largeDiskQueueCommitBytes(false), bytesInput(0), bytesDurable(0),
targetVolatileBytes(SERVER_KNOBS->TLOG_SPILL_THRESHOLD), overheadBytesInput(0), overheadBytesDurable(0),
peekMemoryLimiter(SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES),
concurrentLogRouterReads(SERVER_KNOBS->CONCURRENT_LOG_ROUTER_READS), ignorePopRequest(false),
ignorePopDeadline(), ignorePopUid(), dataFolder(folder), toBePopped(),
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
commitLatencyDist(Histogram::getHistogram(LiteralStringRef("tLog"),
LiteralStringRef("commit"),
tLog: Track tlog commit latencies in histogram
2020-11-13 07:54:43 +08:00
Histogram::Unit::microseconds)) {
Rename ::TRUE and ::FALSE in BooleanParams to ::True and ::False so as to not conflict with the TRUE and FALSE macros provided by the Windows and MacOS SDKs.
2021-07-17 15:11:40 +08:00
cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::True);
tLog: Track tlog commit latencies in histogram
2020-11-13 07:54:43 +08:00
}
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
};
Initial repository commit
2017-05-26 04:48:44 +08:00
struct LogData : NonCopyable, public ReferenceCounted<LogData> {
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
struct TagData : NonCopyable, public ReferenceCounted<TagData> {
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
std::deque<std::pair<Version, LengthPrefixedStringRef>> versionMessages;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
bool
nothingPersistent; // true means tag is *known* to have no messages in persistentData. false means nothing.
bool poppedRecently; // `popped` has changed since last updatePersistentData
Version popped; // see popped version tracking contract below
Version persistentPopped; // The popped version recorded in the btree.
Version versionForPoppedLocation; // `poppedLocation` was calculated at this popped version
IDiskQueue::location poppedLocation; // The location of the earliest commit with data for this tag.
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
bool unpoppedRecovered;
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
Tag tag;
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TagData(Tag tag,
Version popped,
IDiskQueue::location poppedLocation,
bool nothingPersistent,
bool poppedRecently,
bool unpoppedRecovered)
: tag(tag), nothingPersistent(nothingPersistent), poppedRecently(poppedRecently), popped(popped),
persistentPopped(0), versionForPoppedLocation(0), poppedLocation(poppedLocation),
unpoppedRecovered(unpoppedRecovered) {}
Initial repository commit
2017-05-26 04:48:44 +08:00
Replace BOOST_NOEXCEPT with noexcept
2020-06-10 08:33:41 +08:00
TagData(TagData&& r) noexcept
: versionMessages(std::move(r.versionMessages)), nothingPersistent(r.nothingPersistent),
poppedRecently(r.poppedRecently), popped(r.popped), persistentPopped(r.persistentPopped),
versionForPoppedLocation(r.versionForPoppedLocation), poppedLocation(r.poppedLocation), tag(r.tag),
unpoppedRecovered(r.unpoppedRecovered) {}
void operator=(TagData&& r) noexcept {
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
versionMessages = std::move(r.versionMessages);
nothingPersistent = r.nothingPersistent;
poppedRecently = r.poppedRecently;
Initial repository commit
2017-05-26 04:48:44 +08:00
popped = r.popped;
DiskQueue should only pop based off of persisted popped tag versions This commit is to fix a bug where popping a tag between updatePersistentData and popDiskQueue can cause the TLog to recover to an incorrect understanding of what data it has available. The following series of events need to happen to trigger this bug: Tag 1:1 is popped to version 10 updatePersistentData is run... updatePersistentPopped runs and we persistentData stores 1:1 as popped to 10 A mutation is spilled for 1:1 at version 11 at location 1000 A mutation is spilled for 1:1 at version 21 at location 5000 updatePersistentData finishes and commits the btree changes Tag 1:1 is popped to version 20 popDiskQueue runs The btree is read for spilled mutations with version >=20 The minimum location required for the disk queue is found to be location 5000 The disk queue is popped to location 5000 The TLog crashes The worker restarts, and reloads the TLog files from disk restorePersistentPopped restores tag 1:1 as having been popped to version 10 Parallel peeks are received for tag 1:1 starting at version 0 The first peek is less than the popped version, so we respond with no data, and an end version of 10 The second peek starts at version 10, which is greater than the popped version The btree is read for spilled mutations, and we find that there is a mutation at version 11 at location 1000 Location 1000 is read in the DiskQueue The resulting page read at Location 1000 was popped pre-crash, and thus might either (a) be corrupt or (b) have an incorrect sequence number. The fix to this is to force popDiskQueue/updatePoppedLocation to use the popped version that was persisted to disk, and not the most recently popped version for the given tag. This bug doesn't manifest in simulation, because we don't have any code that peeks at a lower version than what has been popped.
2019-12-13 09:16:22 +08:00
persistentPopped = r.persistentPopped;
Rename poppedLocationForVersion -> versionForPoppedLocation
2019-12-20 02:24:31 +08:00
versionForPoppedLocation = r.versionForPoppedLocation;
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
poppedLocation = r.poppedLocation;
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
tag = r.tag;
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
unpoppedRecovered = r.unpoppedRecovered;
Initial repository commit
2017-05-26 04:48:44 +08:00
}
// Erase messages not needed to update *from* versions >= before (thus, messages with toversion <= before)
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> eraseMessagesBefore(TagData* self,
Version before,
TLogData* tlogData,
Reference<LogData> logData,
TaskPriority taskID) {
while (!self->versionMessages.empty() && self->versionMessages.front().first < before) {
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
Version version = self->versionMessages.front().first;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
std::pair<int, int>& sizes = logData->version_sizes[version];
Initial repository commit
2017-05-26 04:48:44 +08:00
int64_t messagesErased = 0;
Fix some conflicts and remote debugging trace events
2020-07-23 14:35:46 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
while (!self->versionMessages.empty() && self->versionMessages.front().first == version) {
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
auto const& m = self->versionMessages.front();
Initial repository commit
2017-05-26 04:48:44 +08:00
++messagesErased;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (self->tag.locality != tagLocalityTxs && self->tag != txsTag) {
Initial repository commit
2017-05-26 04:48:44 +08:00
sizes.first -= m.second.expectedSize();
significant cpu optimization in update storage
2018-08-05 03:36:48 +08:00
} else {
sizes.second -= m.second.expectedSize();
Initial repository commit
2017-05-26 04:48:44 +08:00
}
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
self->versionMessages.pop_front();
Initial repository commit
2017-05-26 04:48:44 +08:00
}
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
int64_t bytesErased = messagesErased * SERVER_KNOBS->VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD;
added overhead bytes durable to complement overhead bytes input
2018-08-22 13:35:04 +08:00
logData->bytesDurable += bytesErased;
add the same number for shared metrics and local metrics
2017-06-29 06:21:54 +08:00
tlogData->bytesDurable += bytesErased;
added overhead bytes durable to complement overhead bytes input
2018-08-22 13:35:04 +08:00
tlogData->overheadBytesDurable += bytesErased;
Rewrite all `Void _ = wait(...)` -> `wait(...)`. This takes advantage of the new actorcompiler functionality to avoid having duplicate definitions of `Void _` when trying to feed the un-actorompiled source through clang.
2018-08-11 04:57:10 +08:00
wait(yield(taskID));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Future<Void> eraseMessagesBefore(Version before,
TLogData* tlogData,
Reference<LogData> logData,
TaskPriority taskID) {
added overhead bytes durable to complement overhead bytes input
2018-08-22 13:35:04 +08:00
return eraseMessagesBefore(this, before, tlogData, logData, taskID);
Initial repository commit
2017-05-26 04:48:44 +08:00
}
};
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Map<Version, std::pair<IDiskQueue::location, IDiskQueue::location>>
versionLocation; // For the version of each entry that was push()ed, the [start, end) location of the serialized
// bytes
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
Initial repository commit
2017-05-26 04:48:44 +08:00
/*
Popped version tracking contract needed by log system to implement ILogCursor::popped():
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
- Log server tracks for each (possible) tag a popped_version
Impl: TagData::popped (in memory) and persistTagPoppedKeys (in persistentData)
- popped_version(tag) is <= the maximum version for which log server (or a predecessor) is ever asked to pop the
tag Impl: Only increased by tLogPop() in response to either a pop request or recovery from a predecessor
- popped_version(tag) is > the maximum version for which log server is unable to peek messages due to previous
pops (on this server or a predecessor) Impl: Increased by tLogPop() atomically with erasing messages from memory;
persisted by updatePersistentData() atomically with erasing messages from store; messages are not erased from queue
where popped_version is not persisted
- LockTLogReply returns all tags which either have messages, or which have nonzero popped_versions
Impl: tag_data is present for all such tags
- peek(tag, v) returns the popped_version for tag if that is greater than v
Impl: Check tag_data->popped (after all waits)
Initial repository commit
2017-05-26 04:48:44 +08:00
*/
fix: when a new log is recruited on a shared log, all outstanding commits need to be notified that they are stopped, because there is no longer a guarantee that their queueCommittedVersion will advance
2018-03-20 08:48:28 +08:00
AsyncTrigger stopCommit;
Initial repository commit
2017-05-26 04:48:44 +08:00
bool stopped, initialized;
DBRecoveryCount recoveryCount;
Add Alex comment on tLog
2020-11-13 05:29:11 +08:00
// If persistentDataVersion != persistentDurableDataVersion,
// then spilling is happening from persistentDurableDataVersion to persistentDataVersion.
// Data less than persistentDataDurableVersion is spilled on disk (or fully popped from the TLog);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
VersionMetricHandle persistentDataVersion,
persistentDataDurableVersion; // The last version number in the portion of the log (written|durable) to
// persistentData
Add Alex comment on tLog
2020-11-13 05:29:11 +08:00
NotifiedVersion version;
NotifiedVersion queueCommittedVersion; // The disk queue has committed up until the queueCommittedVersion version.
Initial repository commit
2017-05-26 04:48:44 +08:00
Version queueCommittingVersion;
Add Alex comment on tLog
2020-11-13 05:29:11 +08:00
Version knownCommittedVersion; // The maximum version that a proxy has told us that is committed (all TLogs have
// ack'd a commit for this version).
Version durableKnownCommittedVersion, minKnownCommittedVersion;
Version queuePoppedVersion; // The disk queue has been popped up until the location which represents this version.
Remove debug trace events
2020-07-23 03:20:22 +08:00
Version minPoppedTagVersion;
Add Alex comment on tLog
2020-11-13 05:29:11 +08:00
Tag minPoppedTag; // The tag that makes tLog hold its data and cause tLog's disk queue increasing.
Initial repository commit
2017-05-26 04:48:44 +08:00
Deque<std::pair<Version, Standalone<VectorRef<uint8_t>>>> messageBlocks;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
std::vector<std::vector<Reference<TagData>>> tag_data; // tag.locality | tag.id
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
int unpoppedRecoveredTags;
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
Reference<TagData> getTagData(Tag tag) {
Small code refactoring.
2019-04-02 04:56:45 +08:00
int idx = tag.toTagDataIndex();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (idx >= tag_data.size()) {
tag_data.resize(idx + 1);
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (tag.id >= tag_data[idx].size()) {
tag_data[idx].resize(tag.id + 1);
optimized tag lookups on the tlog by removing one level of vectors
2018-03-21 01:41:42 +08:00
}
return tag_data[idx][tag.id];
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// only callable after getTagData returns a null reference
Reference<TagData> createTagData(Tag tag,
Version popped,
bool nothingPersistent,
bool poppedRecently,
bool unpoppedRecovered) {
if (tag.locality != tagLocalityLogRouter && tag.locality != tagLocalityTxs && tag != txsTag && allTags.size() &&
!allTags.count(tag) && popped <= recoveredAt) {
fix: off by one error on popping missing tags
2018-07-10 06:43:22 +08:00
popped = recoveredAt + 1;
fix: remote logs did not pop all of the data for removed logs on recovery because data for the missing tag was not recorded yet at the time of recovery
2018-07-07 07:10:41 +08:00
}
Add uses of makeReference
2020-11-07 15:50:55 +08:00
auto newTagData = makeReference<TagData>(tag, popped, 0, nothingPersistent, poppedRecently, unpoppedRecovered);
Small code refactoring.
2019-04-02 04:56:45 +08:00
tag_data[tag.toTagDataIndex()][tag.id] = newTagData;
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
return newTagData;
}
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Map<Version, std::pair<int, int>> version_sizes;
Initial repository commit
2017-05-26 04:48:44 +08:00
CounterCollection cc;
Counter bytesInput;
Counter bytesDurable;
UID logId;
Revert "Revert "Make protocol version a type"" This reverts commit 455bf3b3ec9d5a347b68bf4fa89bf042f5ac312e.
2019-06-19 05:49:04 +08:00
ProtocolVersion protocolVersion;
Initial repository commit
2017-05-26 04:48:44 +08:00
Version newPersistentDataVersion;
Future<Void> removed;
PromiseStream<Future<Void>> addActor;
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
TLogData* tLogData;
fix: do not reply to recruitment until we are sure the log commits to the queue
2018-04-30 13:08:24 +08:00
Promise<Void> recoveryComplete, committingQueue;
fix: if we cannot find a tag, it must have been popped at the recovery version.
2018-04-23 06:08:38 +08:00
Version unrecoveredBefore, recoveredAt;
Initial repository commit
2017-05-26 04:48:44 +08:00
Fix parallel peek stalling for 10min when a TLog generation is destroyed. `peekTracker` was held on the Shared TLog (TLogData), whereas peeks are received and replied to as part of a TLog instance (LogData). When a peek was received on a TLog, it was registered into peekTracker along with the ReplyPromise. If the TLog was then removed as part of a no-longer-needed generation of TLogs, there is nothing left to reply to the request, but by holding onto the ReplyPromise in peekTracker, we leave the remote end with an expectation that we will reply. Then, 10min later, peekTrackerCleanup runs and finally times out the peek cursor, thus preventing FDB from being completely stuck. Now, each TLog generation has its own `peekTracker`, and when a TLog is destroyed, it times out all of the pending peek curors that are still expecting a response. This will then trigger the client to re-issue them to the next generation of TLogs, thus removing the 10min gap to do so.
2019-07-10 08:27:36 +08:00
struct PeekTrackerData {
Revise comments based on review suggestions Ack. Jingyu and Xin for their suggestions.
2020-11-06 08:33:07 +08:00
std::map<int, Promise<std::pair<Version, bool>>>
sequence_version; // second: Version is peeked begin version. bool is onlySpilled
Fix parallel peek stalling for 10min when a TLog generation is destroyed. `peekTracker` was held on the Shared TLog (TLogData), whereas peeks are received and replied to as part of a TLog instance (LogData). When a peek was received on a TLog, it was registered into peekTracker along with the ReplyPromise. If the TLog was then removed as part of a no-longer-needed generation of TLogs, there is nothing left to reply to the request, but by holding onto the ReplyPromise in peekTracker, we leave the remote end with an expectation that we will reply. Then, 10min later, peekTrackerCleanup runs and finally times out the peek cursor, thus preventing FDB from being completely stuck. Now, each TLog generation has its own `peekTracker`, and when a TLog is destroyed, it times out all of the pending peek curors that are still expecting a response. This will then trigger the client to re-issue them to the next generation of TLogs, thus removing the 10min gap to do so.
2019-07-10 08:27:36 +08:00
double lastUpdate;
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
Tag tag;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
double lastLogged;
int64_t totalPeeks;
int64_t replyBytes;
int64_t duplicatePeeks;
double queueTime;
double queueMax;
double blockTime;
double blockMax;
fix compile errors
2020-04-23 05:36:41 +08:00
double workTime;
double workMax;
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
int64_t unblockedPeeks;
double idleTime;
double idleMax;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
PeekTrackerData() : lastUpdate(0) { resetMetrics(); }
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
void resetMetrics() {
lastLogged = now();
totalPeeks = 0;
replyBytes = 0;
duplicatePeeks = 0;
queueTime = 0;
queueMax = 0;
blockTime = 0;
blockMax = 0;
fix compile errors
2020-04-23 05:36:41 +08:00
workTime = 0;
workMax = 0;
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
unblockedPeeks = 0;
idleTime = 0;
idleMax = 0;
}
Fix parallel peek stalling for 10min when a TLog generation is destroyed. `peekTracker` was held on the Shared TLog (TLogData), whereas peeks are received and replied to as part of a TLog instance (LogData). When a peek was received on a TLog, it was registered into peekTracker along with the ReplyPromise. If the TLog was then removed as part of a no-longer-needed generation of TLogs, there is nothing left to reply to the request, but by holding onto the ReplyPromise in peekTracker, we leave the remote end with an expectation that we will reply. Then, 10min later, peekTrackerCleanup runs and finally times out the peek cursor, thus preventing FDB from being completely stuck. Now, each TLog generation has its own `peekTracker`, and when a TLog is destroyed, it times out all of the pending peek curors that are still expecting a response. This will then trigger the client to re-issue them to the next generation of TLogs, thus removing the 10min gap to do so.
2019-07-10 08:27:36 +08:00
};
std::map<UID, PeekTrackerData> peekTracker;
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
Reference<AsyncVar<Reference<ILogSystem>>> logSystem;
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
Tag remoteTag;
bool isPrimary;
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
int logRouterTags;
fix: remote logs need to wait until the durable known committed version is greater than the recovery version before completing recovery to ensure we will not pick a start version that we do not have
2018-04-28 03:18:42 +08:00
Version logRouterPoppedVersion, logRouterPopToVersion;
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
int8_t locality;
a cleaner solution to preventing tlogs from peeking log routers
2018-04-21 04:25:22 +08:00
UID recruitmentID;
Make LogData aware of the spill type it was created to perform. The spilling type is now pulled out of the request, and then stored on LogData for later access, and persisted in the tlog metadata per tlog generation. It turns out that serializing types as Unversioned is a bit wonky.
2019-10-02 10:09:47 +08:00
TLogSpillType logSpillType;
fix: remote logs did not pop all of the data for removed logs on recovery because data for the missing tag was not recorded yet at the time of recovery
2018-07-07 07:10:41 +08:00
std::set<Tag> allTags;
fix: during destruction logData could attempt to dereference tLogData after it has been deleted
2018-08-10 03:38:35 +08:00
Future<Void> terminated;
use FlowLock for implementing critical section Instead of using Promises and future to implement critcal section use FlowLock
2019-05-10 05:56:54 +08:00
FlowLock execOpLock;
tlog to snapshot exactly at exec version
2019-04-08 20:23:48 +08:00
bool execOpCommitInProgress;
shard the txs tag so that more transaction logs are involved in its recovery
2019-06-20 09:15:09 +08:00
int txsTags;
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
explicit LogData(TLogData* tLogData,
TLogInterface interf,
Tag remoteTag,
bool isPrimary,
int logRouterTags,
int txsTags,
UID recruitmentID,
ProtocolVersion protocolVersion,
TLogSpillType logSpillType,
std::vector<Tag> tags,
std::string context)
: tLogData(tLogData), knownCommittedVersion(0), logId(interf.id()), cc("TLog", interf.id().toString()),
bytesInput("BytesInput", cc), bytesDurable("BytesDurable", cc), remoteTag(remoteTag), isPrimary(isPrimary),
logRouterTags(logRouterTags), txsTags(txsTags), recruitmentID(recruitmentID), protocolVersion(protocolVersion),
logSpillType(logSpillType), logSystem(new AsyncVar<Reference<ILogSystem>>()), logRouterPoppedVersion(0),
durableKnownCommittedVersion(0), minKnownCommittedVersion(0), queuePoppedVersion(0),
allTags(tags.begin(), tags.end()), terminated(tLogData->terminated.getFuture()), minPoppedTagVersion(0),
minPoppedTag(invalidTag),
// These are initialized differently on init() or recovery
recoveryCount(), stopped(false), initialized(false), queueCommittingVersion(0),
newPersistentDataVersion(invalidVersion), unrecoveredBefore(1), recoveredAt(1), unpoppedRecoveredTags(0),
logRouterPopToVersion(0), locality(tagLocalityInvalid), execOpCommitInProgress(false) {
startRole(Role::TRANSACTION_LOG,
interf.id(),
tLogData->workerID,
{ { "SharedTLog", tLogData->dbgid.shortString() } },
context);
Emit traces regularly about role assignment We are currently emitting Role transition traces when a role starts and when it ends. While this is useful for debugging, it doesn't work well with tools that inject data and might potentially miss some trace lines. We do decorate each trace lines with the roles assigned to that particular process, however, this is not sufficient for tools that can make use of the UID -> Role mapping
2020-05-09 07:27:57 +08:00
addActor.send(traceRole(Role::TRANSACTION_LOG, interf.id()));
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
Initial repository commit
2017-05-26 04:48:44 +08:00
persistentDataVersion.init(LiteralStringRef("TLog.PersistentDataVersion"), cc.id);
persistentDataDurableVersion.init(LiteralStringRef("TLog.PersistentDataDurableVersion"), cc.id);
version.initMetric(LiteralStringRef("TLog.Version"), cc.id);
queueCommittedVersion.initMetric(LiteralStringRef("TLog.QueueCommittedVersion"), cc.id);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
specialCounter(cc, "Version", [this]() { return this->version.get(); });
Add Alex comment on tLog
2020-11-13 05:29:11 +08:00
specialCounter(cc, "QueueCommittedVersion", [this]() { return this->queueCommittedVersion.get(); });
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
specialCounter(cc, "PersistentDataVersion", [this]() { return this->persistentDataVersion; });
specialCounter(cc, "PersistentDataDurableVersion", [this]() { return this->persistentDataDurableVersion; });
specialCounter(cc, "KnownCommittedVersion", [this]() { return this->knownCommittedVersion; });
specialCounter(cc, "QueuePoppedVersion", [this]() { return this->queuePoppedVersion; });
Add Alex comment on tLog
2020-11-13 05:29:11 +08:00
specialCounter(cc, "MinPoppedTagVersion", [this]() { return this->minPoppedTagVersion; });
// The locality and id of the tag that is responsible for making the TLog hold onto its oldest piece of data.
// If disk queues are growing and no one is sure why, then you shall look at this to find the tag responsible
// for why the TLog thinks it can't throw away data.
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
specialCounter(cc, "MinPoppedTagLocality", [this]() { return this->minPoppedTag.locality; });
specialCounter(cc, "MinPoppedTagId", [this]() { return this->minPoppedTag.id; });
specialCounter(cc, "SharedBytesInput", [tLogData]() { return tLogData->bytesInput; });
specialCounter(cc, "SharedBytesDurable", [tLogData]() { return tLogData->bytesDurable; });
specialCounter(cc, "SharedOverheadBytesInput", [tLogData]() { return tLogData->overheadBytesInput; });
specialCounter(cc, "SharedOverheadBytesDurable", [tLogData]() { return tLogData->overheadBytesDurable; });
Replace ResourceLimiter with FlowLock.
2019-03-04 08:49:42 +08:00
specialCounter(cc, "PeekMemoryReserved", [tLogData]() { return tLogData->peekMemoryLimiter.activePermits(); });
specialCounter(cc, "PeekMemoryRequestsStalled", [tLogData]() { return tLogData->peekMemoryLimiter.waiters(); });
Fix typo
2021-02-23 02:18:43 +08:00
specialCounter(cc, "Generation", [this]() { return this->recoveryCount; });
Initial repository commit
2017-05-26 04:48:44 +08:00
}
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
~LogData() {
Revert "Make the sim2 run loop match the behavior of the net2 run loop."
2021-06-23 05:50:01 +08:00
endRole(Role::TRANSACTION_LOG, logId, "Error", true);
if (!terminated.isReady()) {
tLogData->bytesDurable += bytesInput.getValue() - bytesDurable.getValue();
TraceEvent("TLogBytesWhenRemoved", logId)
.detail("SharedBytesInput", tLogData->bytesInput)
.detail("SharedBytesDurable", tLogData->bytesDurable)
.detail("LocalBytesInput", bytesInput.getValue())
.detail("LocalBytesDurable", bytesDurable.getValue());
ASSERT_ABORT(tLogData->bytesDurable <= tLogData->bytesInput);
Key logIdKey = BinaryWriter::toValue(logId, Unversioned());
tLogData->persistentData->clear(singleKeyRange(logIdKey.withPrefix(persistCurrentVersionKeys.begin)));
tLogData->persistentData->clear(
singleKeyRange(logIdKey.withPrefix(persistKnownCommittedVersionKeys.begin)));
tLogData->persistentData->clear(singleKeyRange(logIdKey.withPrefix(persistLocalityKeys.begin)));
tLogData->persistentData->clear(singleKeyRange(logIdKey.withPrefix(persistLogRouterTagsKeys.begin)));
tLogData->persistentData->clear(singleKeyRange(logIdKey.withPrefix(persistTxsTagsKeys.begin)));
tLogData->persistentData->clear(singleKeyRange(logIdKey.withPrefix(persistRecoveryCountKeys.begin)));
tLogData->persistentData->clear(singleKeyRange(logIdKey.withPrefix(persistProtocolVersionKeys.begin)));
tLogData->persistentData->clear(singleKeyRange(logIdKey.withPrefix(persistTLogSpillTypeKeys.begin)));
tLogData->persistentData->clear(singleKeyRange(logIdKey.withPrefix(persistRecoveryLocationKey)));
Key msgKey = logIdKey.withPrefix(persistTagMessagesKeys.begin);
tLogData->persistentData->clear(KeyRangeRef(msgKey, strinc(msgKey)));
Key msgRefKey = logIdKey.withPrefix(persistTagMessageRefsKeys.begin);
tLogData->persistentData->clear(KeyRangeRef(msgRefKey, strinc(msgRefKey)));
Key poppedKey = logIdKey.withPrefix(persistTagPoppedKeys.begin);
tLogData->persistentData->clear(KeyRangeRef(poppedKey, strinc(poppedKey)));
}
for (auto it = peekTracker.begin(); it != peekTracker.end(); ++it) {
for (auto seq : it->second.sequence_version) {
if (!seq.second.isSet()) {
seq.second.sendError(timed_out());
Fix parallel peek stalling for 10min when a TLog generation is destroyed. `peekTracker` was held on the Shared TLog (TLogData), whereas peeks are received and replied to as part of a TLog instance (LogData). When a peek was received on a TLog, it was registered into peekTracker along with the ReplyPromise. If the TLog was then removed as part of a no-longer-needed generation of TLogs, there is nothing left to reply to the request, but by holding onto the ReplyPromise in peekTracker, we leave the remote end with an expectation that we will reply. Then, 10min later, peekTrackerCleanup runs and finally times out the peek cursor, thus preventing FDB from being completely stuck. Now, each TLog generation has its own `peekTracker`, and when a TLog is destroyed, it times out all of the pending peek curors that are still expecting a response. This will then trigger the client to re-issue them to the next generation of TLogs, thus removing the 10min gap to do so.
2019-07-10 08:27:36 +08:00
}
}
}
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
LogEpoch epoch() const { return recoveryCount; }
Make TLogServer able to spill by value and by reference ...and test it in simulation, but not combined yet. It turns out that because of txsTag, we basically had to support spill-by-value anyway. Thus, if we treat all tags like txsTag when spilling and peeking, then we have an easy way to bring the two spilling types back into one implementation.
2019-10-03 16:25:42 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
bool shouldSpillByValue(Tag t) const {
Make TLogServer able to spill by value and by reference ...and test it in simulation, but not combined yet. It turns out that because of txsTag, we basically had to support spill-by-value anyway. Thus, if we treat all tags like txsTag when spilling and peeking, then we have an easy way to bring the two spilling types back into one implementation.
2019-10-03 16:25:42 +08:00
switch (logSpillType) {
case TLogSpillType::VALUE:
return true;
case TLogSpillType::REFERENCE:
return t.locality == tagLocalityTxs || t == txsTag;
default:
ASSERT(false);
return false;
}
}
Code review nits const correctness and file renaming in comment. Co-Authored-By: Jingyu Zhou <jingyuzhou@gmail.com>
2019-10-04 06:53:39 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
bool shouldSpillByReference(Tag t) const { return !shouldSpillByValue(t); }
Initial repository commit
2017-05-26 04:48:44 +08:00
};
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
template <class T>
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void TLogQueue::push(T const& qe, Reference<LogData> logData) {
BinaryWriter wr(Unversioned()); // outer framing is not versioned
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
wr << uint32_t(0);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
IncludeVersion(ProtocolVersion::withTLogQueueEntryRef()).write(wr); // payload is versioned
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
wr << qe;
wr << uint8_t(1);
*(uint32_t*)wr.getData() = wr.getLength() - sizeof(uint32_t) - sizeof(uint8_t);
Track both start and end versions in versionLocation.
2019-02-08 09:02:31 +08:00
const IDiskQueue::location startloc = queue->getNextPushLocation();
// FIXME: push shouldn't return anything. We should call getNextPushLocation() again.
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
const IDiskQueue::location endloc = queue->push(wr.toValue());
Revert "Make the sim2 run loop match the behavior of the net2 run loop."
2021-06-23 05:50:01 +08:00
//TraceEvent("TLogQueueVersionWritten", dbgid).detail("Size", wr.getLength() - sizeof(uint32_t) - sizeof(uint8_t)).detail("Loc", loc);
Track both start and end versions in versionLocation.
2019-02-08 09:02:31 +08:00
logData->versionLocation[qe.version] = std::make_pair(startloc, endloc);
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
}
Implement proper cleanup of disk queue when spilling refs.
2019-02-08 09:02:37 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void TLogQueue::forgetBefore(Version upToVersion, Reference<LogData> logData) {
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
// Keep only the given and all subsequent version numbers
// Find the first version >= upTo
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
auto v = logData->versionLocation.lower_bound(upToVersion);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (v == logData->versionLocation.begin())
return;
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (v == logData->versionLocation.end()) {
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
v = logData->versionLocation.lastItem();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
} else {
Remove random bits of code that were either unneeded or leftover from debugging.
2019-03-16 12:01:23 +08:00
v.decrementNonEnd();
}
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->versionLocation.erase(logData->versionLocation.begin(),
v); // ... and then we erase that previous version and all prior versions
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
}
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void TLogQueue::pop(IDiskQueue::location upToLocation) {
queue->pop(upToLocation);
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void TLogQueue::updateVersionSizes(const TLogQueueEntry& result,
TLogData* tLog,
IDiskQueue::location start,
IDiskQueue::location end) {
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
auto it = tLog->id_data.find(result.id);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (it != tLog->id_data.end()) {
Track both start and end versions in versionLocation.
2019-02-08 09:02:31 +08:00
it->second->versionLocation[result.version] = std::make_pair(start, end);
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> tLogLock(TLogData* self, ReplyPromise<TLogLockResult> reply, Reference<LogData> logData) {
Initial repository commit
2017-05-26 04:48:44 +08:00
state Version stopVersion = logData->version.get();
TEST(true); // TLog stopped by recovering master
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TEST(logData->stopped); // logData already stopped
TEST(!logData->stopped); // logData not yet stopped
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TraceEvent("TLogStop", logData->logId)
.detail("Ver", stopVersion)
.detail("IsStopped", logData->stopped)
.detail("QueueCommitted", logData->queueCommittedVersion.get());
Initial repository commit
2017-05-26 04:48:44 +08:00
logData->stopped = true;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!logData->recoveryComplete.isSet()) {
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
logData->recoveryComplete.sendError(end_of_stream());
fix: the tlog did not cancel recovery properly when stopped
2018-01-13 09:18:14 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
// Lock once the current version has been committed
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(logData->queueCommittedVersion.whenAtLeast(stopVersion));
Initial repository commit
2017-05-26 04:48:44 +08:00
ASSERT(stopVersion == logData->version.get());
TLogLockResult result;
result.end = stopVersion;
result.knownCommittedVersion = logData->knownCommittedVersion;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TraceEvent("TLogStop2", self->dbgid)
.detail("LogId", logData->logId)
.detail("Ver", stopVersion)
.detail("IsStopped", logData->stopped)
.detail("QueueCommitted", logData->queueCommittedVersion.get())
.detail("KnownCommitted", result.knownCommittedVersion);
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
reply.send(result);
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void updatePersistentPopped(TLogData* self, Reference<LogData> logData, Reference<LogData::TagData> data) {
if (!data->poppedRecently)
return;
self->persistentData->set(
KeyValueRef(persistTagPoppedKey(logData->logId, data->tag), persistTagPoppedValue(data->popped)));
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
data->poppedRecently = false;
DiskQueue should only pop based off of persisted popped tag versions This commit is to fix a bug where popping a tag between updatePersistentData and popDiskQueue can cause the TLog to recover to an incorrect understanding of what data it has available. The following series of events need to happen to trigger this bug: Tag 1:1 is popped to version 10 updatePersistentData is run... updatePersistentPopped runs and we persistentData stores 1:1 as popped to 10 A mutation is spilled for 1:1 at version 11 at location 1000 A mutation is spilled for 1:1 at version 21 at location 5000 updatePersistentData finishes and commits the btree changes Tag 1:1 is popped to version 20 popDiskQueue runs The btree is read for spilled mutations with version >=20 The minimum location required for the disk queue is found to be location 5000 The disk queue is popped to location 5000 The TLog crashes The worker restarts, and reloads the TLog files from disk restorePersistentPopped restores tag 1:1 as having been popped to version 10 Parallel peeks are received for tag 1:1 starting at version 0 The first peek is less than the popped version, so we respond with no data, and an end version of 10 The second peek starts at version 10, which is greater than the popped version The btree is read for spilled mutations, and we find that there is a mutation at version 11 at location 1000 Location 1000 is read in the DiskQueue The resulting page read at Location 1000 was popped pre-crash, and thus might either (a) be corrupt or (b) have an incorrect sequence number. The fix to this is to force popDiskQueue/updatePoppedLocation to use the popped version that was persisted to disk, and not the most recently popped version for the given tag. This bug doesn't manifest in simulation, because we don't have any code that peeks at a lower version than what has been popped.
2019-12-13 09:16:22 +08:00
data->persistentPopped = data->popped;
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (data->nothingPersistent)
return;
Initial repository commit
2017-05-26 04:48:44 +08:00
Make TLogServer able to spill by value and by reference ...and test it in simulation, but not combined yet. It turns out that because of txsTag, we basically had to support spill-by-value anyway. Thus, if we treat all tags like txsTag when spilling and peeking, then we have an easy way to bring the two spilling types back into one implementation.
2019-10-03 16:25:42 +08:00
if (logData->shouldSpillByValue(data->tag)) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentData->clear(KeyRangeRef(persistTagMessagesKey(logData->logId, data->tag, Version(0)),
persistTagMessagesKey(logData->logId, data->tag, data->popped)));
Spill txsTag by value under TagMsg/ and not TagMsgRef/ There's not a tremendous reason as to why this matters now, but I feel like I might regret sometime later not keeping the same schema under the same key.
2019-03-04 09:01:52 +08:00
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentData->clear(KeyRangeRef(persistTagMessageRefsKey(logData->logId, data->tag, Version(0)),
persistTagMessageRefsKey(logData->logId, data->tag, data->popped)));
Spill txsTag by value under TagMsg/ and not TagMsgRef/ There's not a tremendous reason as to why this matters now, but I feel like I might regret sometime later not keeping the same schema under the same key.
2019-03-04 09:01:52 +08:00
}
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
if (data->popped > logData->persistentDataVersion) {
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
data->nothingPersistent = true;
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> updatePoppedLocation(TLogData* self, Reference<LogData> logData, Reference<LogData::TagData> data) {
Make TLogServer able to spill by value and by reference ...and test it in simulation, but not combined yet. It turns out that because of txsTag, we basically had to support spill-by-value anyway. Thus, if we treat all tags like txsTag when spilling and peeking, then we have an easy way to bring the two spilling types back into one implementation.
2019-10-03 16:25:42 +08:00
// For anything spilled by value, we do not need to track its popped location.
if (logData->shouldSpillByValue(data->tag)) {
Remove random bits of code that were either unneeded or leftover from debugging.
2019-03-16 12:01:23 +08:00
return Void();
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (data->versionForPoppedLocation >= data->persistentPopped)
return Void();
Rename poppedLocationForVersion -> versionForPoppedLocation
2019-12-20 02:24:31 +08:00
data->versionForPoppedLocation = data->persistentPopped;
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
DiskQueue should only pop based off of persisted popped tag versions This commit is to fix a bug where popping a tag between updatePersistentData and popDiskQueue can cause the TLog to recover to an incorrect understanding of what data it has available. The following series of events need to happen to trigger this bug: Tag 1:1 is popped to version 10 updatePersistentData is run... updatePersistentPopped runs and we persistentData stores 1:1 as popped to 10 A mutation is spilled for 1:1 at version 11 at location 1000 A mutation is spilled for 1:1 at version 21 at location 5000 updatePersistentData finishes and commits the btree changes Tag 1:1 is popped to version 20 popDiskQueue runs The btree is read for spilled mutations with version >=20 The minimum location required for the disk queue is found to be location 5000 The disk queue is popped to location 5000 The TLog crashes The worker restarts, and reloads the TLog files from disk restorePersistentPopped restores tag 1:1 as having been popped to version 10 Parallel peeks are received for tag 1:1 starting at version 0 The first peek is less than the popped version, so we respond with no data, and an end version of 10 The second peek starts at version 10, which is greater than the popped version The btree is read for spilled mutations, and we find that there is a mutation at version 11 at location 1000 Location 1000 is read in the DiskQueue The resulting page read at Location 1000 was popped pre-crash, and thus might either (a) be corrupt or (b) have an incorrect sequence number. The fix to this is to force popDiskQueue/updatePoppedLocation to use the popped version that was persisted to disk, and not the most recently popped version for the given tag. This bug doesn't manifest in simulation, because we don't have any code that peeks at a lower version than what has been popped.
2019-12-13 09:16:22 +08:00
// Use persistentPopped and not popped, so that a pop update received after spilling doesn't cause
// us to remove data that still is pointed to by SpilledData in the btree.
if (data->persistentPopped <= logData->persistentDataVersion) {
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
// Recover the next needed location in the Disk Queue from the index.
Create RangeResult type alias
2021-05-04 04:14:16 +08:00
RangeResult kvrefs = wait(self->persistentData->readRange(
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
KeyRangeRef(persistTagMessageRefsKey(logData->logId, data->tag, data->persistentPopped),
persistTagMessageRefsKey(logData->logId, data->tag, logData->persistentDataVersion + 1)),
1));
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
if (kvrefs.empty()) {
// Nothing was persistent after all.
data->nothingPersistent = true;
} else {
VectorRef<SpilledData> spilledData;
BinaryReader r(kvrefs[0].value, AssumeVersion(logData->protocolVersion));
r >> spilledData;
for (const SpilledData& sd : spilledData) {
DiskQueue should only pop based off of persisted popped tag versions This commit is to fix a bug where popping a tag between updatePersistentData and popDiskQueue can cause the TLog to recover to an incorrect understanding of what data it has available. The following series of events need to happen to trigger this bug: Tag 1:1 is popped to version 10 updatePersistentData is run... updatePersistentPopped runs and we persistentData stores 1:1 as popped to 10 A mutation is spilled for 1:1 at version 11 at location 1000 A mutation is spilled for 1:1 at version 21 at location 5000 updatePersistentData finishes and commits the btree changes Tag 1:1 is popped to version 20 popDiskQueue runs The btree is read for spilled mutations with version >=20 The minimum location required for the disk queue is found to be location 5000 The disk queue is popped to location 5000 The TLog crashes The worker restarts, and reloads the TLog files from disk restorePersistentPopped restores tag 1:1 as having been popped to version 10 Parallel peeks are received for tag 1:1 starting at version 0 The first peek is less than the popped version, so we respond with no data, and an end version of 10 The second peek starts at version 10, which is greater than the popped version The btree is read for spilled mutations, and we find that there is a mutation at version 11 at location 1000 Location 1000 is read in the DiskQueue The resulting page read at Location 1000 was popped pre-crash, and thus might either (a) be corrupt or (b) have an incorrect sequence number. The fix to this is to force popDiskQueue/updatePoppedLocation to use the popped version that was persisted to disk, and not the most recently popped version for the given tag. This bug doesn't manifest in simulation, because we don't have any code that peeks at a lower version than what has been popped.
2019-12-13 09:16:22 +08:00
if (sd.version >= data->persistentPopped) {
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
data->poppedLocation = sd.start;
Rename poppedLocationForVersion -> versionForPoppedLocation
2019-12-20 02:24:31 +08:00
data->versionForPoppedLocation = sd.version;
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
break;
}
}
}
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
}
DiskQueue should only pop based off of persisted popped tag versions This commit is to fix a bug where popping a tag between updatePersistentData and popDiskQueue can cause the TLog to recover to an incorrect understanding of what data it has available. The following series of events need to happen to trigger this bug: Tag 1:1 is popped to version 10 updatePersistentData is run... updatePersistentPopped runs and we persistentData stores 1:1 as popped to 10 A mutation is spilled for 1:1 at version 11 at location 1000 A mutation is spilled for 1:1 at version 21 at location 5000 updatePersistentData finishes and commits the btree changes Tag 1:1 is popped to version 20 popDiskQueue runs The btree is read for spilled mutations with version >=20 The minimum location required for the disk queue is found to be location 5000 The disk queue is popped to location 5000 The TLog crashes The worker restarts, and reloads the TLog files from disk restorePersistentPopped restores tag 1:1 as having been popped to version 10 Parallel peeks are received for tag 1:1 starting at version 0 The first peek is less than the popped version, so we respond with no data, and an end version of 10 The second peek starts at version 10, which is greater than the popped version The btree is read for spilled mutations, and we find that there is a mutation at version 11 at location 1000 Location 1000 is read in the DiskQueue The resulting page read at Location 1000 was popped pre-crash, and thus might either (a) be corrupt or (b) have an incorrect sequence number. The fix to this is to force popDiskQueue/updatePoppedLocation to use the popped version that was persisted to disk, and not the most recently popped version for the given tag. This bug doesn't manifest in simulation, because we don't have any code that peeks at a lower version than what has been popped.
2019-12-13 09:16:22 +08:00
if (data->persistentPopped >= logData->persistentDataVersion || data->nothingPersistent) {
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
// Then the location must be in memory.
DiskQueue should only pop based off of persisted popped tag versions This commit is to fix a bug where popping a tag between updatePersistentData and popDiskQueue can cause the TLog to recover to an incorrect understanding of what data it has available. The following series of events need to happen to trigger this bug: Tag 1:1 is popped to version 10 updatePersistentData is run... updatePersistentPopped runs and we persistentData stores 1:1 as popped to 10 A mutation is spilled for 1:1 at version 11 at location 1000 A mutation is spilled for 1:1 at version 21 at location 5000 updatePersistentData finishes and commits the btree changes Tag 1:1 is popped to version 20 popDiskQueue runs The btree is read for spilled mutations with version >=20 The minimum location required for the disk queue is found to be location 5000 The disk queue is popped to location 5000 The TLog crashes The worker restarts, and reloads the TLog files from disk restorePersistentPopped restores tag 1:1 as having been popped to version 10 Parallel peeks are received for tag 1:1 starting at version 0 The first peek is less than the popped version, so we respond with no data, and an end version of 10 The second peek starts at version 10, which is greater than the popped version The btree is read for spilled mutations, and we find that there is a mutation at version 11 at location 1000 Location 1000 is read in the DiskQueue The resulting page read at Location 1000 was popped pre-crash, and thus might either (a) be corrupt or (b) have an incorrect sequence number. The fix to this is to force popDiskQueue/updatePoppedLocation to use the popped version that was persisted to disk, and not the most recently popped version for the given tag. This bug doesn't manifest in simulation, because we don't have any code that peeks at a lower version than what has been popped.
2019-12-13 09:16:22 +08:00
auto locationIter = logData->versionLocation.lower_bound(data->persistentPopped);
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
if (locationIter != logData->versionLocation.end()) {
data->poppedLocation = locationIter->value.first;
Rename poppedLocationForVersion -> versionForPoppedLocation
2019-12-20 02:24:31 +08:00
data->versionForPoppedLocation = locationIter->key;
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
} else {
// No data on disk and no data in RAM.
// This TLog instance will be removed soon anyway, so we temporarily freeze our poppedLocation
// to avoid trying to track what the ending location of this TLog instance was.
}
}
return Void();
}
Add Alex comment on tLog
2020-11-13 05:29:11 +08:00
// It runs against the oldest TLog instance, calculates the first location in the disk queue that contains un-popped
// data, and then issues a pop to the disk queue at that location so that anything earlier can be
// removed/forgotten/overwritten. In effect, it applies the effect of TLogPop RPCs to disk.
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> popDiskQueue(TLogData* self, Reference<LogData> logData) {
if (!logData->initialized)
return Void();
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
std::vector<Future<Void>> updates;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (int tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
for (int tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
Reference<LogData::TagData> tagData = logData->tag_data[tagLocality][tagId];
if (tagData) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
updates.push_back(updatePoppedLocation(self, logData, tagData));
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
}
}
}
wait(waitForAll(updates));
fix: the tlog would not pop data from the disk queue after a storage server was removed, because the tag still exists in memory on the logs fix: we could incorrectly make data durable if eraseMessagesFromMemory was in progress while running updatePersistentData the quiet database check now ensure that tlogs have no more than 30 seconds of versions unpopped from the disk queue
2019-05-21 14:58:45 +08:00
IDiskQueue::location minLocation = 0;
Version minVersion = 0;
fix: do not pop the disk queue past the persistentDataVersion
2019-05-22 01:40:30 +08:00
auto locationIter = logData->versionLocation.lower_bound(logData->persistentDataVersion);
if (locationIter != logData->versionLocation.end()) {
minLocation = locationIter->value.first;
minVersion = locationIter->key;
fix: the tlog would not pop data from the disk queue after a storage server was removed, because the tag still exists in memory on the logs fix: we could incorrectly make data durable if eraseMessagesFromMemory was in progress while running updatePersistentData the quiet database check now ensure that tlogs have no more than 30 seconds of versions unpopped from the disk queue
2019-05-21 14:58:45 +08:00
}
added logging on the TLog for the tag with smallest popped version
2019-09-13 07:22:01 +08:00
logData->minPoppedTagVersion = std::numeric_limits<Version>::max();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (int tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
for (int tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
Reference<LogData::TagData> tagData = logData->tag_data[tagLocality][tagId];
Make TLogServer able to spill by value and by reference ...and test it in simulation, but not combined yet. It turns out that because of txsTag, we basically had to support spill-by-value anyway. Thus, if we treat all tags like txsTag when spilling and peeking, then we have an easy way to bring the two spilling types back into one implementation.
2019-10-03 16:25:42 +08:00
if (tagData && logData->shouldSpillByReference(tagData->tag)) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!tagData->nothingPersistent) {
added logging on the TLog for the tag with smallest popped version
2019-09-13 07:22:01 +08:00
minLocation = std::min(minLocation, tagData->poppedLocation);
minVersion = std::min(minVersion, tagData->popped);
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if ((!tagData->nothingPersistent || tagData->versionMessages.size()) &&
tagData->popped < logData->minPoppedTagVersion) {
added logging on the TLog for the tag with smallest popped version
2019-09-13 07:22:01 +08:00
logData->minPoppedTagVersion = tagData->popped;
logData->minPoppedTag = tagData->tag;
}
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
}
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (self->queueCommitEnd.get() > 0) {
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
Version lastCommittedVersion = logData->queueCommittedVersion.get();
IDiskQueue::location lastCommittedLocation = minLocation;
auto locationIter = logData->versionLocation.lower_bound(lastCommittedVersion);
if (locationIter != logData->versionLocation.end()) {
lastCommittedLocation = locationIter->value.first;
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentQueue->pop(std::min(minLocation, lastCommittedLocation));
fix: the tlog would not pop data from the disk queue after a storage server was removed, because the tag still exists in memory on the logs fix: we could incorrectly make data durable if eraseMessagesFromMemory was in progress while running updatePersistentData the quiet database check now ensure that tlogs have no more than 30 seconds of versions unpopped from the disk queue
2019-05-21 14:58:45 +08:00
logData->queuePoppedVersion = std::max(logData->queuePoppedVersion, minVersion);
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
}
return Void();
}
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> updatePersistentData(TLogData* self, Reference<LogData> logData, Version newPersistentDataVersion) {
state BinaryWriter wr(Unversioned());
Initial repository commit
2017-05-26 04:48:44 +08:00
// PERSIST: Changes self->persistentDataVersion and writes and commits the relevant changes
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ASSERT(newPersistentDataVersion <= logData->version.get());
ASSERT(newPersistentDataVersion <= logData->queueCommittedVersion.get());
ASSERT(newPersistentDataVersion > logData->persistentDataVersion);
ASSERT(logData->persistentDataVersion == logData->persistentDataDurableVersion);
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
logData->newPersistentDataVersion = newPersistentDataVersion;
Initial repository commit
2017-05-26 04:48:44 +08:00
Attempt to normalize trace events: * Detail names now all start with an uppercase character and contain no underscores. Ideally these should be head-first camel case, though that was harder to check. * Type names have the same rules, except they allow one underscore (to support a usage pattern Context_Type). The first character after the underscore is also uppercase. * Use seconds instead of milliseconds in details. Added a check when events are logged in simulation that logs a message to stderr if the first two rules above aren't followed. This probably doesn't address every instance of the above problems, but all of the events I was able to hit in simulation pass the check.
2018-06-09 02:11:08 +08:00
//TraceEvent("UpdatePersistentData", self->dbgid).detail("Seq", newPersistentDataSeq);
Initial repository commit
2017-05-26 04:48:44 +08:00
state bool anyData = false;
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
Initial repository commit
2017-05-26 04:48:44 +08:00
// For all existing tags
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
state int tagLocality = 0;
state int tagId = 0;
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
for (tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
state Reference<LogData::TagData> tagData = logData->tag_data[tagLocality][tagId];
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (tagData) {
wait(tagData->eraseMessagesBefore(tagData->popped, self, logData, TaskPriority::UpdateStorage));
optimized tag lookups on the tlog by removing one level of vectors
2018-03-21 01:41:42 +08:00
state Version currentVersion = 0;
// Clear recently popped versions from persistentData if necessary
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
updatePersistentPopped(self, logData, tagData);
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
state Version lastVersion = std::numeric_limits<Version>::min();
state IDiskQueue::location firstLocation = std::numeric_limits<IDiskQueue::location>::max();
optimized tag lookups on the tlog by removing one level of vectors
2018-03-21 01:41:42 +08:00
// Transfer unpopped messages with version numbers less than newPersistentDataVersion to persistentData
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
state std::deque<std::pair<Version, LengthPrefixedStringRef>>::iterator msg =
tagData->versionMessages.begin();
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
state int refSpilledTagCount = 0;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wr = BinaryWriter(AssumeVersion(logData->protocolVersion));
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
// We prefix our spilled locations with a count, so that we can read this back out as a VectorRef.
wr << uint32_t(0);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
while (msg != tagData->versionMessages.end() && msg->first <= newPersistentDataVersion) {
optimized tag lookups on the tlog by removing one level of vectors
2018-03-21 01:41:42 +08:00
currentVersion = msg->first;
anyData = true;
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
tagData->nothingPersistent = false;
optimized tag lookups on the tlog by removing one level of vectors
2018-03-21 01:41:42 +08:00
Make TLogServer able to spill by value and by reference ...and test it in simulation, but not combined yet. It turns out that because of txsTag, we basically had to support spill-by-value anyway. Thus, if we treat all tags like txsTag when spilling and peeking, then we have an easy way to bring the two spilling types back into one implementation.
2019-10-03 16:25:42 +08:00
if (logData->shouldSpillByValue(tagData->tag)) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wr = BinaryWriter(Unversioned());
for (; msg != tagData->versionMessages.end() && msg->first == currentVersion; ++msg) {
Spill txsTag by value
2019-03-04 04:57:33 +08:00
wr << msg->second.toStringRef();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentData->set(KeyValueRef(
persistTagMessagesKey(logData->logId, tagData->tag, currentVersion), wr.toValue()));
Spill txsTag by value
2019-03-04 04:57:33 +08:00
} else {
// spill everything else by reference
const IDiskQueue::location begin = logData->versionLocation[currentVersion].first;
const IDiskQueue::location end = logData->versionLocation[currentVersion].second;
Spill (start,length) instead of (begin,end) to save a few bytes.
2019-03-04 04:57:41 +08:00
ASSERT(end > begin && end.lo - begin.lo < std::numeric_limits<uint32_t>::max());
uint32_t length = static_cast<uint32_t>(end.lo - begin.lo);
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
refSpilledTagCount++;
Solve overreading via pre-calculating tag bytes per commit
2019-03-04 04:57:36 +08:00
Checksum DiskQueue pages on read, but at a lower priority. If a server has its data spilled, then it's behind the 5s window. Feeding it data is less important than committing, so we can hide the extra CPU usage from checksumming the read amplified disk queue pages.
2019-03-16 12:01:19 +08:00
uint32_t size = 0;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (; msg != tagData->versionMessages.end() && msg->first == currentVersion; ++msg) {
Solve overreading via pre-calculating tag bytes per commit
2019-03-04 04:57:36 +08:00
// Fast forward until we find a new version.
size += msg->second.expectedSize();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
SpilledData spilledData(currentVersion, begin, length, size);
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
wr << spilledData;
Spill txsTag by value
2019-03-04 04:57:33 +08:00
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
lastVersion = std::max(currentVersion, lastVersion);
firstLocation = std::min(begin, firstLocation);
Spill txsTag by value
2019-03-04 04:57:33 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if ((wr.getLength() + sizeof(SpilledData) >
SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH)) {
Add a knob to limit the number of commits indexed per key. Theoretically, we could spill 20MB of 22B mutations for one key, which would generate a very long value being stored in SQLite, and very inefficiently read back. This stops that from being a problem, at the cost of some extra write calls.
2019-05-04 03:48:28 +08:00
*(uint32_t*)wr.getData() = refSpilledTagCount;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentData->set(KeyValueRef(
persistTagMessageRefsKey(logData->logId, tagData->tag, lastVersion), wr.toValue()));
Add a knob to limit the number of commits indexed per key. Theoretically, we could spill 20MB of 22B mutations for one key, which would generate a very long value being stored in SQLite, and very inefficiently read back. This stops that from being a problem, at the cost of some extra write calls.
2019-05-04 03:48:28 +08:00
tagData->poppedLocation = std::min(tagData->poppedLocation, firstLocation);
refSpilledTagCount = 0;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wr = BinaryWriter(AssumeVersion(logData->protocolVersion));
Add a knob to limit the number of commits indexed per key. Theoretically, we could spill 20MB of 22B mutations for one key, which would generate a very long value being stored in SQLite, and very inefficiently read back. This stops that from being a problem, at the cost of some extra write calls.
2019-05-04 03:48:28 +08:00
wr << uint32_t(0);
}
A giant translation of TaskFooPriority -> TaskPriority::Foo This is so that APIs that take priorities don't take ints, which are common and easy to accidentally pass the wrong thing.
2019-06-25 17:47:35 +08:00
Future<Void> f = yield(TaskPriority::UpdateStorage);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!f.isReady()) {
Solve overreading via pre-calculating tag bytes per commit
2019-03-04 04:57:36 +08:00
wait(f);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
msg = std::upper_bound(tagData->versionMessages.begin(),
tagData->versionMessages.end(),
std::make_pair(currentVersion, LengthPrefixedStringRef()),
CompareFirst<std::pair<Version, LengthPrefixedStringRef>>());
Spill txsTag by value
2019-03-04 04:57:33 +08:00
}
Remove all code related to writing and cleaning up old-style spilling.
2019-02-08 09:02:39 +08:00
}
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
}
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
if (refSpilledTagCount > 0) {
*(uint32_t*)wr.getData() = refSpilledTagCount;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentData->set(
KeyValueRef(persistTagMessageRefsKey(logData->logId, tagData->tag, lastVersion), wr.toValue()));
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
tagData->poppedLocation = std::min(tagData->poppedLocation, firstLocation);
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
}
optimized tag lookups on the tlog by removing one level of vectors
2018-03-21 01:41:42 +08:00
A giant translation of TaskFooPriority -> TaskPriority::Foo This is so that APIs that take priorities don't take ints, which are common and easy to accidentally pass the wrong thing.
2019-06-25 17:47:35 +08:00
wait(yield(TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
}
Temporarily add verification of spilled tag data.
2019-02-08 09:02:34 +08:00
Implement limiting how many bytes recovery will read. This time, track what location in the DiskQueue has been spilled in persistent state, and then feed it back into the disk queue before recovery. This also introduces an ASSERT that recovery only reads exactly the bytes that it needs to have in memory.
2019-03-16 12:01:22 +08:00
auto locationIter = logData->versionLocation.lower_bound(newPersistentDataVersion);
if (locationIter != logData->versionLocation.end()) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentData->set(
KeyValueRef(persistRecoveryLocationKey, BinaryWriter::toValue(locationIter->value.first, Unversioned())));
Implement limiting how many bytes recovery will read. This time, track what location in the DiskQueue has been spilled in persistent state, and then feed it back into the disk queue before recovery. This also introduces an ASSERT that recovery only reads exactly the bytes that it needs to have in memory.
2019-03-16 12:01:22 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentData->set(
KeyValueRef(BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistCurrentVersionKeys.begin),
BinaryWriter::toValue(newPersistentDataVersion, Unversioned())));
self->persistentData->set(KeyValueRef(
BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistKnownCommittedVersionKeys.begin),
BinaryWriter::toValue(logData->knownCommittedVersion, Unversioned())));
Initial repository commit
2017-05-26 04:48:44 +08:00
logData->persistentDataVersion = newPersistentDataVersion;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(self->persistentData->commit()); // SOMEDAY: This seems to be running pretty often, should we slow it down???
wait(delay(0, TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// Now that the changes we made to persistentData are durable, erase the data we moved from memory and the queue,
// increase bytesDurable accordingly, and update persistentDataDurableVersion.
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TEST(anyData); // TLog moved data to persistentData
Initial repository commit
2017-05-26 04:48:44 +08:00
logData->persistentDataDurableVersion = newPersistentDataVersion;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
for (tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
if (logData->tag_data[tagLocality][tagId]) {
wait(logData->tag_data[tagLocality][tagId]->eraseMessagesBefore(
newPersistentDataVersion + 1, self, logData, TaskPriority::UpdateStorage));
A giant translation of TaskFooPriority -> TaskPriority::Foo This is so that APIs that take priorities don't take ints, which are common and easy to accidentally pass the wrong thing.
2019-06-25 17:47:35 +08:00
wait(yield(TaskPriority::UpdateStorage));
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
}
}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->version_sizes.erase(logData->version_sizes.begin(),
logData->version_sizes.lower_bound(logData->persistentDataDurableVersion));
Initial repository commit
2017-05-26 04:48:44 +08:00
A giant translation of TaskFooPriority -> TaskPriority::Foo This is so that APIs that take priorities don't take ints, which are common and easy to accidentally pass the wrong thing.
2019-06-25 17:47:35 +08:00
wait(yield(TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
while (!logData->messageBlocks.empty() && logData->messageBlocks.front().first <= newPersistentDataVersion) {
int64_t bytesErased =
int64_t(logData->messageBlocks.front().second.size()) * SERVER_KNOBS->TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR;
add the same number for shared metrics and local metrics
2017-06-29 06:21:54 +08:00
logData->bytesDurable += bytesErased;
self->bytesDurable += bytesErased;
Initial repository commit
2017-05-26 04:48:44 +08:00
logData->messageBlocks.pop_front();
A giant translation of TaskFooPriority -> TaskPriority::Foo This is so that APIs that take priorities don't take ints, which are common and easy to accidentally pass the wrong thing.
2019-06-25 17:47:35 +08:00
wait(yield(TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->bytesDurable.getValue() > logData->bytesInput.getValue() || self->bytesDurable > self->bytesInput) {
TraceEvent(SevError, "BytesDurableTooLarge", logData->logId)
.detail("SharedBytesInput", self->bytesInput)
.detail("SharedBytesDurable", self->bytesDurable)
.detail("LocalBytesInput", logData->bytesInput.getValue())
.detail("LocalBytesDurable", logData->bytesDurable.getValue());
added additional logging about bytesInput and bytesDurable
2017-06-29 04:29:40 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
ASSERT(logData->bytesDurable.getValue() <= logData->bytesInput.getValue());
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
ASSERT(self->bytesDurable <= self->bytesInput);
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (self->queueCommitEnd.get() > 0) {
Implement proper cleanup of disk queue when spilling refs.
2019-02-08 09:02:37 +08:00
// FIXME: Maintain a heap of tags ordered by version to make this O(1) instead of O(n).
Version minVersion = std::numeric_limits<Version>::max();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
for (tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
Implement proper cleanup of disk queue when spilling refs.
2019-02-08 09:02:37 +08:00
Reference<LogData::TagData> tagData = logData->tag_data[tagLocality][tagId];
if (tagData) {
Make TLogServer able to spill by value and by reference ...and test it in simulation, but not combined yet. It turns out that because of txsTag, we basically had to support spill-by-value anyway. Thus, if we treat all tags like txsTag when spilling and peeking, then we have an easy way to bring the two spilling types back into one implementation.
2019-10-03 16:25:42 +08:00
if (logData->shouldSpillByValue(tagData->tag)) {
Spill txsTag by value
2019-03-04 04:57:33 +08:00
minVersion = std::min(minVersion, newPersistentDataVersion);
} else {
minVersion = std::min(minVersion, tagData->popped);
}
Implement proper cleanup of disk queue when spilling refs.
2019-02-08 09:02:37 +08:00
}
}
}
if (minVersion != std::numeric_limits<Version>::max()) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentQueue->forgetBefore(
newPersistentDataVersion,
logData); // SOMEDAY: this can cause a slow task (~0.5ms), presumably from erasing too many versions.
// Should we limit the number of versions cleared at a time?
Implement proper cleanup of disk queue when spilling refs.
2019-02-08 09:02:37 +08:00
}
}
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
logData->newPersistentDataVersion = invalidVersion;
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> tLogPopCore(TLogData* self, Tag inputTag, Version to, Reference<LogData> logData) {
Merging with Master.
2019-11-13 05:01:29 +08:00
state Version upTo = to;
int8_t tagLocality = inputTag.locality;
WIP: should be divided into smaller commits.
2019-07-24 02:45:04 +08:00
if (isPseudoLocality(tagLocality)) {
if (logData->logSystem->get().isValid()) {
Track pseudo tags popping for individual IDs For each log router ID, we track the popped version of each pseudo tag so that the popping only applied to the minimum of these versions. Also add more tracing for popping and epochs.
2019-08-11 01:31:25 +08:00
upTo = logData->logSystem->get()->popPseudoLocalityTag(inputTag, to);
WIP: should be divided into smaller commits.
2019-07-24 02:45:04 +08:00
tagLocality = tagLocalityLogRouter;
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TraceEvent(SevWarn, "TLogPopNoLogSystem", self->dbgid)
.detail("Locality", tagLocality)
.detail("Version", upTo);
Small fixes.
2019-08-17 12:40:57 +08:00
return Void();
WIP: should be divided into smaller commits.
2019-07-24 02:45:04 +08:00
}
Merging with Master.
2019-11-13 05:01:29 +08:00
}
state Tag tag(tagLocality, inputTag.id);
auto tagData = logData->getTagData(tag);
if (!tagData) {
tagData = logData->createTagData(tag, upTo, true, true, false);
} else if (upTo > tagData->popped) {
tagData->popped = upTo;
tagData->poppedRecently = true;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (tagData->unpoppedRecovered && upTo > logData->recoveredAt) {
Merging with Master.
2019-11-13 05:01:29 +08:00
tagData->unpoppedRecovered = false;
logData->unpoppedRecoveredTags--;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TraceEvent("TLogPoppedTag", logData->logId)
.detail("Tags", logData->unpoppedRecoveredTags)
.detail("Tag", tag.toString())
.detail("DurableKCVer", logData->durableKnownCommittedVersion)
.detail("RecoveredAt", logData->recoveredAt);
if (logData->unpoppedRecoveredTags == 0 && logData->durableKnownCommittedVersion >= logData->recoveredAt &&
logData->recoveryComplete.canBeSet()) {
Merging with Master.
2019-11-13 05:01:29 +08:00
logData->recoveryComplete.send(Void());
}
}
add trace event to tLog pop
2021-07-04 14:56:04 +08:00
uint64_t PoppedVersionLag = logData->persistentDataDurableVersion - logData->queuePoppedVersion;
clean up add_trace_event_to_tLog_pop
2021-07-08 02:36:22 +08:00
if ( SERVER_KNOBS->ENABLE_DETAILED_TLOG_POP_TRACE &&
(logData->queuePoppedVersion > 0) && //avoid generating massive events at beginning
add trace event to tLog pop
2021-07-04 14:56:04 +08:00
(tagData->unpoppedRecovered || PoppedVersionLag >= SERVER_KNOBS->TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE)) { //when recovery or long lag
TraceEvent("TLogPopDetails", logData->logId)
.detail("Tag", tagData->tag.toString())
.detail("UpTo", upTo)
.detail("PoppedVersionLag", PoppedVersionLag)
.detail("MinPoppedTag", logData->minPoppedTag.toString())
.detail("QueuePoppedVersion", logData->queuePoppedVersion)
.detail("UnpoppedRecovered", tagData->unpoppedRecovered ? "True" : "False")
.detail("NothingPersistent", tagData->nothingPersistent ? "True" : "False");
}
Fix some conflicts and remote debugging trace events
2020-07-23 14:35:46 +08:00
if (upTo > logData->persistentDataDurableVersion)
Merging with Master.
2019-11-13 05:01:29 +08:00
wait(tagData->eraseMessagesBefore(upTo, self, logData, TaskPriority::TLogPop));
Comment out frequent TLogPop trace event
2020-01-25 03:01:15 +08:00
//TraceEvent("TLogPop", logData->logId).detail("Tag", tag.toString()).detail("To", upTo);
Merging with Master.
2019-11-13 05:01:29 +08:00
}
return Void();
}
Yield while processing ignored pop requests on tlog
2020-11-27 12:36:02 +08:00
ACTOR Future<Void> processPopRequests(TLogData* self, Reference<LogData> logData) {
state std::vector<Future<Void>> ignoredPops;
state std::map<Tag, Version>::const_iterator it;
state int ignoredPopsPlayed = 0;
Removed TLogData::toBePoppedMutex
2020-12-04 01:36:44 +08:00
state std::map<Tag, Version> toBePopped;
toBePopped = std::move(self->toBePopped);
self->toBePopped.clear();
Yield while processing ignored pop requests on tlog
2020-11-27 12:36:02 +08:00
self->ignorePopRequest = false;
self->ignorePopDeadline = 0.0;
self->ignorePopUid = "";
Removed TLogData::toBePoppedMutex
2020-12-04 01:36:44 +08:00
for (it = toBePopped.cbegin(); it != toBePopped.cend(); ++it) {
Yield while processing ignored pop requests on tlog
2020-11-27 12:36:02 +08:00
const auto& [tag, version] = *it;
TraceEvent("PlayIgnoredPop").detail("Tag", tag.toString()).detail("Version", version);
ignoredPops.push_back(tLogPopCore(self, tag, version, logData));
Added TLOG_POP_BATCH_SIZE knob
2020-12-04 01:40:41 +08:00
if (++ignoredPopsPlayed % SERVER_KNOBS->TLOG_POP_BATCH_SIZE == 0) {
Add code coverage macro to processPopRequests
2020-12-04 01:51:32 +08:00
TEST(true); // Yielding while processing pop requests
Yield while processing ignored pop requests on tlog
2020-11-27 12:36:02 +08:00
wait(yield());
}
}
wait(waitForAll(ignoredPops));
Enforce ignorePopDeadline even when no pop requests are sent
2021-02-22 09:38:29 +08:00
TraceEvent("ResetIgnorePopRequest")
.detail("IgnorePopRequest", self->ignorePopRequest)
.detail("IgnorePopDeadline", self->ignorePopDeadline);
Yield while processing ignored pop requests on tlog
2020-11-27 12:36:02 +08:00
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> tLogPop(TLogData* self, TLogPopRequest req, Reference<LogData> logData) {
Move ignore pop logic from tLogPopCore to tLogPop
2020-12-04 06:23:49 +08:00
if (self->ignorePopRequest) {
TraceEvent(SevDebug, "IgnoringPopRequest").detail("IgnorePopDeadline", self->ignorePopDeadline);
auto& v = self->toBePopped[req.tag];
v = std::max(v, req.to);
TraceEvent(SevDebug, "IgnoringPopRequest")
.detail("IgnorePopDeadline", self->ignorePopDeadline)
.detail("Tag", req.tag.toString())
.detail("Version", req.to);
} else {
wait(tLogPopCore(self, req.tag, req.to, logData));
}
Merging with Master.
2019-11-13 05:01:29 +08:00
req.reply.send(Void());
return Void();
}
Add comments and questions to HA and tLog code reading The comments' correctness need to be confirmed by reviewers.
2020-10-28 00:11:56 +08:00
// This function (and updatePersistentData, which is called by this function) run at a low priority and can soak up all
// CPU resources. For this reason, they employ aggressive use of yields to avoid causing slow tasks that could introduce
// latencies for more important work (e.g. commits).
Add Alex comment on tLog
2020-11-13 05:29:11 +08:00
// This actor is just a loop that calls updatePersistentData and popDiskQueue whenever
// (a) there's data to be spilled or (b) we should update metadata after some commits have been fully popped.
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> updateStorage(TLogData* self) {
while (self->spillOrder.size() && !self->id_data.count(self->spillOrder.front())) {
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
self->spillOrder.pop_front();
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!self->spillOrder.size()) {
wait(delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL
: SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL,
TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
}
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
state Reference<LogData> logData = self->id_data[self->spillOrder.front()];
Merge release-6.0 into master
2018-09-22 07:14:39 +08:00
state Version nextVersion = 0;
Initial repository commit
2017-05-26 04:48:44 +08:00
state int totalSize = 0;
Merge release-6.0 into master
2018-09-22 07:14:39 +08:00
state FlowLock::Releaser commitLockReleaser;
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// FIXME: This policy for calculating the cache pop version could end up popping recent data in the remote DC after
// two consecutive recoveries.
// It also does not protect against spilling the cache tag directly, so it is theoretically possible to spill this
// tag; which is not intended to ever happen.
Merging with Master.
2019-11-13 05:01:29 +08:00
Optional<Version> cachePopVersion;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& it : self->id_data) {
if (!it.second->stopped) {
if (it.second->version.get() - it.second->unrecoveredBefore >
SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT + SERVER_KNOBS->MAX_CACHE_VERSIONS) {
Merging with Master.
2019-11-13 05:01:29 +08:00
cachePopVersion = it.second->version.get() - SERVER_KNOBS->MAX_CACHE_VERSIONS;
}
break;
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (cachePopVersion.present()) {
Merging with Master.
2019-11-13 05:01:29 +08:00
state std::vector<Future<Void>> cachePopFutures;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& it : self->id_data) {
cachePopFutures.push_back(tLogPop(self, TLogPopRequest(cachePopVersion.get(), 0, cacheTag), it.second));
Merging with Master.
2019-11-13 05:01:29 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(waitForAll(cachePopFutures));
Merging with Master.
2019-11-13 05:01:29 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->stopped) {
Spill SharedTLog when there's more than one. When switching between spill_type or log_version, a new instance of a SharedTLog is created in the transaction log processes. If this is done in a saturated database, then doubling the amount of memory to hold mutations in memory can cause TLogs to be uncomfortably close to the 8GB OOM limit. Instead, we now thread which UID of a SharedTLog is active, and the other TLog spill out the majority of their mutations.
2019-07-30 14:40:28 +08:00
if (self->bytesInput - self->bytesDurable >= self->targetVolatileBytes) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
while (logData->persistentDataDurableVersion != logData->version.get()) {
fix: tlog spilling on a stopped log was only making one version durable at a time
2018-08-04 02:38:12 +08:00
totalSize = 0;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Map<Version, std::pair<int, int>>::iterator sizeItr = logData->version_sizes.begin();
fix: nextVersion needs to be set to logData->version if version_sizes is empty
2018-08-05 14:53:37 +08:00
nextVersion = logData->version.get();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
while (totalSize < SERVER_KNOBS->REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT &&
sizeItr != logData->version_sizes.end()) {
significant cpu optimization in update storage
2018-08-05 03:36:48 +08:00
totalSize += sizeItr->value.first + sizeItr->value.second;
++sizeItr;
nextVersion = sizeItr == logData->version_sizes.end() ? logData->version.get() : sizeItr->key;
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(logData->queueCommittedVersion.whenAtLeast(nextVersion));
wait(delay(0, TaskPriority::UpdateStorage));
Fix some conflicts and remote debugging trace events
2020-07-23 14:35:46 +08:00
Attempt to normalize trace events: * Detail names now all start with an uppercase character and contain no underscores. Ideally these should be head-first camel case, though that was harder to check. * Type names have the same rules, except they allow one underscore (to support a usage pattern Context_Type). The first character after the underscore is also uppercase. * Use seconds instead of milliseconds in details. Added a check when events are logged in simulation that logs a message to stderr if the first two rules above aren't followed. This probably doesn't address every instance of the above problems, but all of the events I was able to hit in simulation pass the check.
2018-06-09 02:11:08 +08:00
//TraceEvent("TlogUpdatePersist", self->dbgid).detail("LogId", logData->logId).detail("NextVersion", nextVersion).detail("Version", logData->version.get()).detail("PersistentDataDurableVer", logData->persistentDataDurableVersion).detail("QueueCommitVer", logData->queueCommittedVersion.get()).detail("PersistDataVer", logData->persistentDataVersion);
Initial repository commit
2017-05-26 04:48:44 +08:00
if (nextVersion > logData->persistentDataVersion) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(self->persistentDataCommitLock.take());
Merge release-6.0 into master
2018-09-22 07:14:39 +08:00
commitLockReleaser = FlowLock::Releaser(self->persistentDataCommitLock);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(updatePersistentData(self, logData, nextVersion));
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
// Concurrently with this loop, the last stopped TLog could have been removed.
if (self->popOrder.size()) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(popDiskQueue(self, self->id_data[self->popOrder.front()]));
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
}
Merge release-6.0 into master
2018-09-22 07:14:39 +08:00
commitLockReleaser.release();
Initial repository commit
2017-05-26 04:48:44 +08:00
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL
: SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL,
TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->removed.isReady()) {
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
break;
}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->persistentDataDurableVersion == logData->version.get()) {
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
self->spillOrder.pop_front();
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(delay(0.0, TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL
: SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL,
TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
} else if (logData->initialized) {
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
ASSERT(self->spillOrder.size() == 1);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->version_sizes.empty()) {
fix: the Tlog would not update the durable version properly when version_sizes was empty
2018-11-03 04:05:54 +08:00
nextVersion = logData->version.get();
} else {
Fix a bug that would cause active logs to spill aggressively And add some useful logging about when things do or do not spill.
2019-09-11 06:51:30 +08:00
// Double check that a running TLog wasn't wrongly affected by spilling locked SharedTLogs.
ASSERT_WE_THINK(self->targetVolatileBytes == SERVER_KNOBS->TLOG_SPILL_THRESHOLD);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Map<Version, std::pair<int, int>>::iterator sizeItr = logData->version_sizes.begin();
while (totalSize < SERVER_KNOBS->REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT &&
sizeItr != logData->version_sizes.end() &&
(logData->bytesInput.getValue() - logData->bytesDurable.getValue() - totalSize >=
self->targetVolatileBytes ||
sizeItr->value.first == 0)) {
fix: if a storage server already exists in a remote region after converting to fearless, it did not receive mutations between the known committed version and the recovery version
2018-11-03 05:11:39 +08:00
totalSize += sizeItr->value.first + sizeItr->value.second;
++sizeItr;
nextVersion = sizeItr == logData->version_sizes.end() ? logData->version.get() : sizeItr->key;
}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
Attempt to normalize trace events: * Detail names now all start with an uppercase character and contain no underscores. Ideally these should be head-first camel case, though that was harder to check. * Type names have the same rules, except they allow one underscore (to support a usage pattern Context_Type). The first character after the underscore is also uppercase. * Use seconds instead of milliseconds in details. Added a check when events are logged in simulation that logs a message to stderr if the first two rules above aren't followed. This probably doesn't address every instance of the above problems, but all of the events I was able to hit in simulation pass the check.
2018-06-09 02:11:08 +08:00
//TraceEvent("UpdateStorageVer", logData->logId).detail("NextVersion", nextVersion).detail("PersistentDataVersion", logData->persistentDataVersion).detail("TotalSize", totalSize);
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(logData->queueCommittedVersion.whenAtLeast(nextVersion));
wait(delay(0, TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
if (nextVersion > logData->persistentDataVersion) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(self->persistentDataCommitLock.take());
Merge release-6.0 into master
2018-09-22 07:14:39 +08:00
commitLockReleaser = FlowLock::Releaser(self->persistentDataCommitLock);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(updatePersistentData(self, logData, nextVersion));
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
if (self->popOrder.size()) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(popDiskQueue(self, self->id_data[self->popOrder.front()]));
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
}
Merge release-6.0 into master
2018-09-22 07:14:39 +08:00
commitLockReleaser.release();
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (totalSize < SERVER_KNOBS->REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT) {
wait(delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL
: SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL,
TaskPriority::UpdateStorage));
} else {
// recovery wants to commit to persistant data when updatePersistentData is not active, this delay ensures
// that immediately after updatePersist returns another one has not been started yet.
wait(delay(0.0, TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL
: SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL,
TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> updateStorageLoop(TLogData* self) {
A giant translation of TaskFooPriority -> TaskPriority::Foo This is so that APIs that take priorities don't take ints, which are common and easy to accidentally pass the wrong thing.
2019-06-25 17:47:35 +08:00
wait(delay(0, TaskPriority::UpdateStorage));
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
loop { wait(updateStorage(self)); }
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void commitMessages(TLogData* self,
Reference<LogData> logData,
Version version,
const std::vector<TagsAndMessage>& taggedMessages) {
// SOMEDAY: This method of copying messages is reasonably memory efficient, but it's still a lot of bytes copied.
// Find a way to do the memory allocation right as we receive the messages in the network layer.
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
int64_t addedBytes = 0;
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
int64_t overheadBytes = 0;
significant cpu optimization in update storage
2018-08-05 03:36:48 +08:00
int expectedBytes = 0;
int txsBytes = 0;
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!taggedMessages.size()) {
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
return;
}
int msgSize = 0;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& i : taggedMessages) {
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
msgSize += i.message.size();
}
// Grab the last block in the blocks list so we can share its arena
// We pop all of the elements of it to create a "fresh" vector that starts at the end of the previous vector
Standalone<VectorRef<uint8_t>> block;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->messageBlocks.empty()) {
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
block = Standalone<VectorRef<uint8_t>>();
block.reserve(block.arena(), std::max<int64_t>(SERVER_KNOBS->TLOG_MESSAGE_BLOCK_BYTES, msgSize));
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
} else {
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
block = logData->messageBlocks.back().second;
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
}
block.pop_front(block.size());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& msg : taggedMessages) {
if (msg.message.size() > block.capacity() - block.size()) {
Use emplace_back instead of push_back for TLogServer
2019-04-25 01:59:19 +08:00
logData->messageBlocks.emplace_back(version, block);
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
addedBytes += int64_t(block.size()) * SERVER_KNOBS->TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR;
block = Standalone<VectorRef<uint8_t>>();
block.reserve(block.arena(), std::max<int64_t>(SERVER_KNOBS->TLOG_MESSAGE_BLOCK_BYTES, msgSize));
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
DEBUG_TAGS_AND_MESSAGE("TLogCommitMessages", version, msg.getRawMessage())
.detail("UID", self->dbgid)
.detail("LogId", logData->logId);
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
block.append(block.arena(), msg.message.begin(), msg.message.size());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto tag : msg.tags) {
if (logData->locality == tagLocalitySatellite) {
if (!(tag.locality == tagLocalityTxs || tag.locality == tagLocalityLogRouter || tag == txsTag)) {
fix: populateSatelliteTagLocations was broken fix: satellites do not index the upgraded locality
2018-06-18 04:29:17 +08:00
continue;
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
} else if (!(logData->locality == tagLocalitySpecial || logData->locality == tag.locality ||
tag.locality < 0)) {
tlogs do not index tags for other localities
2018-06-02 13:51:08 +08:00
continue;
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (tag.locality == tagLocalityLogRouter) {
if (!logData->logRouterTags) {
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
continue;
}
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
tag.id = tag.id % logData->logRouterTags;
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (tag.locality == tagLocalityTxs) {
Fix issues configuring from sharded txs tag to not Which is an intermingling of what should be two commits: 1. Rely on TLogVersion instead of txsTags==0 2. Copy and index sharded txsTags between KCV and RV as txsTag when configuring log_version 4->3.
2019-07-17 16:25:09 +08:00
if (logData->txsTags > 0) {
tag.id = tag.id % logData->txsTags;
} else {
tag = txsTag;
}
shard the txs tag so that more transaction logs are involved in its recovery
2019-06-20 09:15:09 +08:00
}
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
Reference<LogData::TagData> tagData = logData->getTagData(tag);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!tagData) {
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
tagData = logData->createTagData(tag, 0, true, true, false);
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
}
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
if (version >= tagData->popped) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
tagData->versionMessages.emplace_back(
version, LengthPrefixedStringRef((uint32_t*)(block.end() - msg.message.size())));
if (tagData->versionMessages.back().second.expectedSize() > SERVER_KNOBS->MAX_MESSAGE_SIZE) {
TraceEvent(SevWarnAlways, "LargeMessage")
.detail("Size", tagData->versionMessages.back().second.expectedSize());
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
}
shard the txs tag so that more transaction logs are involved in its recovery
2019-06-20 09:15:09 +08:00
if (tag.locality != tagLocalityTxs && tag != txsTag) {
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
expectedBytes += tagData->versionMessages.back().second.expectedSize();
significant cpu optimization in update storage
2018-08-05 03:36:48 +08:00
} else {
txsBytes += tagData->versionMessages.back().second.expectedSize();
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// The factor of VERSION_MESSAGES_OVERHEAD is intended to be an overestimate of the actual memory used
// to store this data in a std::deque. In practice, this number is probably something like 528/512
// ~= 1.03, but this could vary based on the implementation. There will also be a fixed overhead per
// std::deque, but its size should be trivial relative to the size of the TLog queue and can be thought
// of as increasing the capacity of the queue slightly.
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
overheadBytes += SERVER_KNOBS->VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD;
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
}
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
}
removed unnecessary suppressFor
2018-06-19 08:59:29 +08:00
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
msgSize -= msg.message.size();
}
Use emplace_back instead of push_back for TLogServer
2019-04-25 01:59:19 +08:00
logData->messageBlocks.emplace_back(version, block);
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
addedBytes += int64_t(block.size()) * SERVER_KNOBS->TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR;
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
addedBytes += overheadBytes;
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
logData->version_sizes[version] = std::make_pair(expectedBytes, txsBytes);
logData->bytesInput += addedBytes;
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
self->bytesInput += addedBytes;
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
self->overheadBytesInput += overheadBytes;
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
Attempt to normalize trace events: * Detail names now all start with an uppercase character and contain no underscores. Ideally these should be head-first camel case, though that was harder to check. * Type names have the same rules, except they allow one underscore (to support a usage pattern Context_Type). The first character after the underscore is also uppercase. * Use seconds instead of milliseconds in details. Added a check when events are logged in simulation that logs a message to stderr if the first two rules above aren't followed. This probably doesn't address every instance of the above problems, but all of the events I was able to hit in simulation pass the check.
2018-06-09 02:11:08 +08:00
//TraceEvent("TLogPushed", self->dbgid).detail("Bytes", addedBytes).detail("MessageBytes", messages.size()).detail("Tags", tags.size()).detail("ExpectedBytes", expectedBytes).detail("MCount", mCount).detail("TCount", tCount);
optimized the log router commit path to avoid re-serializing the data
2018-03-17 02:40:21 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void commitMessages(TLogData* self, Reference<LogData> logData, Version version, Arena arena, StringRef messages) {
ArenaReader rd(arena, messages, Unversioned());
optimized memory allocations by using VectorRef<Tag> instead of std::vector<Tag>
2019-11-06 10:07:30 +08:00
self->tempTagMessages.clear();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
while (!rd.empty()) {
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
TagsAndMessage tagsAndMsg;
Refactor deserialization of TagsAndMessages Consolidate deserialization of TagsAndMessages in the structure itself and change both TLog and ServerPeekCursor to use it.
2019-09-05 05:52:09 +08:00
tagsAndMsg.loadFromArena(&rd, nullptr);
optimized memory allocations by using VectorRef<Tag> instead of std::vector<Tag>
2019-11-06 10:07:30 +08:00
self->tempTagMessages.push_back(std::move(tagsAndMsg));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
optimized memory allocations by using VectorRef<Tag> instead of std::vector<Tag>
2019-11-06 10:07:30 +08:00
commitMessages(self, logData, version, self->tempTagMessages);
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Version poppedVersion(Reference<LogData> self, Tag tag) {
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
auto tagData = self->getTagData(tag);
fix: if we cannot find a tag, it must have been popped at the recovery version.
2018-04-23 06:08:38 +08:00
if (!tagData) {
There were still use cases where this checks are necessary
2019-08-01 08:45:21 +08:00
if (tag == txsTag || tag.locality == tagLocalityTxs) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
return 0;
}
fix: if we cannot find a tag, it must have been popped at the recovery version.
2018-04-23 06:08:38 +08:00
return self->recoveredAt;
}
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
return tagData->popped;
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
std::deque<std::pair<Version, LengthPrefixedStringRef>>& getVersionMessages(Reference<LogData> self, Tag tag) {
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
auto tagData = self->getTagData(tag);
if (!tagData) {
Initial repository commit
2017-05-26 04:48:44 +08:00
static std::deque<std::pair<Version, LengthPrefixedStringRef>> empty;
return empty;
}
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
return tagData->versionMessages;
Initial repository commit
2017-05-26 04:48:44 +08:00
};
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void peekMessagesFromMemory(Reference<LogData> self,
TLogPeekRequest const& req,
BinaryWriter& messages,
Version& endVersion) {
ASSERT(!messages.getLength());
Initial repository commit
2017-05-26 04:48:44 +08:00
Rename several variables in TLogServer.actor.cpp to follow our normal camel case conventions. I didn't rename every variable here, because some appear to be data structures (like a map) following the pattern keydesc_valuedesc, and I wasn't sure that the straightforward keydescValuedesc rename made sense. I did rename a couple of instances of these where it seemed reasonable, though.
2018-06-02 01:18:07 +08:00
auto& deque = getVersionMessages(self, req.tag);
Remove trace-calls to printable (in non-workloads)
2019-03-19 06:03:43 +08:00
//TraceEvent("TLogPeekMem", self->dbgid).detail("Tag", req.tag1).detail("PDS", self->persistentDataSequence).detail("PDDS", self->persistentDataDurableSequence).detail("Oldest", map1.empty() ? 0 : map1.begin()->key ).detail("OldestMsgCount", map1.empty() ? 0 : map1.begin()->value.size());
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Version begin = std::max(req.begin, self->persistentDataDurableVersion + 1);
auto it = std::lower_bound(deque.begin(),
deque.end(),
std::make_pair(begin, LengthPrefixedStringRef()),
CompareFirst<std::pair<Version, LengthPrefixedStringRef>>());
Initial repository commit
2017-05-26 04:48:44 +08:00
Version currentVersion = -1;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (; it != deque.end(); ++it) {
if (it->first != currentVersion) {
Initial repository commit
2017-05-26 04:48:44 +08:00
if (messages.getLength() >= SERVER_KNOBS->DESIRED_TOTAL_BYTES) {
fix: do not tell storage servers about large sections of empty versions, because it can lead them to make mutations durable which have not been committed
2018-04-19 07:06:44 +08:00
endVersion = currentVersion + 1;
Attempt to normalize trace events: * Detail names now all start with an uppercase character and contain no underscores. Ideally these should be head-first camel case, though that was harder to check. * Type names have the same rules, except they allow one underscore (to support a usage pattern Context_Type). The first character after the underscore is also uppercase. * Use seconds instead of milliseconds in details. Added a check when events are logged in simulation that logs a message to stderr if the first two rules above aren't followed. This probably doesn't address every instance of the above problems, but all of the events I was able to hit in simulation pass the check.
2018-06-09 02:11:08 +08:00
//TraceEvent("TLogPeekMessagesReached2", self->dbgid);
Initial repository commit
2017-05-26 04:48:44 +08:00
break;
}
currentVersion = it->first;
Replace -1 as VERSION_HEADER constant for serialization
2019-09-06 02:30:02 +08:00
messages << VERSION_HEADER << currentVersion;
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// We need the 4 byte length prefix to be a TagsAndMessage format, but that prefix is added as part of StringRef
// serialization.
Changing the last suggestions from review.
2020-05-14 09:48:43 +08:00
int offset = messages.getLength();
Initial repository commit
2017-05-26 04:48:44 +08:00
messages << it->second.toStringRef();
Add debugMessagesAndTags, and track mutations in more places. Like: * Leaving the proxy * Entering the TLog * Leaving the TLog * Being read on a cursor All of this brought to you by TagsAndMessage! This also slides in a minor optimization as to how mutations are serialized per target log.
2020-03-27 18:31:04 +08:00
void* data = messages.getData();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
DEBUG_TAGS_AND_MESSAGE(
"TLogPeek", currentVersion, StringRef((uint8_t*)data + offset, messages.getLength() - offset))
.detail("LogId", self->logId)
.detail("PeekTag", req.tag);
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<std::vector<StringRef>> parseMessagesForTag(StringRef commitBlob, Tag tag, int logRouters) {
Peek from new spilled data and not old spilled data.
2019-02-08 09:02:38 +08:00
// See the comment in LogSystem.cpp for the binary format of commitBlob.
Add a yield to parseMessagesForTag
2019-03-04 04:57:37 +08:00
state std::vector<StringRef> relevantMessages;
add simulator support for different process versions and ProtocolVersion test
2020-09-29 01:58:49 +08:00
state BinaryReader rd(commitBlob, AssumeVersion(g_network->protocolVersion()));
Peek from new spilled data and not old spilled data.
2019-02-08 09:02:38 +08:00
while (!rd.empty()) {
Use TagsAndMessage for deserialization in TLogServer
2019-09-06 04:25:02 +08:00
TagsAndMessage tagsAndMessage;
tagsAndMessage.loadFromArena(&rd, nullptr);
for (Tag t : tagsAndMessage.tags) {
if (t == tag || (tag.locality == tagLocalityLogRouter && t.locality == tagLocalityLogRouter &&
t.id % logRouters == tag.id)) {
Peek from new spilled data and not old spilled data.
2019-02-08 09:02:38 +08:00
// Mutations that are in the partially durable span between known comitted version and
// recovery version get copied to the new log generation. These commits might have had more
// log router tags than what now exist, so we mod them down to what we have.
Use TagsAndMessage for deserialization in TLogServer
2019-09-06 04:25:02 +08:00
relevantMessages.push_back(tagsAndMessage.getRawMessage());
break;
Peek from new spilled data and not old spilled data.
2019-02-08 09:02:38 +08:00
}
}
Add a yield to parseMessagesForTag
2019-03-04 04:57:37 +08:00
wait(yield());
Peek from new spilled data and not old spilled data.
2019-02-08 09:02:38 +08:00
}
return relevantMessages;
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> tLogPeekMessages(TLogData* self, TLogPeekRequest req, Reference<LogData> logData) {
Initial repository commit
2017-05-26 04:48:44 +08:00
state BinaryWriter messages(Unversioned());
state BinaryWriter messages2(Unversioned());
state int sequence = -1;
state UID peekId;
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
state double queueStart = now();
Fixed a bug in cache server while peeking at a version lower than popped version and added some logging.
2020-03-27 03:39:07 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.tag.locality == tagLocalityTxs && req.tag.id >= logData->txsTags && logData->txsTags > 0) {
A number of bug fixes of rare correctness errors
2020-04-30 04:50:13 +08:00
req.tag.id = req.tag.id % logData->txsTags;
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.sequence.present()) {
Initial repository commit
2017-05-26 04:48:44 +08:00
try {
peekId = req.sequence.get().first;
sequence = req.sequence.get().second;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (sequence >= SERVER_KNOBS->PARALLEL_GET_MORE_REQUESTS &&
logData->peekTracker.find(peekId) == logData->peekTracker.end()) {
Continuing a parallel peek after a timeout would hang.
2019-10-23 08:04:57 +08:00
throw operation_obsolete();
Continuing a parallel peek after a timeout would hang. This is to guard against the case where 1. Peeks with sequence numbers 0-39 are submitted 2. A 15min pause happens, in which timeout removes the peek tracker data 3. Peeks with sequence numbers 40-59 are submitted, with the same peekId The second round of peeks wouldn't have the data left that it's allowed to start running peek 40 immediately, and thus would hang for 10min until it gets cleaned up. Also, guard against overflowing the sequence number.
2019-10-23 08:04:57 +08:00
}
Address more issues that cause peek cursors to time out. There were error cases that would cause a peek to terminate early or be cancelled without sending anything to the next peek in line. We would thus end up with the first peek in a sequence waiting on its future, and nothing that exists that would send to that future.
2019-07-16 05:49:24 +08:00
auto& trackerData = logData->peekTracker[peekId];
Remove buggify and unneeded safeguards. The buggify was actually incorrect and broke an invariant, which I then fixed on the other side, but this work was actually unneeded in total. The real issue being fixed was returnIfBlock not sending an error, as well as the other error cases.
2019-07-16 09:51:24 +08:00
if (sequence == 0 && trackerData.sequence_version.find(0) == trackerData.sequence_version.end()) {
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
trackerData.tag = req.tag;
Remove buggify and unneeded safeguards. The buggify was actually incorrect and broke an invariant, which I then fixed on the other side, but this work was actually unneeded in total. The real issue being fixed was returnIfBlock not sending an error, as well as the other error cases.
2019-07-16 09:51:24 +08:00
trackerData.sequence_version[0].send(std::make_pair(req.begin, req.onlySpilled));
Address more issues that cause peek cursors to time out. There were error cases that would cause a peek to terminate early or be cancelled without sending anything to the next peek in line. We would thus end up with the first peek in a sequence waiting on its future, and nothing that exists that would send to that future.
2019-07-16 05:49:24 +08:00
}
auto seqBegin = trackerData.sequence_version.begin();
Remove buggify and unneeded safeguards. The buggify was actually incorrect and broke an invariant, which I then fixed on the other side, but this work was actually unneeded in total. The real issue being fixed was returnIfBlock not sending an error, as well as the other error cases.
2019-07-16 09:51:24 +08:00
// The peek cursor and this comparison need to agree about the maximum number of in-flight requests.
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
while (trackerData.sequence_version.size() &&
seqBegin->first <= sequence - SERVER_KNOBS->PARALLEL_GET_MORE_REQUESTS) {
if (seqBegin->second.canBeSet()) {
Add an ASSERT_WE_THINK that peek cursors don't get timed_out() This should prevent us from regressing and having multi-region recoveries hang for 10min again.
2019-10-15 09:05:06 +08:00
seqBegin->second.sendError(operation_obsolete());
fix: parallel peek requests leaked memory
2018-10-03 08:28:39 +08:00
}
Address more issues that cause peek cursors to time out. There were error cases that would cause a peek to terminate early or be cancelled without sending anything to the next peek in line. We would thus end up with the first peek in a sequence waiting on its future, and nothing that exists that would send to that future.
2019-07-16 05:49:24 +08:00
trackerData.sequence_version.erase(seqBegin);
seqBegin = trackerData.sequence_version.begin();
}
fix: parallel peek requests leaked memory
2018-10-03 08:28:39 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (trackerData.sequence_version.size() && sequence < seqBegin->first) {
Add an ASSERT_WE_THINK that peek cursors don't get timed_out() This should prevent us from regressing and having multi-region recoveries hang for 10min again.
2019-10-15 09:05:06 +08:00
throw operation_obsolete();
Initial repository commit
2017-05-26 04:48:44 +08:00
}
Address more issues that cause peek cursors to time out. There were error cases that would cause a peek to terminate early or be cancelled without sending anything to the next peek in line. We would thus end up with the first peek in a sequence waiting on its future, and nothing that exists that would send to that future.
2019-07-16 05:49:24 +08:00
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
Future<std::pair<Version, bool>> fPrevPeekData = trackerData.sequence_version[sequence].getFuture();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (fPrevPeekData.isReady()) {
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
trackerData.unblockedPeeks++;
double t = now() - trackerData.lastUpdate;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (t > trackerData.idleMax)
trackerData.idleMax = t;
fix compile errors
2020-04-23 05:36:41 +08:00
trackerData.idleTime += t;
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
}
Address more issues that cause peek cursors to time out. There were error cases that would cause a peek to terminate early or be cancelled without sending anything to the next peek in line. We would thus end up with the first peek in a sequence waiting on its future, and nothing that exists that would send to that future.
2019-07-16 05:49:24 +08:00
trackerData.lastUpdate = now();
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
std::pair<Version, bool> prevPeekData = wait(fPrevPeekData);
TLogs should not respond with data earlier than the begin version Parallel peek more code would prefer the begin version it was sent by the previous parallel peek over the request's begin version. This means that a merge cursor trying to advance past message versions would still get old data that it would have to filter out. A simple application of std::max fixes this.
2020-01-22 11:09:07 +08:00
req.begin = std::max(prevPeekData.first, req.begin);
Merge remote-tracking branch 'upstream/master' into peek-cursor-timeout-bug
2019-07-16 08:05:39 +08:00
req.onlySpilled = prevPeekData.second;
Address more issues that cause peek cursors to time out. There were error cases that would cause a peek to terminate early or be cancelled without sending anything to the next peek in line. We would thus end up with the first peek in a sequence waiting on its future, and nothing that exists that would send to that future.
2019-07-16 05:49:24 +08:00
wait(yield());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
} catch (Error& e) {
if (e.code() == error_code_timed_out || e.code() == error_code_operation_obsolete) {
Add an ASSERT_WE_THINK that peek cursors don't get timed_out() This should prevent us from regressing and having multi-region recoveries hang for 10min again.
2019-10-15 09:05:06 +08:00
req.reply.sendError(e);
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
} else {
throw;
}
}
}
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
state double blockStart = now();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.returnIfBlocked && logData->version.get() < req.begin) {
Initial repository commit
2017-05-26 04:48:44 +08:00
req.reply.sendError(end_of_stream());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.sequence.present()) {
Address more issues that cause peek cursors to time out. There were error cases that would cause a peek to terminate early or be cancelled without sending anything to the next peek in line. We would thus end up with the first peek in a sequence waiting on its future, and nothing that exists that would send to that future.
2019-07-16 05:49:24 +08:00
auto& trackerData = logData->peekTracker[peekId];
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
auto& sequenceData = trackerData.sequence_version[sequence + 1];
Add an ASSERT_WE_THINK that peek cursors don't get timed_out() This should prevent us from regressing and having multi-region recoveries hang for 10min again.
2019-10-15 09:05:06 +08:00
trackerData.lastUpdate = now();
Address more issues that cause peek cursors to time out. There were error cases that would cause a peek to terminate early or be cancelled without sending anything to the next peek in line. We would thus end up with the first peek in a sequence waiting on its future, and nothing that exists that would send to that future.
2019-07-16 05:49:24 +08:00
if (!sequenceData.isSet()) {
Merge remote-tracking branch 'upstream/master' into peek-cursor-timeout-bug
2019-07-16 08:05:39 +08:00
sequenceData.send(std::make_pair(req.begin, req.onlySpilled));
Address more issues that cause peek cursors to time out. There were error cases that would cause a peek to terminate early or be cancelled without sending anything to the next peek in line. We would thus end up with the first peek in a sequence waiting on its future, and nothing that exists that would send to that future.
2019-07-16 05:49:24 +08:00
}
}
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
}
Remove trace-calls to printable (in non-workloads)
2019-03-19 06:03:43 +08:00
//TraceEvent("TLogPeekMessages0", self->dbgid).detail("ReqBeginEpoch", req.begin.epoch).detail("ReqBeginSeq", req.begin.sequence).detail("Epoch", self->epoch()).detail("PersistentDataSeq", self->persistentDataSequence).detail("Tag1", req.tag1).detail("Tag2", req.tag2);
Initial repository commit
2017-05-26 04:48:44 +08:00
// Wait until we have something to return that the caller doesn't already have
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->version.get() < req.begin) {
wait(logData->version.whenAtLeast(req.begin));
wait(delay(SERVER_KNOBS->TLOG_PEEK_DELAY, g_network->getCurrentTask()));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->locality != tagLocalitySatellite && req.tag.locality == tagLocalityLogRouter) {
wait(self->concurrentLogRouterReads.take());
tlogs serve reads to log routers at a low priority, to prevent them from using all their resources catching up a remote dc that has been down for a long time increase the amount of memory ratekeeper budgets for tlogs so that there is a gap after the spill threshold to prevent temporarily overshooting the budget
2018-08-05 01:31:30 +08:00
state FlowLock::Releaser globalReleaser(self->concurrentLogRouterReads);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(delay(0.0, TaskPriority::Low));
tlogs serve reads to log routers at a low priority, to prevent them from using all their resources catching up a remote dc that has been down for a long time increase the amount of memory ratekeeper budgets for tlogs so that there is a gap after the spill threshold to prevent temporarily overshooting the budget
2018-08-05 01:31:30 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.begin <= logData->persistentDataDurableVersion && req.tag.locality != tagLocalityTxs && req.tag != txsTag) {
Deprioritize spilled peeks in spill-by-value, and improve its logic. This deprioritizes before calling peekMessagesFromMemory, which should improve the memory usage of the TLog, and makes sure to keep txsTag peeks at a high priority to help recoveries stay fast.
2019-05-04 03:55:41 +08:00
// Reading spilled data will almost always imply that the storage server is >5s behind the rest
// of the cluster. We shouldn't prioritize spending CPU on helping this server catch up
// slightly faster over keeping the rest of the cluster operating normally.
// txsTag is only ever peeked on recovery, and we would still wish to prioritize requests
// that impact recovery duration.
A giant translation of TaskFooPriority -> TaskPriority::Foo This is so that APIs that take priorities don't take ints, which are common and easy to accidentally pass the wrong thing.
2019-06-25 17:47:35 +08:00
wait(delay(0, TaskPriority::TLogSpilledPeekReply));
Deprioritize spilled peeks in spill-by-value, and improve its logic. This deprioritizes before calling peekMessagesFromMemory, which should improve the memory usage of the TLog, and makes sure to keep txsTag peeks at a high priority to help recoveries stay fast.
2019-05-04 03:55:41 +08:00
}
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
state double workStart = now();
merged everything from feature-remote-logs besides the tlog and tagpartitionedlogsystem re-included tags in messages to the tlog previously never committed the LogRouter
2017-06-30 06:50:19 +08:00
Version poppedVer = poppedVersion(logData, req.tag);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (poppedVer > req.begin) {
merged everything from feature-remote-logs besides the tlog and tagpartitionedlogsystem re-included tags in messages to the tlog previously never committed the LogRouter
2017-06-30 06:50:19 +08:00
TLogPeekReply rep;
rep.maxKnownVersion = logData->version.get();
log routers now calculate a precise version to pop for their log router tag
2018-06-22 06:29:46 +08:00
rep.minKnownCommittedVersion = logData->minKnownCommittedVersion;
merged everything from feature-remote-logs besides the tlog and tagpartitionedlogsystem re-included tags in messages to the tlog previously never committed the LogRouter
2017-06-30 06:50:19 +08:00
rep.popped = poppedVer;
rep.end = poppedVer;
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
rep.onlySpilled = false;
fixed a large number of bugs
2017-07-14 03:29:21 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.sequence.present()) {
Fix parallel peek stalling for 10min when a TLog generation is destroyed. `peekTracker` was held on the Shared TLog (TLogData), whereas peeks are received and replied to as part of a TLog instance (LogData). When a peek was received on a TLog, it was registered into peekTracker along with the ReplyPromise. If the TLog was then removed as part of a no-longer-needed generation of TLogs, there is nothing left to reply to the request, but by holding onto the ReplyPromise in peekTracker, we leave the remote end with an expectation that we will reply. Then, 10min later, peekTrackerCleanup runs and finally times out the peek cursor, thus preventing FDB from being completely stuck. Now, each TLog generation has its own `peekTracker`, and when a TLog is destroyed, it times out all of the pending peek curors that are still expecting a response. This will then trigger the client to re-issue them to the next generation of TLogs, thus removing the 10min gap to do so.
2019-07-10 08:27:36 +08:00
auto& trackerData = logData->peekTracker[peekId];
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
auto& sequenceData = trackerData.sequence_version[sequence + 1];
fixed a large number of bugs
2017-07-14 03:29:21 +08:00
trackerData.lastUpdate = now();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (trackerData.sequence_version.size() && sequence + 1 < trackerData.sequence_version.begin()->first) {
Add an ASSERT_WE_THINK that peek cursors don't get timed_out() This should prevent us from regressing and having multi-region recoveries hang for 10min again.
2019-10-15 09:05:06 +08:00
req.reply.sendError(operation_obsolete());
Address more issues that cause peek cursors to time out. There were error cases that would cause a peek to terminate early or be cancelled without sending anything to the next peek in line. We would thus end up with the first peek in a sequence waiting on its future, and nothing that exists that would send to that future.
2019-07-16 05:49:24 +08:00
if (!sequenceData.isSet())
Make TLogs have consistent parallel peek behavior. TLogServer and LogRouter had some leftover code from me trying to be more "correct" about parallel peek semantics, but those changes weren't reflected in the OldTLog* files. I've reverted the changes, as realistically, they are more likely to waste CPU than improve TLog behavior.
2020-01-22 10:23:16 +08:00
sequenceData.sendError(operation_obsolete());
fix: parallel peek requests leaked memory
2018-10-03 08:28:39 +08:00
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (sequenceData.isSet()) {
if (sequenceData.getFuture().get().first != rep.end) {
TEST(true); // tlog peek second attempt ended at a different version
Add an ASSERT_WE_THINK that peek cursors don't get timed_out() This should prevent us from regressing and having multi-region recoveries hang for 10min again.
2019-10-15 09:05:06 +08:00
req.reply.sendError(operation_obsolete());
fixed a large number of bugs
2017-07-14 03:29:21 +08:00
return Void();
}
} else {
Improve the behavior of parallelPeekMore+onlySpilled. When onlySpilled transitions from true (don't peek memory) to false (do peek memory) as part of a parallel peek, we'll end up wasting the rest of the replies because we'll honor their onlySpilled=true setting and thus not have any additional data to return. Instead, we thread the onlySpilled back through in the same way that the ending version of the last peek is used overrides the requested starting version of the next peek. This simulated the same behavior that the client has, where the value of onlySpilled that we reply with comes back in the next request. I haven't actually seen it be a problem, but this should help make sure the onlySpilled transition when catching up doesn't ever cause any ill effects if a process starts riding the line between onlySpilled settings.
2019-07-09 13:13:09 +08:00
sequenceData.send(std::make_pair(rep.end, rep.onlySpilled));
fixed a large number of bugs
2017-07-14 03:29:21 +08:00
}
fix: parallel get more needs to verify the begin version matches the end of the previous request, because when a peek cursor expires we lose all history, so the same sequence number could start at different versions
2018-06-26 02:15:49 +08:00
rep.begin = req.begin;
fixed a large number of bugs
2017-07-14 03:29:21 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
req.reply.send(rep);
merged everything from feature-remote-logs besides the tlog and tagpartitionedlogsystem re-included tags in messages to the tlog previously never committed the LogRouter
2017-06-30 06:50:19 +08:00
return Void();
}
Initial repository commit
2017-05-26 04:48:44 +08:00
state Version endVersion = logData->version.get() + 1;
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
state bool onlySpilled = false;
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// grab messages from disk
Remove trace-calls to printable (in non-workloads)
2019-03-19 06:03:43 +08:00
//TraceEvent("TLogPeekMessages", self->dbgid).detail("ReqBeginEpoch", req.begin.epoch).detail("ReqBeginSeq", req.begin.sequence).detail("Epoch", self->epoch()).detail("PersistentDataSeq", self->persistentDataSequence).detail("Tag1", req.tag1).detail("Tag2", req.tag2);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.begin <= logData->persistentDataDurableVersion) {
// Just in case the durable version changes while we are waiting for the read, we grab this data from memory. We
// may or may not actually send it depending on whether we get enough data from disk. SOMEDAY: Only do this if
// an initial attempt to read from disk results in insufficient data and the required data is no longer in
// memory SOMEDAY: Should we only send part of the messages we collected, to actually limit the size of the
// result?
Initial repository commit
2017-05-26 04:48:44 +08:00
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
if (req.onlySpilled) {
endVersion = logData->persistentDataDurableVersion + 1;
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
peekMessagesFromMemory(logData, req, messages2, endVersion);
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->shouldSpillByValue(req.tag)) {
Create RangeResult type alias
2021-05-04 04:14:16 +08:00
RangeResult kvs = wait(self->persistentData->readRange(
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
KeyRangeRef(persistTagMessagesKey(logData->logId, req.tag, req.begin),
persistTagMessagesKey(logData->logId, req.tag, logData->persistentDataDurableVersion + 1)),
SERVER_KNOBS->DESIRED_TOTAL_BYTES,
SERVER_KNOBS->DESIRED_TOTAL_BYTES));
Peek from new spilled data and not old spilled data.
2019-02-08 09:02:38 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& kv : kvs) {
Spill txsTag by value under TagMsg/ and not TagMsgRef/ There's not a tremendous reason as to why this matters now, but I feel like I might regret sometime later not keeping the same schema under the same key.
2019-03-04 09:01:52 +08:00
auto ver = decodeTagMessagesKey(kv.key);
Replace -1 as VERSION_HEADER constant for serialization
2019-09-06 02:30:02 +08:00
messages << VERSION_HEADER << ver;
Spill txsTag by value
2019-03-04 04:57:33 +08:00
messages.serializeBytes(kv.value);
Peek from new spilled data and not old spilled data.
2019-02-08 09:02:38 +08:00
}
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
if (kvs.expectedSize() >= SERVER_KNOBS->DESIRED_TOTAL_BYTES) {
Spill txsTag by value under TagMsg/ and not TagMsgRef/ There's not a tremendous reason as to why this matters now, but I feel like I might regret sometime later not keeping the same schema under the same key.
2019-03-04 09:01:52 +08:00
endVersion = decodeTagMessagesKey(kvs.end()[-1].key) + 1;
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
onlySpilled = true;
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
messages.serializeBytes(messages2.toValue());
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
}
Spill txsTag by value
2019-03-04 04:57:33 +08:00
} else {
// FIXME: Limit to approximately DESIRED_TOTATL_BYTES somehow.
Create RangeResult type alias
2021-05-04 04:14:16 +08:00
RangeResult kvrefs = wait(self->persistentData->readRange(
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
KeyRangeRef(
persistTagMessageRefsKey(logData->logId, req.tag, req.begin),
persistTagMessageRefsKey(logData->logId, req.tag, logData->persistentDataDurableVersion + 1)),
SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK + 1));
Spill txsTag by value
2019-03-04 04:57:33 +08:00
Remove trace-calls to printable (in non-workloads)
2019-03-19 06:03:43 +08:00
//TraceEvent("TLogPeekResults", self->dbgid).detail("ForAddress", req.reply.getEndpoint().getPrimaryAddress()).detail("Tag1Results", s1).detail("Tag2Results", s2).detail("Tag1ResultsLim", kv1.size()).detail("Tag2ResultsLim", kv2.size()).detail("Tag1ResultsLast", kv1.size() ? kv1[0].key : "").detail("Tag2ResultsLast", kv2.size() ? kv2[0].key : "").detail("Limited", limited).detail("NextEpoch", next_pos.epoch).detail("NextSeq", next_pos.sequence).detail("NowEpoch", self->epoch()).detail("NowSeq", self->sequence.getNextSequence());
Spill txsTag by value
2019-03-04 04:57:33 +08:00
Implement hard limiting of memory used to serve peek requests.
2019-03-04 04:57:38 +08:00
state std::vector<std::pair<IDiskQueue::location, IDiskQueue::location>> commitLocations;
Solve overreading via pre-calculating tag bytes per commit
2019-03-04 04:57:36 +08:00
state bool earlyEnd = false;
uint32_t mutationBytes = 0;
Replace ResourceLimiter with FlowLock.
2019-03-04 08:49:42 +08:00
state uint64_t commitBytes = 0;
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
state Version firstVersion = std::numeric_limits<Version>::max();
Add a knob to limit amount of data read from sqlite for one PeekRequest. This prevents peeking from degrading over time if there are a very large number of SpilledData entries for one particular tag.
2019-05-02 08:40:31 +08:00
for (int i = 0; i < kvrefs.size() && i < SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK; i++) {
auto& kv = kvrefs[i];
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
VectorRef<SpilledData> spilledData;
Persist the protocol version of a TLog instance when it is created. This allows us to do easy upgrades of SpilledData in the future, if the need arises, because we then have a protocol version to compare against.
2019-03-16 12:01:17 +08:00
BinaryReader r(kv.value, AssumeVersion(logData->protocolVersion));
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
r >> spilledData;
for (const SpilledData& sd : spilledData) {
Solve overreading via pre-calculating tag bytes per commit
2019-03-04 04:57:36 +08:00
if (mutationBytes >= SERVER_KNOBS->DESIRED_TOTAL_BYTES) {
earlyEnd = true;
break;
}
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
if (sd.version >= req.begin) {
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
firstVersion = std::min(firstVersion, sd.version);
Spill (start,length) instead of (begin,end) to save a few bytes.
2019-03-04 04:57:41 +08:00
const IDiskQueue::location end = sd.start.lo + sd.length;
Use emplace_back instead of push_back for TLogServer
2019-04-25 01:59:19 +08:00
commitLocations.emplace_back(sd.start, end);
Implement hard limiting of memory used to serve peek requests.
2019-03-04 04:57:38 +08:00
// This isn't perfect, because we aren't accounting for page boundaries, but should be
// close enough.
Spill (start,length) instead of (begin,end) to save a few bytes.
2019-03-04 04:57:41 +08:00
commitBytes += sd.length;
Solve overreading via pre-calculating tag bytes per commit
2019-03-04 04:57:36 +08:00
mutationBytes += sd.mutationBytes;
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (earlyEnd)
break;
Spill txsTag by value
2019-03-04 04:57:33 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
earlyEnd = earlyEnd || (kvrefs.size() >= SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK + 1);
wait(self->peekMemoryLimiter.take(TaskPriority::TLogSpilledPeekReply, commitBytes));
Replace ResourceLimiter with FlowLock.
2019-03-04 08:49:42 +08:00
state FlowLock::Releaser memoryReservation(self->peekMemoryLimiter, commitBytes);
Implement hard limiting of memory used to serve peek requests.
2019-03-04 04:57:38 +08:00
state std::vector<Future<Standalone<StringRef>>> messageReads;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
messageReads.reserve(commitLocations.size());
Implement hard limiting of memory used to serve peek requests.
2019-03-04 04:57:38 +08:00
for (const auto& pair : commitLocations) {
Rename ::TRUE and ::FALSE in BooleanParams to ::True and ::False so as to not conflict with the TRUE and FALSE macros provided by the Windows and MacOS SDKs.
2021-07-17 15:11:40 +08:00
messageReads.push_back(self->rawPersistentQueue->read(pair.first, pair.second, CheckHashes::True));
Implement hard limiting of memory used to serve peek requests.
2019-03-04 04:57:38 +08:00
}
commitLocations.clear();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(waitForAll(messageReads));
Spill txsTag by value
2019-03-04 04:57:33 +08:00
Add a yield to parseMessagesForTag
2019-03-04 04:57:37 +08:00
state Version lastRefMessageVersion = 0;
state int index = 0;
loop {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (index >= messageReads.size())
break;
Add a yield to parseMessagesForTag
2019-03-04 04:57:37 +08:00
Standalone<StringRef> queueEntryData = messageReads[index].get();
Spill txsTag by value
2019-03-04 04:57:33 +08:00
uint8_t valid;
const uint32_t length = *(uint32_t*)queueEntryData.begin();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
queueEntryData = queueEntryData.substr(4, queueEntryData.size() - 4);
BinaryReader rd(queueEntryData, IncludeVersion());
Add a yield to parseMessagesForTag
2019-03-04 04:57:37 +08:00
state TLogQueueEntry entry;
Spill txsTag by value
2019-03-04 04:57:33 +08:00
rd >> entry >> valid;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ASSERT(valid == 0x01);
ASSERT(length + sizeof(valid) == queueEntryData.size());
Spill txsTag by value
2019-03-04 04:57:33 +08:00
Replace -1 as VERSION_HEADER constant for serialization
2019-09-06 02:30:02 +08:00
messages << VERSION_HEADER << entry.version;
Spill txsTag by value
2019-03-04 04:57:33 +08:00
Use TagsAndMessage for deserialization in TLogServer
2019-09-06 04:25:02 +08:00
std::vector<StringRef> rawMessages =
wait(parseMessagesForTag(entry.messages, req.tag, logData->logRouterTags));
for (const StringRef& msg : rawMessages) {
messages.serializeBytes(msg);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
DEBUG_TAGS_AND_MESSAGE("TLogPeekFromDisk", entry.version, msg)
.detail("UID", self->dbgid)
.detail("LogId", logData->logId)
.detail("PeekTag", req.tag);
Spill txsTag by value
2019-03-04 04:57:33 +08:00
}
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
lastRefMessageVersion = entry.version;
Add a yield to parseMessagesForTag
2019-03-04 04:57:37 +08:00
index++;
Peek from new spilled data and not old spilled data.
2019-02-08 09:02:38 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
Spill txsTag by value
2019-03-04 04:57:33 +08:00
messageReads.clear();
Replace ResourceLimiter with FlowLock.
2019-03-04 08:49:42 +08:00
memoryReservation.release();
Peek from new spilled data and not old spilled data.
2019-02-08 09:02:38 +08:00
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
if (earlyEnd) {
Spill txsTag by value
2019-03-04 04:57:33 +08:00
endVersion = lastRefMessageVersion + 1;
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
onlySpilled = true;
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
messages.serializeBytes(messages2.toValue());
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
}
Spill txsTag by value
2019-03-04 04:57:33 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
} else {
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
if (req.onlySpilled) {
endVersion = logData->persistentDataDurableVersion + 1;
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
peekMessagesFromMemory(logData, req, messages, endVersion);
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
}
Listen to multiple addresses and start using vector<NetworkAdddress> in Endpoint - This patch will make FDB listen to multiple addresses given via command line. Although, we'll still use first address in most places, this patch starts using vector<NetworkAddress> in Endpoint at some basic places. - When sending packets to an endpoint, pick a random network address in endpoints - Renames Endpoint::address to Endpoint::addresses since it now holds a vector of addresses.
2018-10-31 04:44:37 +08:00
//TraceEvent("TLogPeekResults", self->dbgid).detail("ForAddress", req.reply.getEndpoint().getPrimaryAddress()).detail("MessageBytes", messages.getLength()).detail("NextEpoch", next_pos.epoch).detail("NextSeq", next_pos.sequence).detail("NowSeq", self->sequence.getNextSequence());
Initial repository commit
2017-05-26 04:48:44 +08:00
}
TLogPeekReply reply;
reply.maxKnownVersion = logData->version.get();
log routers now calculate a precise version to pop for their log router tag
2018-06-22 06:29:46 +08:00
reply.minKnownCommittedVersion = logData->minKnownCommittedVersion;
renamed BinaryWriter.toStringRef() to .toValue(), because the function now returns a Standalone<StringRef>() eliminated an unnecessary copy from the proxy commit path eliminated an unnecessary copy from buffered peek cursor
2019-03-29 02:52:50 +08:00
reply.messages = messages.toValue();
merged everything from feature-remote-logs besides the tlog and tagpartitionedlogsystem re-included tags in messages to the tlog previously never committed the LogRouter
2017-06-30 06:50:19 +08:00
reply.end = endVersion;
Save TLog resources by letting peek request only spilled data. If a peek is entirely fulfilled from spilled data, then it's likely that the next peek will be also. It is thus wasteful for each of these peeks to call peekMessagesFromMemory, which memcpy's excessively, and then throw all that data away without using it. Now, TLogs will give a hint back to peek cursors about if the provided reply was served entirely from the spilled data, which peek curors then feed back as the hint into their next request. At some point, a cursor will send a request for only spilled data, get an incomplete response, and then be told to send its next request as one that peeks from memory as well, and then it will fully catch up.
2019-05-15 08:07:49 +08:00
reply.onlySpilled = onlySpilled;
merged everything from feature-remote-logs besides the tlog and tagpartitionedlogsystem re-included tags in messages to the tlog previously never committed the LogRouter
2017-06-30 06:50:19 +08:00
Fixed a bug in cache server while peeking at a version lower than popped version and added some logging.
2020-03-27 03:39:07 +08:00
//TraceEvent("TlogPeek", self->dbgid).detail("LogId", logData->logId).detail("Tag", req.tag.toString()).
// detail("BeginVer", req.begin).detail("EndVer", reply.end).
// detail("MsgBytes", reply.messages.expectedSize()).
// detail("ForAddress", req.reply.getEndpoint().getPrimaryAddress());
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.sequence.present()) {
Fix parallel peek stalling for 10min when a TLog generation is destroyed. `peekTracker` was held on the Shared TLog (TLogData), whereas peeks are received and replied to as part of a TLog instance (LogData). When a peek was received on a TLog, it was registered into peekTracker along with the ReplyPromise. If the TLog was then removed as part of a no-longer-needed generation of TLogs, there is nothing left to reply to the request, but by holding onto the ReplyPromise in peekTracker, we leave the remote end with an expectation that we will reply. Then, 10min later, peekTrackerCleanup runs and finally times out the peek cursor, thus preventing FDB from being completely stuck. Now, each TLog generation has its own `peekTracker`, and when a TLog is destroyed, it times out all of the pending peek curors that are still expecting a response. This will then trigger the client to re-issue them to the next generation of TLogs, thus removing the 10min gap to do so.
2019-07-10 08:27:36 +08:00
auto& trackerData = logData->peekTracker[peekId];
Initial repository commit
2017-05-26 04:48:44 +08:00
trackerData.lastUpdate = now();
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
double queueT = blockStart - queueStart;
double blockT = workStart - blockStart;
double workT = now() - workStart;
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
trackerData.totalPeeks++;
trackerData.replyBytes += reply.messages.size();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (queueT > trackerData.queueMax)
trackerData.queueMax = queueT;
if (blockT > trackerData.blockMax)
trackerData.blockMax = blockT;
if (workT > trackerData.workMax)
trackerData.workMax = workT;
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
trackerData.queueTime += queueT;
trackerData.blockTime += blockT;
trackerData.workTime += workT;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
auto& sequenceData = trackerData.sequence_version[sequence + 1];
if (trackerData.sequence_version.size() && sequence + 1 < trackerData.sequence_version.begin()->first) {
Add an ASSERT_WE_THINK that peek cursors don't get timed_out() This should prevent us from regressing and having multi-region recoveries hang for 10min again.
2019-10-15 09:05:06 +08:00
req.reply.sendError(operation_obsolete());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!sequenceData.isSet()) {
Make TLogs have consistent parallel peek behavior. TLogServer and LogRouter had some leftover code from me trying to be more "correct" about parallel peek semantics, but those changes weren't reflected in the OldTLog* files. I've reverted the changes, as realistically, they are more likely to waste CPU than improve TLog behavior.
2020-01-22 10:23:16 +08:00
// It would technically be more correct to .send({req.begin, req.onlySpilled}), as the next
// request might still be in the window of active requests, but LogSystemPeekCursor will
// throw away all future responses upon getting an operation_obsolete(), so computing a
// response will probably be a waste of CPU.
sequenceData.sendError(operation_obsolete());
}
fix: parallel peek requests leaked memory
2018-10-03 08:28:39 +08:00
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (sequenceData.isSet()) {
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
trackerData.duplicatePeeks++;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (sequenceData.getFuture().get().first != reply.end) {
TEST(true); // tlog peek second attempt ended at a different version (2)
Add an ASSERT_WE_THINK that peek cursors don't get timed_out() This should prevent us from regressing and having multi-region recoveries hang for 10min again.
2019-10-15 09:05:06 +08:00
req.reply.sendError(operation_obsolete());
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
}
} else {
Improve the behavior of parallelPeekMore+onlySpilled. When onlySpilled transitions from true (don't peek memory) to false (do peek memory) as part of a parallel peek, we'll end up wasting the rest of the replies because we'll honor their onlySpilled=true setting and thus not have any additional data to return. Instead, we thread the onlySpilled back through in the same way that the ending version of the last peek is used overrides the requested starting version of the next peek. This simulated the same behavior that the client has, where the value of onlySpilled that we reply with comes back in the next request. I haven't actually seen it be a problem, but this should help make sure the onlySpilled transition when catching up doesn't ever cause any ill effects if a process starts riding the line between onlySpilled settings.
2019-07-09 13:13:09 +08:00
sequenceData.send(std::make_pair(reply.end, reply.onlySpilled));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
fix: parallel get more needs to verify the begin version matches the end of the previous request, because when a peek cursor expires we lose all history, so the same sequence number could start at different versions
2018-06-26 02:15:49 +08:00
reply.begin = req.begin;
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
req.reply.send(reply);
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> doQueueCommit(TLogData* self,
Reference<LogData> logData,
std::vector<Reference<LogData>> missingFinalCommit) {
Initial repository commit
2017-05-26 04:48:44 +08:00
state Version ver = logData->version.get();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
state Version commitNumber = self->queueCommitBegin + 1;
fix: log routers use durable known committed version at the time of the pop to determine what is safe to pop from their logs fix: storage server does not advance its version across large version increase until it has data associated with the version
2018-04-19 03:07:29 +08:00
state Version knownCommittedVersion = logData->knownCommittedVersion;
Initial repository commit
2017-05-26 04:48:44 +08:00
self->queueCommitBegin = commitNumber;
logData->queueCommittingVersion = ver;
optimized memory allocations by using VectorRef<Tag> instead of std::vector<Tag>
2019-11-06 10:07:30 +08:00
g_network->setCurrentTask(TaskPriority::TLogCommitReply);
Initial repository commit
2017-05-26 04:48:44 +08:00
Future<Void> c = self->persistentQueue->commit();
self->diskQueueCommitBytes = 0;
self->largeDiskQueueCommitBytes.set(false);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(ioDegradedOrTimeoutError(
c, SERVER_KNOBS->MAX_STORAGE_COMMIT_TIME, self->degraded, SERVER_KNOBS->TLOG_DEGRADED_DURATION));
if (g_network->isSimulated() && !g_simulator.speedUpSimulation && BUGGIFY_WITH_PROB(0.0001)) {
tlog commits very rarely take an additional 6 seconds
2019-03-12 03:11:17 +08:00
wait(delay(6.0));
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(self->queueCommitEnd.whenAtLeast(commitNumber - 1));
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// Calling check_yield instead of yield to avoid a destruction ordering problem in simulation
if (g_network->check_yield(g_network->getCurrentTask())) {
Rewrite all `Void _ = wait(...)` -> `wait(...)`. This takes advantage of the new actorcompiler functionality to avoid having duplicate definitions of `Void _` when trying to feed the un-actorompiled source through clang.
2018-08-11 04:57:10 +08:00
wait(delay(0, g_network->getCurrentTask()));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ASSERT(ver > logData->queueCommittedVersion.get());
Initial repository commit
2017-05-26 04:48:44 +08:00
fix: remote logs need to wait until the durable known committed version is greater than the recovery version before completing recovery to ensure we will not pick a start version that we do not have
2018-04-28 03:18:42 +08:00
logData->durableKnownCommittedVersion = knownCommittedVersion;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->unpoppedRecoveredTags == 0 && knownCommittedVersion >= logData->recoveredAt &&
logData->recoveryComplete.canBeSet()) {
TraceEvent("TLogRecoveryComplete", logData->logId)
.detail("Tags", logData->unpoppedRecoveredTags)
.detail("DurableKCVer", logData->durableKnownCommittedVersion)
.detail("RecoveredAt", logData->recoveredAt);
fix: remote logs need to wait until the durable known committed version is greater than the recovery version before completing recovery to ensure we will not pick a start version that we do not have
2018-04-28 03:18:42 +08:00
logData->recoveryComplete.send(Void());
}
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
typo
2021-03-20 04:19:26 +08:00
//TraceEvent("TLogCommitDurable", self->dbgid).detail("Version", ver);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->logSystem->get() &&
(!logData->isPrimary || logData->logRouterPoppedVersion < logData->logRouterPopToVersion)) {
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
logData->logRouterPoppedVersion = ver;
fix: log routers use durable known committed version at the time of the pop to determine what is safe to pop from their logs fix: storage server does not advance its version across large version increase until it has data associated with the version
2018-04-19 03:07:29 +08:00
logData->logSystem->get()->pop(ver, logData->remoteTag, knownCommittedVersion, logData->locality);
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
fix: we cannot set queueCommitEnd until we have popped the log system to prevent the popped version from going backwards
2018-05-02 13:20:25 +08:00
logData->queueCommittedVersion.set(ver);
self->queueCommitEnd.set(commitNumber);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& it : missingFinalCommit) {
TraceEvent("TLogCommitMissingFinalCommit", self->dbgid)
.detail("LogId", logData->logId)
.detail("Version", it->version.get())
.detail("QueueVer", it->queueCommittedVersion.get());
TEST(true); // A TLog was replaced before having a chance to commit its queue
fix: the shared tlog could fail to update a stopped tlog’s queueCommitVersion to version if a second tlog registered before it could issue the first commit for the tlog
2019-03-30 11:11:30 +08:00
it->queueCommittedVersion.set(it->version.get());
}
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> commitQueue(TLogData* self) {
Initial repository commit
2017-05-26 04:48:44 +08:00
state Reference<LogData> logData;
fixed a rare simulation bug where missingFinalCommit could be skipped by two successive logSystem changes
2021-05-20 04:26:01 +08:00
state std::vector<Reference<LogData>> missingFinalCommit;
Initial repository commit
2017-05-26 04:48:44 +08:00
loop {
Fix a case of a bool being used as an integer.
2018-08-11 05:37:35 +08:00
int foundCount = 0;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto it : self->id_data) {
if (!it.second->stopped) {
logData = it.second;
foundCount++;
} else if (it.second->version.get() >
std::max(it.second->queueCommittingVersion, it.second->queueCommittedVersion.get())) {
fix: the shared tlog could fail to update a stopped tlog’s queueCommitVersion to version if a second tlog registered before it could issue the first commit for the tlog
2019-03-30 11:11:30 +08:00
missingFinalCommit.push_back(it.second);
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
ASSERT(foundCount < 2);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!foundCount) {
wait(self->newLogData.onTrigger());
Initial repository commit
2017-05-26 04:48:44 +08:00
continue;
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TraceEvent("CommitQueueNewLog", self->dbgid)
.detail("LogId", logData->logId)
.detail("Version", logData->version.get())
.detail("Committing", logData->queueCommittingVersion)
.detail("Commmitted", logData->queueCommittedVersion.get());
if (logData->committingQueue.canBeSet()) {
fix: do not reply to recruitment until we are sure the log commits to the queue
2018-04-30 13:08:24 +08:00
logData->committingQueue.send(Void());
}
Initial repository commit
2017-05-26 04:48:44 +08:00
loop {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->stopped && logData->version.get() == std::max(logData->queueCommittingVersion,
logData->queueCommittedVersion.get())) {
wait(logData->queueCommittedVersion.whenAtLeast(logData->version.get()));
Initial repository commit
2017-05-26 04:48:44 +08:00
break;
}
choose {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(wait(logData->version.whenAtLeast(
std::max(logData->queueCommittingVersion, logData->queueCommittedVersion.get()) + 1))) {
while (self->queueCommitBegin != self->queueCommitEnd.get() &&
!self->largeDiskQueueCommitBytes.get()) {
wait(self->queueCommitEnd.whenAtLeast(self->queueCommitBegin) ||
self->largeDiskQueueCommitBytes.onChange());
Initial repository commit
2017-05-26 04:48:44 +08:00
}
fix: the shared tlog could fail to update a stopped tlog’s queueCommitVersion to version if a second tlog registered before it could issue the first commit for the tlog
2019-03-30 11:11:30 +08:00
self->sharedActors.send(doQueueCommit(self, logData, missingFinalCommit));
missingFinalCommit.clear();
Initial repository commit
2017-05-26 04:48:44 +08:00
}
Rewrite all `Void _ = wait(...)` -> `wait(...)`. This takes advantage of the new actorcompiler functionality to avoid having duplicate definitions of `Void _` when trying to feed the un-actorompiled source through clang.
2018-08-11 04:57:10 +08:00
when(wait(self->newLogData.onTrigger())) {}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> tLogCommit(TLogData* self,
TLogCommitRequest req,
Reference<LogData> logData,
PromiseStream<Void> warningCollectorInput) {
Add additional trace information
2020-08-29 03:02:51 +08:00
state Span span("TLog:tLogCommit"_loc, req.spanContext);
Initial repository commit
2017-05-26 04:48:44 +08:00
state Optional<UID> tlogDebugID;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.debugID.present()) {
Replace g_random and g_nondeterministic_random with functions deterministicRandom() and nondeterministicRandom() that return thread_local random number generators. Delete g_debug_random and trace_random. Allow only deterministicRandom() to be seeded, and require it to be seeded from each thread on which it is used.
2019-05-11 05:01:52 +08:00
tlogDebugID = nondeterministicRandom()->randomUniqueID();
Initial repository commit
2017-05-26 04:48:44 +08:00
g_traceBatch.addAttach("CommitAttachID", req.debugID.get().first(), tlogDebugID.get().first());
g_traceBatch.addEvent("CommitDebug", tlogDebugID.get().first(), "TLog.tLogCommit.BeforeWaitForVersion");
}
log routers now calculate a precise version to pop for their log router tag
2018-06-22 06:29:46 +08:00
logData->minKnownCommittedVersion = std::max(logData->minKnownCommittedVersion, req.minKnownCommittedVersion);
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(logData->version.whenAtLeast(req.prevVersion));
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// Calling check_yield instead of yield to avoid a destruction ordering problem in simulation
if (g_network->check_yield(g_network->getCurrentTask())) {
Rewrite all `Void _ = wait(...)` -> `wait(...)`. This takes advantage of the new actorcompiler functionality to avoid having duplicate definitions of `Void _` when trying to feed the un-actorompiled source through clang.
2018-08-11 04:57:10 +08:00
wait(delay(0, g_network->getCurrentTask()));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
added a hard memory limit to the TLog to prevent it from running out of memory. Because remote logs are not ratekeeper controlled this is their only protection
2018-06-19 08:22:40 +08:00
state double waitStartT = 0;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
while (self->bytesInput - self->bytesDurable >= SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES && !logData->stopped) {
added a hard memory limit to the TLog to prevent it from running out of memory. Because remote logs are not ratekeeper controlled this is their only protection
2018-06-19 08:22:40 +08:00
if (now() - waitStartT >= 1) {
TraceEvent(SevWarn, "TLogUpdateLag", logData->logId)
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
.detail("Version", logData->version.get())
.detail("PersistentDataVersion", logData->persistentDataVersion)
.detail("PersistentDataDurableVersion", logData->persistentDataDurableVersion);
added a hard memory limit to the TLog to prevent it from running out of memory. Because remote logs are not ratekeeper controlled this is their only protection
2018-06-19 08:22:40 +08:00
waitStartT = now();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(delayJittered(.005, TaskPriority::TLogCommit));
tlog to snapshot exactly at exec version
2019-04-08 20:23:48 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->stopped) {
req.reply.sendError(tlog_stopped());
logData->stop check right after execOpHold wait
2019-05-10 09:15:17 +08:00
return Void();
}
tLog: Track tlog commit latencies in histogram
2020-11-13 07:54:43 +08:00
state double beforeCommitT = now();
// Not a duplicate (check relies on critical section between here self->version.set() below!)
state bool isNotDuplicate = (logData->version.get() == req.prevVersion);
if (isNotDuplicate) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.debugID.present())
Initial repository commit
2017-05-26 04:48:44 +08:00
g_traceBatch.addEvent("CommitDebug", tlogDebugID.get().first(), "TLog.tLogCommit.Before");
Snapshot based backup and resotre implementation
2019-02-28 07:40:33 +08:00
//TraceEvent("TLogCommit", logData->logId).detail("Version", req.version);
commitMessages(self, logData, req.version, req.arena, req.messages);
logData->knownCommittedVersion = std::max(logData->knownCommittedVersion, req.knownCommittedVersion);
remove snap v1 related code
2019-07-23 06:44:49 +08:00
TLogQueueEntryRef qe;
TLog bug fixes
2019-05-26 05:12:18 +08:00
// Log the changes to the persistent queue, to be committed by commitQueue()
qe.version = req.version;
qe.knownCommittedVersion = logData->knownCommittedVersion;
qe.messages = req.messages;
qe.id = logData->logId;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentQueue->push(qe, logData);
Initial repository commit
2017-05-26 04:48:44 +08:00
self->diskQueueCommitBytes += qe.expectedSize();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (self->diskQueueCommitBytes > SERVER_KNOBS->MAX_QUEUE_COMMIT_BYTES) {
Initial repository commit
2017-05-26 04:48:44 +08:00
self->largeDiskQueueCommitBytes.set(true);
}
// Notifies the commitQueue actor to commit persistentQueue, and also unblocks tLogPeekMessages actors
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->version.set(req.version);
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.debugID.present())
Initial repository commit
2017-05-26 04:48:44 +08:00
g_traceBatch.addEvent("CommitDebug", tlogDebugID.get().first(), "TLog.tLogCommit.AfterTLogCommit");
}
// Send replies only once all prior messages have been received and committed.
fix: when a new log is recruited on a shared log, all outstanding commits need to be notified that they are stopped, because there is no longer a guarantee that their queueCommittedVersion will advance
2018-03-20 08:48:28 +08:00
state Future<Void> stopped = logData->stopCommit.onTrigger();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(
timeoutWarning(logData->queueCommittedVersion.whenAtLeast(req.version) || stopped, 0.1, warningCollectorInput));
fix: when a new log is recruited on a shared log, all outstanding commits need to be notified that they are stopped, because there is no longer a guarantee that their queueCommittedVersion will advance
2018-03-20 08:48:28 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (stopped.isReady()) {
fix: when a new log is recruited on a shared log, all outstanding commits need to be notified that they are stopped, because there is no longer a guarantee that their queueCommittedVersion will advance
2018-03-20 08:48:28 +08:00
ASSERT(logData->stopped);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
req.reply.sendError(tlog_stopped());
fix: when a new log is recruited on a shared log, all outstanding commits need to be notified that they are stopped, because there is no longer a guarantee that their queueCommittedVersion will advance
2018-03-20 08:48:28 +08:00
return Void();
}
Initial repository commit
2017-05-26 04:48:44 +08:00
tLog: Track tlog commit latencies in histogram
2020-11-13 07:54:43 +08:00
if (isNotDuplicate) {
self->commitLatencyDist->sampleSeconds(now() - beforeCommitT);
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (req.debugID.present())
Initial repository commit
2017-05-26 04:48:44 +08:00
g_traceBatch.addEvent("CommitDebug", tlogDebugID.get().first(), "TLog.tLogCommit.After");
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
req.reply.send(logData->durableKnownCommittedVersion);
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> initPersistentState(TLogData* self, Reference<LogData> logData) {
wait(self->persistentDataCommitLock.take());
Merge release-6.0 into master
2018-09-22 07:14:39 +08:00
state FlowLock::Releaser commitLockReleaser(self->persistentDataCommitLock);
Initial repository commit
2017-05-26 04:48:44 +08:00
// PERSIST: Initial setup of persistentData for a brand new tLog for a new database
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
state IKeyValueStore* storage = self->persistentData;
wait(ioTimeoutError(storage->init(), SERVER_KNOBS->TLOG_MAX_CREATE_DURATION));
storage->set(persistFormat);
storage->set(
KeyValueRef(BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistCurrentVersionKeys.begin),
BinaryWriter::toValue(logData->version.get(), Unversioned())));
storage->set(KeyValueRef(
BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistKnownCommittedVersionKeys.begin),
BinaryWriter::toValue(logData->knownCommittedVersion, Unversioned())));
storage->set(KeyValueRef(BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistLocalityKeys.begin),
BinaryWriter::toValue(logData->locality, Unversioned())));
storage->set(
KeyValueRef(BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistLogRouterTagsKeys.begin),
BinaryWriter::toValue(logData->logRouterTags, Unversioned())));
storage->set(KeyValueRef(BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistTxsTagsKeys.begin),
BinaryWriter::toValue(logData->txsTags, Unversioned())));
storage->set(
KeyValueRef(BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistRecoveryCountKeys.begin),
BinaryWriter::toValue(logData->recoveryCount, Unversioned())));
storage->set(
KeyValueRef(BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistProtocolVersionKeys.begin),
BinaryWriter::toValue(logData->protocolVersion, Unversioned())));
storage->set(
KeyValueRef(BinaryWriter::toValue(logData->logId, Unversioned()).withPrefix(persistTLogSpillTypeKeys.begin),
BinaryWriter::toValue(logData->logSpillType, AssumeVersion(logData->protocolVersion))));
for (auto tag : logData->allTags) {
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
ASSERT(!logData->getTagData(tag));
logData->createTagData(tag, 0, true, true, true);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
updatePersistentPopped(self, logData, logData->getTagData(tag));
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
TraceEvent("TLogInitCommit", logData->logId);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(ioTimeoutError(self->persistentData->commit(), SERVER_KNOBS->TLOG_MAX_CREATE_DURATION));
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> rejoinMasters(TLogData* self,
TLogInterface tli,
DBRecoveryCount recoveryCount,
Future<Void> registerWithMaster,
bool isPrimary) {
state UID lastMasterID(0, 0);
Initial repository commit
2017-05-26 04:48:44 +08:00
loop {
auto const& inf = self->dbInfo->get();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
bool isDisplaced =
!std::count(inf.priorCommittedLogServers.begin(), inf.priorCommittedLogServers.end(), tli.id());
if (isPrimary) {
isDisplaced =
isDisplaced && inf.recoveryCount >= recoveryCount && inf.recoveryState != RecoveryState::UNINITIALIZED;
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
isDisplaced = isDisplaced &&
((inf.recoveryCount > recoveryCount && inf.recoveryState != RecoveryState::UNINITIALIZED) ||
(inf.recoveryCount == recoveryCount && inf.recoveryState == RecoveryState::FULLY_RECOVERED));
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
}
Refactor: check displacement in LogSystemConfig
2019-05-30 05:45:34 +08:00
isDisplaced = isDisplaced && !inf.logSystemConfig.hasTLog(tli.id());
if (isDisplaced) {
TraceEvent("TLogDisplaced", tli.id())
.detail("Reason", "DBInfoDoesNotContain")
.detail("RecoveryCount", recoveryCount)
.detail("InfRecoveryCount", inf.recoveryCount)
.detail("RecoveryState", (int)inf.recoveryState)
.detail("LogSysConf", describe(inf.logSystemConfig.tLogs))
.detail("PriorLogs", describe(inf.priorCommittedLogServers))
.detail("OldLogGens", inf.logSystemConfig.oldTLogs.size());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (BUGGIFY)
wait(delay(SERVER_KNOBS->BUGGIFY_WORKER_REMOVED_MAX_LAG * deterministicRandom()->random01()));
Initial repository commit
2017-05-26 04:48:44 +08:00
throw worker_removed();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (registerWithMaster.isReady()) {
if (self->dbInfo->get().master.id() != lastMasterID) {
// The TLogRejoinRequest is needed to establish communications with a new master, which doesn't have our
// TLogInterface
Add new api to get shared tlogs id and address
2018-03-03 08:50:30 +08:00
TLogRejoinRequest req(tli);
Removed TLogVersion logging. Added logging of SharedTLog ID for each TLog. Switched ID logged for TLogRejoining event to the TLog instead of the SharedTLog. Made some parameters to startRole passed by reference.
2020-02-15 04:33:43 +08:00
TraceEvent("TLogRejoining", tli.id()).detail("Master", self->dbInfo->get().master.id());
do not register with the master until recovery of the queue is complete, to avoid having the master wait a long time for a peek response
2017-09-19 08:39:12 +08:00
choose {
Replace scalar root types with proper messages
2019-08-29 05:40:50 +08:00
when(TLogRejoinReply rep =
wait(brokenPromiseToNever(self->dbInfo->get().master.tlogRejoin.getReply(req)))) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (rep.masterIsRecovered)
lastMasterID = self->dbInfo->get().master.id();
do not register with the master until recovery of the queue is complete, to avoid having the master wait a long time for a peek response
2017-09-19 08:39:12 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(wait(self->dbInfo->onChange())) {}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
do not register with the master until recovery of the queue is complete, to avoid having the master wait a long time for a peek response
2017-09-19 08:39:12 +08:00
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(self->dbInfo->onChange());
Initial repository commit
2017-05-26 04:48:44 +08:00
}
do not register with the master until recovery of the queue is complete, to avoid having the master wait a long time for a peek response
2017-09-19 08:39:12 +08:00
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(registerWithMaster || self->dbInfo->onChange());
do not register with the master until recovery of the queue is complete, to avoid having the master wait a long time for a peek response
2017-09-19 08:39:12 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> respondToRecovered(TLogInterface tli, Promise<Void> recoveryComplete) {
Merge branch 'master' into feature-remote-logs # Conflicts: # fdbserver/DBCoreState.h # fdbserver/LogSystem.h # fdbserver/LogSystemPeekCursor.actor.cpp # fdbserver/TLogServer.actor.cpp
2018-01-15 05:40:24 +08:00
state bool finishedRecovery = true;
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
try {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(recoveryComplete.getFuture());
} catch (Error& e) {
if (e.code() != error_code_end_of_stream) {
Merge branch 'master' into feature-remote-logs # Conflicts: # fdbserver/DBCoreState.h # fdbserver/LogSystem.h # fdbserver/LogSystemPeekCursor.actor.cpp # fdbserver/TLogServer.actor.cpp
2018-01-15 05:40:24 +08:00
throw;
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
}
Merge branch 'master' into feature-remote-logs # Conflicts: # fdbserver/DBCoreState.h # fdbserver/LogSystem.h # fdbserver/LogSystemPeekCursor.actor.cpp # fdbserver/TLogServer.actor.cpp
2018-01-15 05:40:24 +08:00
finishedRecovery = false;
}
Attempt to normalize trace events: * Detail names now all start with an uppercase character and contain no underscores. Ideally these should be head-first camel case, though that was harder to check. * Type names have the same rules, except they allow one underscore (to support a usage pattern Context_Type). The first character after the underscore is also uppercase. * Use seconds instead of milliseconds in details. Added a check when events are logged in simulation that logs a message to stderr if the first two rules above aren't followed. This probably doesn't address every instance of the above problems, but all of the events I was able to hit in simulation pass the check.
2018-06-09 02:11:08 +08:00
TraceEvent("TLogRespondToRecovered", tli.id()).detail("Finished", finishedRecovery);
Initial repository commit
2017-05-26 04:48:44 +08:00
loop {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TLogRecoveryFinishedRequest req = waitNext(tli.recoveryFinished.getFuture());
if (finishedRecovery) {
fix: the tlog did not cancel recovery properly when stopped
2018-01-13 09:18:14 +08:00
req.reply.send(Void());
} else {
req.reply.send(Never());
}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> cleanupPeekTrackers(LogData* logData) {
Initial repository commit
2017-05-26 04:48:44 +08:00
loop {
fix: peek cursors were being timed out every 10 minutes, instead of 10 minutes after the last use fix: if an interface is changed while we are not waiting in getMore, we will not reset the sequence to 0.
2017-08-10 06:58:06 +08:00
double minTimeUntilExpiration = SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME;
Fix parallel peek stalling for 10min when a TLog generation is destroyed. `peekTracker` was held on the Shared TLog (TLogData), whereas peeks are received and replied to as part of a TLog instance (LogData). When a peek was received on a TLog, it was registered into peekTracker along with the ReplyPromise. If the TLog was then removed as part of a no-longer-needed generation of TLogs, there is nothing left to reply to the request, but by holding onto the ReplyPromise in peekTracker, we leave the remote end with an expectation that we will reply. Then, 10min later, peekTrackerCleanup runs and finally times out the peek cursor, thus preventing FDB from being completely stuck. Now, each TLog generation has its own `peekTracker`, and when a TLog is destroyed, it times out all of the pending peek curors that are still expecting a response. This will then trigger the client to re-issue them to the next generation of TLogs, thus removing the 10min gap to do so.
2019-07-10 08:27:36 +08:00
auto it = logData->peekTracker.begin();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
while (it != logData->peekTracker.end()) {
fix: peek cursors were being timed out every 10 minutes, instead of 10 minutes after the last use fix: if an interface is changed while we are not waiting in getMore, we will not reset the sequence to 0.
2017-08-10 06:58:06 +08:00
double timeUntilExpiration = it->second.lastUpdate + SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME - now();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (timeUntilExpiration < 1.0e-6) {
for (auto seq : it->second.sequence_version) {
if (!seq.second.isSet()) {
Initial repository commit
2017-05-26 04:48:44 +08:00
seq.second.sendError(timed_out());
}
}
Fix parallel peek stalling for 10min when a TLog generation is destroyed. `peekTracker` was held on the Shared TLog (TLogData), whereas peeks are received and replied to as part of a TLog instance (LogData). When a peek was received on a TLog, it was registered into peekTracker along with the ReplyPromise. If the TLog was then removed as part of a no-longer-needed generation of TLogs, there is nothing left to reply to the request, but by holding onto the ReplyPromise in peekTracker, we leave the remote end with an expectation that we will reply. Then, 10min later, peekTrackerCleanup runs and finally times out the peek cursor, thus preventing FDB from being completely stuck. Now, each TLog generation has its own `peekTracker`, and when a TLog is destroyed, it times out all of the pending peek curors that are still expecting a response. This will then trigger the client to re-issue them to the next generation of TLogs, thus removing the 10min gap to do so.
2019-07-10 08:27:36 +08:00
it = logData->peekTracker.erase(it);
Initial repository commit
2017-05-26 04:48:44 +08:00
} else {
fix: peek cursors were being timed out every 10 minutes, instead of 10 minutes after the last use fix: if an interface is changed while we are not waiting in getMore, we will not reset the sequence to 0.
2017-08-10 06:58:06 +08:00
minTimeUntilExpiration = std::min(minTimeUntilExpiration, timeUntilExpiration);
Initial repository commit
2017-05-26 04:48:44 +08:00
++it;
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(delay(minTimeUntilExpiration));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> logPeekTrackers(LogData* logData) {
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
loop {
int64_t logThreshold = 1;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->peekTracker.size() > SERVER_KNOBS->PEEK_LOGGING_AMOUNT) {
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
std::vector<int64_t> peekCounts;
peekCounts.reserve(logData->peekTracker.size());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& it : logData->peekTracker) {
more compile fixes
2020-04-23 05:41:17 +08:00
peekCounts.push_back(it.second.totalPeeks);
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
size_t pivot = peekCounts.size() - SERVER_KNOBS->PEEK_LOGGING_AMOUNT;
std::nth_element(peekCounts.begin(), peekCounts.begin() + pivot, peekCounts.end());
logThreshold = std::max<int64_t>(1, peekCounts[pivot]);
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
}
int logCount = 0;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& it : logData->peekTracker) {
if (it.second.totalPeeks >= logThreshold) {
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
logCount++;
TraceEvent("PeekMetrics", logData->logId)
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
.detail("Tag", it.second.tag.toString())
.detail("Elapsed", now() - it.second.lastLogged)
.detail("MeanReplyBytes", it.second.replyBytes / it.second.totalPeeks)
.detail("TotalPeeks", it.second.totalPeeks)
.detail("UnblockedPeeks", it.second.unblockedPeeks)
.detail("DuplicatePeeks", it.second.duplicatePeeks)
.detail("Sequence",
it.second.sequence_version.size() ? it.second.sequence_version.begin()->first : -1)
.detail("IdleSeconds", it.second.idleTime)
.detail("IdleMax", it.second.idleMax)
.detail("QueueSeconds", it.second.queueTime)
.detail("QueueMax", it.second.queueMax)
.detail("BlockSeconds", it.second.blockTime)
.detail("BlockMax", it.second.blockMax)
.detail("WorkSeconds", it.second.workTime)
.detail("WorkMax", it.second.workMax);
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
it.second.resetMetrics();
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(delay(SERVER_KNOBS->PEEK_LOGGING_DELAY * std::max(1, logCount)));
added logging for parallel peeks on TLogs
2020-04-23 05:24:45 +08:00
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void getQueuingMetrics(TLogData* self, Reference<LogData> logData, TLogQueuingMetricsRequest const& req) {
Initial repository commit
2017-05-26 04:48:44 +08:00
TLogQueuingMetricsReply reply;
reply.localTime = now();
reply.instanceID = self->instanceID;
reply.bytesInput = self->bytesInput;
reply.bytesDurable = self->bytesDurable;
reply.storageBytes = self->persistentData->getStorageBytes();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// FIXME: Add the knownCommittedVersion to this message and change ratekeeper to use that version.
fix: we have to use durableKnownCommittedVersion, because the is the true lower bound on the recovery version of the remote logs fixed a compiler error
2018-09-19 07:29:03 +08:00
reply.v = logData->durableKnownCommittedVersion;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
req.reply.send(reply);
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> tLogSnapCreate(TLogSnapRequest snapReq, TLogData* self, Reference<LogData> logData) {
snap v2: TLog related changes
2019-06-20 02:16:30 +08:00
if (self->ignorePopUid != snapReq.snapUID.toString()) {
snapReq.reply.sendError(operation_failed());
return Void();
}
get rid of all timeouts and other changes
2019-07-21 16:00:29 +08:00
ExecCmdValueString snapArg(snapReq.snapPayload);
snap v2: TLog related changes
2019-06-20 02:16:30 +08:00
try {
Fix #2009 posix compliant args for snapshot binary
2019-08-29 01:52:56 +08:00
int err = wait(execHelper(&snapArg, snapReq.snapUID, self->dataFolder, snapReq.role.toString()));
snap v2: TLog related changes
2019-06-20 02:16:30 +08:00
std::string uidStr = snapReq.snapUID.toString();
TraceEvent("ExecTraceTLog")
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
.detail("Uid", uidStr)
.detail("Status", err)
.detail("Role", snapReq.role)
.detail("Value", self->dataFolder)
.detail("ExecPayload", snapReq.snapPayload)
.detail("PersistentDataVersion", logData->persistentDataVersion)
.detail("PersistentDatadurableVersion", logData->persistentDataDurableVersion)
.detail("QueueCommittedVersion", logData->queueCommittedVersion.get())
.detail("Version", logData->version.get());
snap v2: TLog related changes
2019-06-20 02:16:30 +08:00
few minor fixes
2019-07-13 07:26:28 +08:00
if (err != 0) {
throw operation_failed();
}
snap v2: TLog related changes
2019-06-20 02:16:30 +08:00
snapReq.reply.send(Void());
} catch (Error& e) {
includeCancelled for tLogSnapCreate
2019-07-05 01:20:29 +08:00
TraceEvent("TLogExecHelperError").error(e, true /*includeCancelled */);
get rid of all timeouts and other changes
2019-07-21 16:00:29 +08:00
if (e.code() != error_code_operation_cancelled) {
includeCancelled for tLogSnapCreate
2019-07-05 01:20:29 +08:00
snapReq.reply.sendError(e);
get rid of all timeouts and other changes
2019-07-21 16:00:29 +08:00
} else {
throw e;
includeCancelled for tLogSnapCreate
2019-07-05 01:20:29 +08:00
}
snap v2: TLog related changes
2019-06-20 02:16:30 +08:00
}
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> tLogEnablePopReq(TLogEnablePopRequest enablePopReq, TLogData* self, Reference<LogData> logData) {
enable/disable pop req with UID mis-match to fail
2019-07-05 01:37:33 +08:00
if (self->ignorePopUid != enablePopReq.snapUID.toString()) {
TraceEvent(SevWarn, "TLogPopDisableEnableUidMismatch")
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
.detail("IgnorePopUid", self->ignorePopUid)
.detail("UidStr", enablePopReq.snapUID.toString());
enable/disable pop req with UID mis-match to fail
2019-07-05 01:37:33 +08:00
enablePopReq.reply.sendError(operation_failed());
return Void();
}
Yield while processing ignored pop requests on tlog
2020-11-27 12:36:02 +08:00
TraceEvent("EnableTLogPlayAllIgnoredPops2")
.detail("UidStr", enablePopReq.snapUID.toString())
.detail("IgnorePopUid", self->ignorePopUid)
Addressed review comments
2020-12-03 08:23:13 +08:00
.detail("IgnorePopRequest", self->ignorePopRequest)
Yield while processing ignored pop requests on tlog
2020-11-27 12:36:02 +08:00
.detail("IgnorePopDeadline", self->ignorePopDeadline)
.detail("PersistentDataVersion", logData->persistentDataVersion)
.detail("PersistentDataDurableVersion", logData->persistentDataDurableVersion)
.detail("QueueCommittedVersion", logData->queueCommittedVersion.get())
.detail("Version", logData->version.get());
wait(processPopRequests(self, logData));
enable/disable pop req with UID mis-match to fail
2019-07-05 01:37:33 +08:00
enablePopReq.reply.send(Void());
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> serveTLogInterface(TLogData* self,
TLogInterface tli,
Reference<LogData> logData,
PromiseStream<Void> warningCollectorInput) {
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
state Future<Void> dbInfoChange = Void();
Initial repository commit
2017-05-26 04:48:44 +08:00
loop choose {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(wait(dbInfoChange)) {
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
dbInfoChange = self->dbInfo->onChange();
bool found = false;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (self->dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
for (auto& logs : self->dbInfo->get().logSystemConfig.tLogs) {
if (std::count(logs.tLogs.begin(), logs.tLogs.end(), logData->logId)) {
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
found = true;
break;
}
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (found && self->dbInfo->get().logSystemConfig.recruitmentID == logData->recruitmentID) {
logData->logSystem->set(ILogSystem::fromServerDBInfo(self->dbgid, self->dbInfo->get()));
if (!logData->isPrimary) {
logData->logSystem->get()->pop(logData->logRouterPoppedVersion,
logData->remoteTag,
logData->durableKnownCommittedVersion,
logData->locality);
fix: pop the log routers again after the log system updates
2018-04-20 05:33:31 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!logData->isPrimary && logData->stopped) {
Add LogId to all TLog TraceEvents that have it.
2019-03-16 12:01:12 +08:00
TraceEvent("TLogAlreadyStopped", self->dbgid).detail("LogId", logData->logId);
added tag localities fix: remote logs need to stop the master when they are stopped
2017-08-04 07:16:36 +08:00
logData->removed = logData->removed && logData->logSystem->get()->endEpoch();
}
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
} else {
logData->logSystem->set(Reference<ILogSystem>());
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(TLogPeekRequest req = waitNext(tli.peekMessages.getFuture())) {
logData->addActor.send(tLogPeekMessages(self, req, logData));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(TLogPopRequest req = waitNext(tli.popMessages.getFuture())) {
exec op to all tlog, restore change in test &other - exec operation to go to all the TLogs - minor bug fix in tlog - restore implementation for the simulator - restore snap UID to be stored in restartInfo.ini - test cases added - indentation and trace file fixes
2019-03-11 22:31:44 +08:00
logData->addActor.send(tLogPop(self, req, logData));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(TLogCommitRequest req = waitNext(tli.commit.getFuture())) {
Attempt to normalize trace events: * Detail names now all start with an uppercase character and contain no underscores. Ideally these should be head-first camel case, though that was harder to check. * Type names have the same rules, except they allow one underscore (to support a usage pattern Context_Type). The first character after the underscore is also uppercase. * Use seconds instead of milliseconds in details. Added a check when events are logged in simulation that logs a message to stderr if the first two rules above aren't followed. This probably doesn't address every instance of the above problems, but all of the events I was able to hit in simulation pass the check.
2018-06-09 02:11:08 +08:00
//TraceEvent("TLogCommitReq", logData->logId).detail("Ver", req.version).detail("PrevVer", req.prevVersion).detail("LogVer", logData->version.get());
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
ASSERT(logData->isPrimary);
Initial repository commit
2017-05-26 04:48:44 +08:00
TEST(logData->stopped); // TLogCommitRequest while stopped
if (!logData->stopped)
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->addActor.send(tLogCommit(self, req, logData, warningCollectorInput));
Initial repository commit
2017-05-26 04:48:44 +08:00
else
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
req.reply.sendError(tlog_stopped());
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(ReplyPromise<TLogLockResult> reply = waitNext(tli.lock.getFuture())) {
logData->addActor.send(tLogLock(self, reply, logData));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(TLogQueuingMetricsRequest req = waitNext(tli.getQueuingMetrics.getFuture())) {
added a workload which tests killing an entire region, and recovering from the failure with data loss. fix: we cannot pop the txs tag from remote logs until they have a full copy of the txnStateStore fix: we have to modify all of history, we cannot stop after finding a local remote
2018-09-18 09:32:39 +08:00
getQueuingMetrics(self, logData, req);
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(TLogConfirmRunningRequest req = waitNext(tli.confirmRunning.getFuture())) {
if (req.debugID.present()) {
Replace g_random and g_nondeterministic_random with functions deterministicRandom() and nondeterministicRandom() that return thread_local random number generators. Delete g_debug_random and trace_random. Allow only deterministicRandom() to be seeded, and require it to be seeded from each thread on which it is used.
2019-05-11 05:01:52 +08:00
UID tlogDebugID = nondeterministicRandom()->randomUniqueID();
Initial repository commit
2017-05-26 04:48:44 +08:00
g_traceBatch.addAttach("TransactionAttachID", req.debugID.get().first(), tlogDebugID.first());
g_traceBatch.addEvent("TransactionDebug", tlogDebugID.first(), "TLogServer.TLogConfirmRunningRequest");
}
if (!logData->stopped)
req.reply.send(Void());
else
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
req.reply.sendError(tlog_stopped());
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(TLogDisablePopRequest req = waitNext(tli.disablePopRequest.getFuture())) {
snap v2: TLog related changes
2019-06-20 02:16:30 +08:00
if (self->ignorePopUid != "") {
TraceEvent(SevWarn, "TLogPopDisableonDisable")
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
.detail("IgnorePopUid", self->ignorePopUid)
.detail("UidStr", req.snapUID.toString())
.detail("PersistentDataVersion", logData->persistentDataVersion)
.detail("PersistentDatadurableVersion", logData->persistentDataDurableVersion)
.detail("QueueCommittedVersion", logData->queueCommittedVersion.get())
.detail("Version", logData->version.get());
enable/disable pop req with UID mis-match to fail
2019-07-05 01:37:33 +08:00
req.reply.sendError(operation_failed());
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// FIXME: As part of reverting snapshot V1, make ignorePopUid a UID instead of string
get rid of all timeouts and other changes
2019-07-21 16:00:29 +08:00
self->ignorePopRequest = true;
enable/disable pop req with UID mis-match to fail
2019-07-05 01:37:33 +08:00
self->ignorePopUid = req.snapUID.toString();
self->ignorePopDeadline = g_network->now() + SERVER_KNOBS->TLOG_IGNORE_POP_AUTO_ENABLE_DELAY;
req.reply.send(Void());
snap v2: TLog related changes
2019-06-20 02:16:30 +08:00
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(TLogEnablePopRequest enablePopReq = waitNext(tli.enablePopRequest.getFuture())) {
logData->addActor.send(tLogEnablePopReq(enablePopReq, self, logData));
snap v2: TLog related changes
2019-06-20 02:16:30 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(TLogSnapRequest snapReq = waitNext(tli.snapRequest.getFuture())) {
logData->addActor.send(tLogSnapCreate(snapReq, self, logData));
snap v2: TLog related changes
2019-06-20 02:16:30 +08:00
}
Enforce ignorePopDeadline even when no pop requests are sent
2021-02-22 09:38:29 +08:00
when(wait(self->ignorePopRequest ? delayUntil(self->ignorePopDeadline) : Never())) {
TEST(true); // Hit ignorePopDeadline
TraceEvent("EnableTLogPlayAllIgnoredPops").detail("IgnoredPopDeadline", self->ignorePopDeadline);
logData->addActor.send(processPopRequests(self, logData));
}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void removeLog(TLogData* self, Reference<LogData> logData) {
TraceEvent("TLogRemoved", self->dbgid)
.detail("LogId", logData->logId)
.detail("Input", logData->bytesInput.getValue())
.detail("Durable", logData->bytesDurable.getValue());
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
logData->stopped = true;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!logData->recoveryComplete.isSet()) {
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
logData->recoveryComplete.sendError(end_of_stream());
fix: the tlog did not cancel recovery properly when stopped
2018-01-13 09:18:14 +08:00
}
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->addActor = PromiseStream<Future<Void>>(); // there could be items still in the promise stream if one of the
// actors threw an error immediately
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
self->id_data.erase(logData->logId);
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
while (self->popOrder.size() && !self->id_data.count(self->popOrder.front())) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->popOrder.pop_front();
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
}
Small code refactoring
2019-04-30 01:48:09 +08:00
if (self->id_data.size() == 0) {
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
throw worker_removed();
}
}
Remove unresolved questions
2020-11-12 14:27:52 +08:00
// remote tLog pull data from log routers
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> pullAsyncData(TLogData* self,
Reference<LogData> logData,
std::vector<Tag> tags,
Version beginVersion,
Optional<Version> endVersion,
bool poppedIsKnownCommitted) {
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
state Future<Void> dbInfoChange = Void();
state Reference<ILogSystem::IPeekCursor> r;
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
state Version tagAt = beginVersion;
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
state Version lastVer = 0;
Log Routers will prefer to peek from satellite logs. Formerly, they would prefer to peek from the primary's logs. Testing of a failed region rejoining the cluster revealed that this becomes quite a strain on the primary logs when extremely large volumes of peek requests are coming from the Log Routers. It happens that we have satellites that contain the same mutations with Log Router tags, that have no other peeking load, so we can prefer to use the satellite to peek rather than the primary to distribute load across TLogs better. Unfortunately, this revealed a latent bug in how tagged mutations in the KnownCommittedVersion->RecoveryVersion gap were copied across generations when the number of log router tags were decreased. Satellite TLogs would be assigned log router tags using the team-building based logic in getPushLocations(), whereas TLogs would internally re-index tags according to tag.id%logRouterTags. This mismatch would mean that we could have: Log0 -2:0 ----- -2:0 Log 0 Log1 -2:1 \ >--- -2:1,-2:0 (-2:2 mod 2 becomes -2:0) Log 1 Log2 -2:2 / And now we have data that's tagged as -2:0 on a TLog that's not the preferred location for -2:0, and therefore a BestLocationOnly cursor would miss the mutations. This was never noticed before, as we never used a satellite as a preferred location to peek from. Merge cursors always peek from all locations, and thus a peek for -2:0 that needed data from the satellites would have gone to both TLogs and merged the results. We now take this mod-based re-indexing into account when assigning which TLogs need to recover which tags from the previous generation, to make sure that tag.id%logRouterTags always results in the assigned TLog being the preferred location. Unfortunately, previously existing will potentially have existing satellites with log router tags indexed incorrectly, so this transition needs to be gated on a `log_version` transition. Old LogSets will have an old LogVersion, and we won't prefer the sattelite for peeking. Log Sets post-6.2 (opt-in) or post-6.3 (default) will be indexed correctly, and therefore we can safely offload peeking onto the satellites.
2019-07-09 13:25:01 +08:00
if (endVersion.present()) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TraceEvent("TLogRestoreReplicationFactor", self->dbgid)
.detail("LogId", logData->logId)
.detail("Locality", logData->locality)
.detail("RecoverFrom", beginVersion)
.detail("RecoverTo", endVersion.get());
Log Routers will prefer to peek from satellite logs. Formerly, they would prefer to peek from the primary's logs. Testing of a failed region rejoining the cluster revealed that this becomes quite a strain on the primary logs when extremely large volumes of peek requests are coming from the Log Routers. It happens that we have satellites that contain the same mutations with Log Router tags, that have no other peeking load, so we can prefer to use the satellite to peek rather than the primary to distribute load across TLogs better. Unfortunately, this revealed a latent bug in how tagged mutations in the KnownCommittedVersion->RecoveryVersion gap were copied across generations when the number of log router tags were decreased. Satellite TLogs would be assigned log router tags using the team-building based logic in getPushLocations(), whereas TLogs would internally re-index tags according to tag.id%logRouterTags. This mismatch would mean that we could have: Log0 -2:0 ----- -2:0 Log 0 Log1 -2:1 \ >--- -2:1,-2:0 (-2:2 mod 2 becomes -2:0) Log 1 Log2 -2:2 / And now we have data that's tagged as -2:0 on a TLog that's not the preferred location for -2:0, and therefore a BestLocationOnly cursor would miss the mutations. This was never noticed before, as we never used a satellite as a preferred location to peek from. Merge cursors always peek from all locations, and thus a peek for -2:0 that needed data from the satellites would have gone to both TLogs and merged the results. We now take this mod-based re-indexing into account when assigning which TLogs need to recover which tags from the previous generation, to make sure that tag.id%logRouterTags always results in the assigned TLog being the preferred location. Unfortunately, previously existing will potentially have existing satellites with log router tags indexed incorrectly, so this transition needs to be gated on a `log_version` transition. Old LogSets will have an old LogVersion, and we won't prefer the sattelite for peeking. Log Sets post-6.2 (opt-in) or post-6.3 (default) will be indexed correctly, and therefore we can safely offload peeking onto the satellites.
2019-07-09 13:25:01 +08:00
}
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
while (!endVersion.present() || logData->version.get() < endVersion.get()) {
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
loop {
choose {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(wait(r ? r->getMore(TaskPriority::TLogCommit) : Never())) { break; }
when(wait(dbInfoChange)) {
if (logData->logSystem->get()) {
r = logData->logSystem->get()->peek(logData->logId, tagAt, endVersion, tags, true);
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
} else {
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
r = Reference<ILogSystem::IPeekCursor>();
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
}
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
dbInfoChange = logData->logSystem->onChange();
}
}
}
added a hard memory limit to the TLog to prevent it from running out of memory. Because remote logs are not ratekeeper controlled this is their only protection
2018-06-19 08:22:40 +08:00
state double waitStartT = 0;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
while (self->bytesInput - self->bytesDurable >= SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES && !logData->stopped) {
added a hard memory limit to the TLog to prevent it from running out of memory. Because remote logs are not ratekeeper controlled this is their only protection
2018-06-19 08:22:40 +08:00
if (now() - waitStartT >= 1) {
TraceEvent(SevWarn, "TLogUpdateLag", logData->logId)
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
.detail("Version", logData->version.get())
.detail("PersistentDataVersion", logData->persistentDataVersion)
.detail("PersistentDataDurableVersion", logData->persistentDataDurableVersion);
added a hard memory limit to the TLog to prevent it from running out of memory. Because remote logs are not ratekeeper controlled this is their only protection
2018-06-19 08:22:40 +08:00
waitStartT = now();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(delayJittered(.005, TaskPriority::TLogCommit));
added a hard memory limit to the TLog to prevent it from running out of memory. Because remote logs are not ratekeeper controlled this is their only protection
2018-06-19 08:22:40 +08:00
}
added yields to the log router and tlogs after processing a version
2018-09-05 08:16:44 +08:00
state Version ver = 0;
state std::vector<TagsAndMessage> messages;
loop {
state bool foundMessage = r->hasMessage();
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
if (!foundMessage || r->version().version != ver) {
ASSERT(r->version().version > lastVer);
if (ver) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->stopped || (endVersion.present() && ver > endVersion.get())) {
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
return Void();
}
fix: known committed version must be updated before creating the tlogQueueEntryRef
2018-06-27 14:21:30 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (poppedIsKnownCommitted) {
fix: known committed version must be updated before creating the tlogQueueEntryRef
2018-06-27 14:21:30 +08:00
logData->knownCommittedVersion = std::max(logData->knownCommittedVersion, r->popped());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->minKnownCommittedVersion =
std::max(logData->minKnownCommittedVersion, r->getMinKnownCommittedVersion());
fix: known committed version must be updated before creating the tlogQueueEntryRef
2018-06-27 14:21:30 +08:00
}
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
commitMessages(self, logData, ver, messages);
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (self->terminated.isSet()) {
fix: do not write to the persistent queue when we are terminated, which could happen if shutdown was caused by setting a promise in the asyncPullData loop
2018-07-14 08:01:31 +08:00
return Void();
}
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
// Log the changes to the persistent queue, to be committed by commitQueue()
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
AlternativeTLogQueueEntryRef qe;
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
qe.version = ver;
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
qe.knownCommittedVersion = logData->knownCommittedVersion;
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
qe.alternativeMessages = &messages;
fixed a large number of bugs
2017-07-14 03:29:21 +08:00
qe.id = logData->logId;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentQueue->push(qe, logData);
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
added tag localities fix: remote logs need to stop the master when they are stopped
2017-08-04 07:16:36 +08:00
self->diskQueueCommitBytes += qe.expectedSize();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (self->diskQueueCommitBytes > SERVER_KNOBS->MAX_QUEUE_COMMIT_BYTES) {
added tag localities fix: remote logs need to stop the master when they are stopped
2017-08-04 07:16:36 +08:00
self->largeDiskQueueCommitBytes.set(true);
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// Notifies the commitQueue actor to commit persistentQueue, and also unblocks tLogPeekMessages
// actors
logData->version.set(ver);
wait(yield(TaskPriority::TLogCommit));
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
}
lastVer = ver;
ver = r->version().version;
fix: messages vector was not cleared
2018-03-18 01:24:44 +08:00
messages.clear();
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
if (!foundMessage) {
ver--;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (ver > logData->version.get()) {
if (logData->stopped || (endVersion.present() && ver > endVersion.get())) {
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
return Void();
}
fix: known committed version must be updated before creating the tlogQueueEntryRef
2018-06-27 14:21:30 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (poppedIsKnownCommitted) {
fix: known committed version must be updated before creating the tlogQueueEntryRef
2018-06-27 14:21:30 +08:00
logData->knownCommittedVersion = std::max(logData->knownCommittedVersion, r->popped());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->minKnownCommittedVersion =
std::max(logData->minKnownCommittedVersion, r->getMinKnownCommittedVersion());
fix: known committed version must be updated before creating the tlogQueueEntryRef
2018-06-27 14:21:30 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (self->terminated.isSet()) {
fix: do not write to the persistent queue when we are terminated, which could happen if shutdown was caused by setting a promise in the asyncPullData loop
2018-07-14 08:01:31 +08:00
return Void();
}
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
// Log the changes to the persistent queue, to be committed by commitQueue()
TLogQueueEntryRef qe;
qe.version = ver;
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
qe.knownCommittedVersion = logData->knownCommittedVersion;
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
qe.messages = StringRef();
fixed a large number of bugs
2017-07-14 03:29:21 +08:00
qe.id = logData->logId;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentQueue->push(qe, logData);
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
added tag localities fix: remote logs need to stop the master when they are stopped
2017-08-04 07:16:36 +08:00
self->diskQueueCommitBytes += qe.expectedSize();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (self->diskQueueCommitBytes > SERVER_KNOBS->MAX_QUEUE_COMMIT_BYTES) {
added tag localities fix: remote logs need to stop the master when they are stopped
2017-08-04 07:16:36 +08:00
self->largeDiskQueueCommitBytes.set(true);
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// Notifies the commitQueue actor to commit persistentQueue, and also unblocks tLogPeekMessages
// actors
logData->version.set(ver);
wait(yield(TaskPriority::TLogCommit));
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
}
break;
}
}
Use emplace_back instead of push_back for TLogServer
2019-04-25 01:59:19 +08:00
messages.emplace_back(r->getMessageWithTags(), r->getTags());
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
r->nextMessage();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
tagAt = std::max(r->version().version, logData->version.get() + 1);
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
}
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
return Void();
changed to using a vector of logSets instead of a duplicate set of logs for remote servers finished porting changes to the tlog everything but peeking is finished in the TagPartitionedLogSystem
2017-07-10 05:46:16 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> tLogCore(TLogData* self,
Reference<LogData> logData,
TLogInterface tli,
bool pulledRecoveryVersions) {
if (logData->removed.isReady()) {
wait(delay(0)); // to avoid iterator invalidation in restorePersistentState when removed is already ready
Initial repository commit
2017-05-26 04:48:44 +08:00
ASSERT(logData->removed.isError());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->removed.getError().code() != error_code_worker_removed) {
Initial repository commit
2017-05-26 04:48:44 +08:00
throw logData->removed.getError();
}
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
removeLog(self, logData);
return Void();
Initial repository commit
2017-05-26 04:48:44 +08:00
}
state PromiseStream<Void> warningCollectorInput;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
state Future<Void> warningCollector =
timeoutWarningCollector(warningCollectorInput.getFuture(), 1.0, "TLogQueueCommitSlow", self->dbgid);
state Future<Void> error = actorCollection(logData->addActor.getFuture());
logData->addActor.send(waitFailureServer(tli.waitFailure.getFuture()));
logData->addActor.send(logData->removed);
// FIXME: update tlogMetrics to include new information, or possibly only have one copy for the shared instance
logData->addActor.send(traceCounters("TLogMetrics",
logData->logId,
SERVER_KNOBS->STORAGE_LOGGING_DELAY,
&logData->cc,
Added new StorageBytes member to StorageMetrics and TLogMetrics (for newest TLog version only). Moved StorageBytes detail from SpecialCounters to the traceCounters() decorator callback to avoid calling getStorageBytes(), which makes a system call, four extra times on storage servers and eight extra times on logs.
2021-04-08 16:09:47 +08:00
logData->logId.toString() + "/TLogMetrics",
Revert "Make the sim2 run loop match the behavior of the net2 run loop."
2021-06-23 05:50:01 +08:00
[self=self](TraceEvent& te) {
Added new StorageBytes member to StorageMetrics and TLogMetrics (for newest TLog version only). Moved StorageBytes detail from SpecialCounters to the traceCounters() decorator callback to avoid calling getStorageBytes(), which makes a system call, four extra times on storage servers and eight extra times on logs.
2021-04-08 16:09:47 +08:00
StorageBytes sbTlog = self->persistentData->getStorageBytes();
te.detail("KvstoreBytesUsed", sbTlog.used);
te.detail("KvstoreBytesFree", sbTlog.free);
te.detail("KvstoreBytesAvailable", sbTlog.available);
te.detail("KvstoreBytesTotal", sbTlog.total);
te.detail("KvstoreBytesTemp", sbTlog.temp);
StorageBytes sbQueue = self->rawPersistentQueue->getStorageBytes();
te.detail("QueueDiskBytesUsed", sbQueue.used);
te.detail("QueueDiskBytesFree", sbQueue.free);
te.detail("QueueDiskBytesAvailable", sbQueue.available);
te.detail("QueueDiskBytesTotal", sbQueue.total);
te.detail("QueueDiskBytesTemp", sbQueue.temp);
}));
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->addActor.send(serveTLogInterface(self, tli, logData, warningCollectorInput));
logData->addActor.send(cleanupPeekTrackers(logData.getPtr()));
logData->addActor.send(logPeekTrackers(logData.getPtr()));
if (!logData->isPrimary) {
Revert "fix: remote logs should reply until they have recovered through recoverAt" This reverts commit 3c0c03c004705164ddc2dae48f93f1df1806b986.
2018-04-17 14:17:16 +08:00
std::vector<Tag> tags;
tags.push_back(logData->remoteTag);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->addActor.send(
pullAsyncData(self,
logData,
tags,
pulledRecoveryVersions ? logData->recoveredAt + 1 : logData->unrecoveredBefore,
Optional<Version>(),
true));
Revert "fix: remote logs should reply until they have recovered through recoverAt" This reverts commit 3c0c03c004705164ddc2dae48f93f1df1806b986.
2018-04-17 14:17:16 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
try {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(error);
Initial repository commit
2017-05-26 04:48:44 +08:00
throw internal_error();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
} catch (Error& e) {
if (e.code() != error_code_worker_removed)
Initial repository commit
2017-05-26 04:48:44 +08:00
throw;
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
removeLog(self, logData);
return Void();
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
ACTOR Future<Void> checkEmptyQueue(TLogData* self) {
TraceEvent("TLogCheckEmptyQueueBegin", self->dbgid);
try {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
bool recoveryFinished = wait(self->persistentQueue->initializeRecovery(0));
Implement limiting how many bytes recovery will read. This time, track what location in the DiskQueue has been spilled in persistent state, and then feed it back into the disk queue before recovery. This also introduces an ASSERT that recovery only reads exactly the bytes that it needs to have in memory.
2019-03-16 12:01:22 +08:00
if (recoveryFinished)
return Void();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TLogQueueEntry r = wait(self->persistentQueue->readNext(self));
Initial repository commit
2017-05-26 04:48:44 +08:00
throw internal_error();
} catch (Error& e) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (e.code() != error_code_end_of_stream)
throw;
Initial repository commit
2017-05-26 04:48:44 +08:00
TraceEvent("TLogCheckEmptyQueueEnd", self->dbgid);
return Void();
}
}
fix: tlogFitness did not consider it better to have one tlog of a better fitness fix: checkStable was not used in all places in better master exists fix: we need to call checkOutstanding on worker registration in all cases fix: in case persistentData is keyValueStoreMemory, we need to make sure it is fully recovered before writing to it
2018-01-05 03:33:02 +08:00
ACTOR Future<Void> checkRecovered(TLogData* self) {
TraceEvent("TLogCheckRecoveredBegin", self->dbgid);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Optional<Value> v = wait(self->persistentData->readValue(StringRef()));
fix: tlogFitness did not consider it better to have one tlog of a better fitness fix: checkStable was not used in all places in better master exists fix: we need to call checkOutstanding on worker registration in all cases fix: in case persistentData is keyValueStoreMemory, we need to make sure it is fully recovered before writing to it
2018-01-05 03:33:02 +08:00
TraceEvent("TLogCheckRecoveredEnd", self->dbgid);
return Void();
}
Comments:Add comments to DiskQueue No functional change.
2019-07-30 10:17:10 +08:00
// Recovery persistent state of tLog from disk
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> restorePersistentState(TLogData* self,
LocalityData locality,
Promise<Void> oldLog,
Promise<Void> recovered,
PromiseStream<InitializeTLogRequest> tlogRequests) {
Initial repository commit
2017-05-26 04:48:44 +08:00
state double startt = now();
state Reference<LogData> logData;
state KeyRange tagKeys;
// PERSIST: Read basic state from persistentData; replay persistentQueue but don't erase it
TraceEvent("TLogRestorePersistentState", self->dbgid);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
state IKeyValueStore* storage = self->persistentData;
Merge branch 'master' of github.com:apple/foundationdb into feature-redwood # Conflicts: # fdbrpc/AsyncFileCached.actor.h # fdbserver/IKeyValueStore.h # fdbserver/KeyValueStoreMemory.actor.cpp # fdbserver/workloads/StatusWorkload.actor.cpp # tests/fast/SidebandWithStatus.txt # tests/rare/LargeApiCorrectnessStatus.txt # tests/slow/DDBalanceAndRemoveStatus.txt
2018-09-20 18:39:55 +08:00
wait(storage->init());
Initial repository commit
2017-05-26 04:48:44 +08:00
state Future<Optional<Value>> fFormat = storage->readValue(persistFormat.key);
Implement limiting how many bytes recovery will read. This time, track what location in the DiskQueue has been spilled in persistent state, and then feed it back into the disk queue before recovery. This also introduces an ASSERT that recovery only reads exactly the bytes that it needs to have in memory.
2019-03-16 12:01:22 +08:00
state Future<Optional<Value>> fRecoveryLocation = storage->readValue(persistRecoveryLocationKey);
Create RangeResult type alias
2021-05-04 04:14:16 +08:00
state Future<RangeResult> fVers = storage->readRange(persistCurrentVersionKeys);
state Future<RangeResult> fKnownCommitted = storage->readRange(persistKnownCommittedVersionKeys);
state Future<RangeResult> fLocality = storage->readRange(persistLocalityKeys);
state Future<RangeResult> fLogRouterTags = storage->readRange(persistLogRouterTagsKeys);
state Future<RangeResult> fTxsTags = storage->readRange(persistTxsTagsKeys);
state Future<RangeResult> fRecoverCounts = storage->readRange(persistRecoveryCountKeys);
state Future<RangeResult> fProtocolVersions = storage->readRange(persistProtocolVersionKeys);
state Future<RangeResult> fTLogSpillTypes = storage->readRange(persistTLogSpillTypeKeys);
Initial repository commit
2017-05-26 04:48:44 +08:00
// FIXME: metadata in queue?
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(waitForAll(std::vector{ fFormat, fRecoveryLocation }));
wait(waitForAll(std::vector{ fVers,
fKnownCommitted,
fLocality,
fLogRouterTags,
fTxsTags,
fRecoverCounts,
fProtocolVersions,
fTLogSpillTypes }));
if (fFormat.get().present() && !persistFormatReadableRange.contains(fFormat.get().get())) {
// FIXME: remove when we no longer need to test upgrades from 4.X releases
if (g_network->isSimulated()) {
fix: only flush and exit in simulation
2018-06-15 04:48:30 +08:00
TraceEvent("ElapsedTime").detail("SimTime", now()).detail("RealTime", 0).detail("RandomUnseed", 0);
flushAndExit(0);
}
upgrades from 5.X mostly supported, still some remaining correctness problems
2018-01-29 03:52:54 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TraceEvent(SevError, "UnsupportedDBFormat", self->dbgid)
.detail("Format", fFormat.get().get())
.detail("Expected", persistFormat.value.toString());
Initial repository commit
2017-05-26 04:48:44 +08:00
throw worker_recovery_failed();
}
if (!fFormat.get().present()) {
Create RangeResult type alias
2021-05-04 04:14:16 +08:00
RangeResult v = wait(self->persistentData->readRange(KeyRangeRef(StringRef(), LiteralStringRef("\xff")), 1));
Initial repository commit
2017-05-26 04:48:44 +08:00
if (!v.size()) {
TEST(true); // The DB is completely empty, so it was never initialized. Delete it.
throw worker_removed();
} else {
// This should never happen
Remove trace-calls to printable (in non-workloads)
2019-03-19 06:03:43 +08:00
TraceEvent(SevError, "NoDBFormatKey", self->dbgid).detail("FirstKey", v[0].key);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ASSERT(false);
Initial repository commit
2017-05-26 04:48:44 +08:00
throw worker_recovery_failed();
}
}
state std::vector<Future<ErrorOr<Void>>> removed;
Bump persistFormat in TLogServer to differ from OldTLogServer* Though this format is being deprecated in favor of an eventual plumbing through of TLogVersion, we should probably bump it anyway. And also remove the fallback to OldTLogServer code. It should never be executed, as OldTLogServer_6_0 is entirely relied upon to execute OldTLogServer_4_6.
2019-03-16 12:01:13 +08:00
ASSERT(fFormat.get().get() == LiteralStringRef("FoundationDB/LogServer/3/0"));
Initial repository commit
2017-05-26 04:48:44 +08:00
ASSERT(fVers.get().size() == fRecoverCounts.get().size());
got rid of persistUnrecoveredBefore added persistLocality
2018-06-18 05:44:33 +08:00
state std::map<UID, int8_t> id_locality;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto it : fLocality.get()) {
id_locality[BinaryReader::fromStringRef<UID>(it.key.removePrefix(persistLocalityKeys.begin), Unversioned())] =
BinaryReader::fromStringRef<int8_t>(it.value, Unversioned());
fix: after recovering from disk, only allow peeking data the was fully recovered
2018-01-07 05:49:13 +08:00
}
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
state std::map<UID, int> id_logRouterTags;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto it : fLogRouterTags.get()) {
id_logRouterTags[BinaryReader::fromStringRef<UID>(it.key.removePrefix(persistLogRouterTagsKeys.begin),
Unversioned())] =
BinaryReader::fromStringRef<int>(it.value, Unversioned());
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
}
shard the txs tag so that more transaction logs are involved in its recovery
2019-06-20 09:15:09 +08:00
state std::map<UID, int> id_txsTags;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto it : fTxsTags.get()) {
id_txsTags[BinaryReader::fromStringRef<UID>(it.key.removePrefix(persistTxsTagsKeys.begin), Unversioned())] =
BinaryReader::fromStringRef<int>(it.value, Unversioned());
shard the txs tag so that more transaction logs are involved in its recovery
2019-06-20 09:15:09 +08:00
}
fix: persist the known committed version on the tlogs
2018-04-21 08:55:46 +08:00
state std::map<UID, Version> id_knownCommitted;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto it : fKnownCommitted.get()) {
id_knownCommitted[BinaryReader::fromStringRef<UID>(it.key.removePrefix(persistKnownCommittedVersionKeys.begin),
Unversioned())] =
BinaryReader::fromStringRef<Version>(it.value, Unversioned());
fix: persist the known committed version on the tlogs
2018-04-21 08:55:46 +08:00
}
Implement limiting how many bytes recovery will read. This time, track what location in the DiskQueue has been spilled in persistent state, and then feed it back into the disk queue before recovery. This also introduces an ASSERT that recovery only reads exactly the bytes that it needs to have in memory.
2019-03-16 12:01:22 +08:00
state IDiskQueue::location minimumRecoveryLocation = 0;
if (fRecoveryLocation.get().present()) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
minimumRecoveryLocation =
BinaryReader::fromStringRef<IDiskQueue::location>(fRecoveryLocation.get().get(), Unversioned());
Implement limiting how many bytes recovery will read. This time, track what location in the DiskQueue has been spilled in persistent state, and then feed it back into the disk queue before recovery. This also introduces an ASSERT that recovery only reads exactly the bytes that it needs to have in memory.
2019-03-16 12:01:22 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
state int idx = 0;
do not register with the master until recovery of the queue is complete, to avoid having the master wait a long time for a peek response
2017-09-19 08:39:12 +08:00
state Promise<Void> registerWithMaster;
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
state std::map<UID, TLogInterface> id_interf;
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
state std::vector<std::pair<Version, UID>> logsByVersion;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (idx = 0; idx < fVers.get().size(); idx++) {
Initial repository commit
2017-05-26 04:48:44 +08:00
state KeyRef rawId = fVers.get()[idx].key.removePrefix(persistCurrentVersionKeys.begin);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
UID id1 = BinaryReader::fromStringRef<UID>(rawId, Unversioned());
UID id2 = BinaryReader::fromStringRef<UID>(
fRecoverCounts.get()[idx].key.removePrefix(persistRecoveryCountKeys.begin), Unversioned());
Initial repository commit
2017-05-26 04:48:44 +08:00
ASSERT(id1 == id2);
Addressed review comments.
2018-03-03 09:56:49 +08:00
TLogInterface recruited(id1, self->dbgid, locality);
Initial repository commit
2017-05-26 04:48:44 +08:00
recruited.initEndpoints();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
DUMPTOKEN(recruited.peekMessages);
DUMPTOKEN(recruited.popMessages);
DUMPTOKEN(recruited.commit);
DUMPTOKEN(recruited.lock);
DUMPTOKEN(recruited.getQueuingMetrics);
DUMPTOKEN(recruited.confirmRunning);
DUMPTOKEN(recruited.waitFailure);
DUMPTOKEN(recruited.recoveryFinished);
DUMPTOKEN(recruited.disablePopRequest);
DUMPTOKEN(recruited.enablePopRequest);
DUMPTOKEN(recruited.snapRequest);
ProtocolVersion protocolVersion =
BinaryReader::fromStringRef<ProtocolVersion>(fProtocolVersions.get()[idx].value, Unversioned());
TLogSpillType logSpillType = BinaryReader::fromStringRef<TLogSpillType>(fTLogSpillTypes.get()[idx].value,
AssumeVersion(protocolVersion));
// We do not need the remoteTag, because we will not be loading any additional data
logData = makeReference<LogData>(self,
recruited,
Tag(),
true,
id_logRouterTags[id1],
id_txsTags[id1],
UID(),
protocolVersion,
logSpillType,
std::vector<Tag>(),
"Restored");
got rid of persistUnrecoveredBefore added persistLocality
2018-06-18 05:44:33 +08:00
logData->locality = id_locality[id1];
Initial repository commit
2017-05-26 04:48:44 +08:00
logData->stopped = true;
self->id_data[id1] = logData;
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
id_interf[id1] = recruited;
Initial repository commit
2017-05-26 04:48:44 +08:00
fix: persist the known committed version on the tlogs
2018-04-21 08:55:46 +08:00
logData->knownCommittedVersion = id_knownCommitted[id1];
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
Version ver = BinaryReader::fromStringRef<Version>(fVers.get()[idx].value, Unversioned());
Initial repository commit
2017-05-26 04:48:44 +08:00
logData->persistentDataVersion = ver;
logData->persistentDataDurableVersion = ver;
logData->version.set(ver);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->recoveryCount =
BinaryReader::fromStringRef<DBRecoveryCount>(fRecoverCounts.get()[idx].value, Unversioned());
logData->removed =
rejoinMasters(self, recruited, logData->recoveryCount, registerWithMaster.getFuture(), false);
Initial repository commit
2017-05-26 04:48:44 +08:00
removed.push_back(errorOr(logData->removed));
Use emplace_back instead of push_back for TLogServer
2019-04-25 01:59:19 +08:00
logsByVersion.emplace_back(ver, id1);
Initial repository commit
2017-05-26 04:48:44 +08:00
add epoch info to trace events when tLog begins
2021-06-10 08:14:36 +08:00
TraceEvent("TLogPersistentStateRestore", self->dbgid)
.detail("LogId", logData->logId)
.detail("Ver", ver)
.detail("RecoveryCount", logData->recoveryCount);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// Restore popped keys. Pop operations that took place after the last (committed) updatePersistentDataVersion
// might be lost, but that is fine because we will get the corresponding data back, too.
tagKeys = prefixRange(rawId.withPrefix(persistTagPoppedKeys.begin));
Initial repository commit
2017-05-26 04:48:44 +08:00
loop {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->removed.isReady())
break;
Create RangeResult type alias
2021-05-04 04:14:16 +08:00
RangeResult data = wait(self->persistentData->readRange(tagKeys, BUGGIFY ? 3 : 1 << 30, 1 << 20));
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!data.size())
break;
((KeyRangeRef&)tagKeys) = KeyRangeRef(keyAfter(data.back().key, tagKeys.arena()), tagKeys.end);
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& kv : data) {
Initial repository commit
2017-05-26 04:48:44 +08:00
Tag tag = decodeTagPoppedKey(rawId, kv.key);
Version popped = decodeTagPoppedValue(kv.value);
changed a trace event name to avoid scope type mismatch on the tag field
2018-08-04 06:53:38 +08:00
TraceEvent("TLogRestorePopped", logData->logId).detail("Tag", tag.toString()).detail("To", popped);
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
auto tagData = logData->getTagData(tag);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ASSERT(!tagData);
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
logData->createTagData(tag, popped, false, false, false);
DiskQueue should only pop based off of persisted popped tag versions This commit is to fix a bug where popping a tag between updatePersistentData and popDiskQueue can cause the TLog to recover to an incorrect understanding of what data it has available. The following series of events need to happen to trigger this bug: Tag 1:1 is popped to version 10 updatePersistentData is run... updatePersistentPopped runs and we persistentData stores 1:1 as popped to 10 A mutation is spilled for 1:1 at version 11 at location 1000 A mutation is spilled for 1:1 at version 21 at location 5000 updatePersistentData finishes and commits the btree changes Tag 1:1 is popped to version 20 popDiskQueue runs The btree is read for spilled mutations with version >=20 The minimum location required for the disk queue is found to be location 5000 The disk queue is popped to location 5000 The TLog crashes The worker restarts, and reloads the TLog files from disk restorePersistentPopped restores tag 1:1 as having been popped to version 10 Parallel peeks are received for tag 1:1 starting at version 0 The first peek is less than the popped version, so we respond with no data, and an end version of 10 The second peek starts at version 10, which is greater than the popped version The btree is read for spilled mutations, and we find that there is a mutation at version 11 at location 1000 Location 1000 is read in the DiskQueue The resulting page read at Location 1000 was popped pre-crash, and thus might either (a) be corrupt or (b) have an incorrect sequence number. The fix to this is to force popDiskQueue/updatePoppedLocation to use the popped version that was persisted to disk, and not the most recently popped version for the given tag. This bug doesn't manifest in simulation, because we don't have any code that peeks at a lower version than what has been popped.
2019-12-13 09:16:22 +08:00
logData->getTagData(tag)->persistentPopped = popped;
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
}
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
std::sort(logsByVersion.begin(), logsByVersion.end());
for (const auto& pair : logsByVersion) {
// TLogs that have been fully spilled won't have queue entries read in the loop below.
self->popOrder.push_back(pair.second);
}
logsByVersion.clear();
Initial repository commit
2017-05-26 04:48:44 +08:00
state Future<Void> allRemoved = waitForAll(removed);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
state UID lastId = UID(1, 1); // initialized so it will not compare equal to a default UID
added a separate knob for tlog’s recoverMemoryLimit
2018-08-22 12:11:23 +08:00
state double recoverMemoryLimit = SERVER_KNOBS->TLOG_RECOVER_MEMORY_LIMIT;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (BUGGIFY)
recoverMemoryLimit =
std::max<double>(SERVER_KNOBS->BUGGIFY_RECOVER_MEMORY_LIMIT, (double)SERVER_KNOBS->TLOG_SPILL_THRESHOLD);
fix: avoid flushing large queues in simulation when checking latency
2017-11-28 09:23:20 +08:00
Initial repository commit
2017-05-26 04:48:44 +08:00
try {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
bool recoveryFinished = wait(self->persistentQueue->initializeRecovery(minimumRecoveryLocation));
Implement limiting how many bytes recovery will read. This time, track what location in the DiskQueue has been spilled in persistent state, and then feed it back into the disk queue before recovery. This also introduces an ASSERT that recovery only reads exactly the bytes that it needs to have in memory.
2019-03-16 12:01:22 +08:00
if (recoveryFinished)
throw end_of_stream();
Initial repository commit
2017-05-26 04:48:44 +08:00
loop {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (allRemoved.isReady()) {
TEST(true); // all tlogs removed during queue recovery
Initial repository commit
2017-05-26 04:48:44 +08:00
throw worker_removed();
}
choose {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(TLogQueueEntry qe = wait(self->persistentQueue->readNext(self))) {
if (qe.id != lastId) {
Initial repository commit
2017-05-26 04:48:44 +08:00
lastId = qe.id;
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
auto it = self->id_data.find(qe.id);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (it != self->id_data.end()) {
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
logData = it->second;
} else {
logData = Reference<LogData>();
}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
Revert "Make the sim2 run loop match the behavior of the net2 run loop."
2021-06-23 05:50:01 +08:00
//TraceEvent("TLogRecoveredQE", self->dbgid).detail("LogId", qe.id).detail("Ver", qe.version).detail("MessageBytes", qe.messages.size()).detail("Tags", qe.tags.size())
// .detail("Tag0", qe.tags.size() ? qe.tags[0].tag : invalidTag).detail("Version",
// logData->version.get());
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData) {
if (!self->spillOrder.size() || self->spillOrder.back() != qe.id) {
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
self->spillOrder.push_back(qe.id);
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->knownCommittedVersion =
std::max(logData->knownCommittedVersion, qe.knownCommittedVersion);
if (qe.version > logData->version.get()) {
added separate logging for overhead bytes
2018-08-22 13:18:38 +08:00
commitMessages(self, logData, qe.version, qe.arena(), qe.messages);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->version.set(qe.version);
logData->queueCommittedVersion.set(qe.version);
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
while (self->bytesInput - self->bytesDurable >= recoverMemoryLimit) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TEST(true); // Flush excess data during TLog queue recovery
TraceEvent("FlushLargeQueueDuringRecovery", self->dbgid)
.detail("LogId", logData->logId)
.detail("BytesInput", self->bytesInput)
.detail("BytesDurable", self->bytesDurable)
.detail("Version", logData->version.get())
.detail("PVer", logData->persistentDataVersion);
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
choose {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(wait(updateStorage(self))) {}
when(wait(allRemoved)) { throw worker_removed(); }
The TLog did not delete data from removed logs The TLog continued to make data from removed logs persistent
2017-08-12 09:08:09 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
Implement limiting how many bytes recovery will read. This time, track what location in the DiskQueue has been spilled in persistent state, and then feed it back into the disk queue before recovery. This also introduces an ASSERT that recovery only reads exactly the bytes that it needs to have in memory.
2019-03-16 12:01:22 +08:00
} else {
// Updating persistRecoveryLocation and persistCurrentVersion at the same time,
// transactionally, should mean that we never read any TLogQueueEntry that has already
// been spilled.
fix: make sure recoveryLocation is always a valid page
2019-03-21 09:12:56 +08:00
ASSERT_WE_THINK(qe.version == logData->version.get());
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(wait(allRemoved)) { throw worker_removed(); }
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
} catch (Error& e) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (e.code() != error_code_end_of_stream)
throw;
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TraceEvent("TLogRestorePersistentStateDone", self->dbgid).detail("Took", now() - startt);
TEST(now() - startt >= 1.0); // TLog recovery took more than 1 second
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto it : self->id_data) {
if (it.second->queueCommittedVersion.get() == 0) {
Attempt to normalize trace events: * Detail names now all start with an uppercase character and contain no underscores. Ideally these should be head-first camel case, though that was harder to check. * Type names have the same rules, except they allow one underscore (to support a usage pattern Context_Type). The first character after the underscore is also uppercase. * Use seconds instead of milliseconds in details. Added a check when events are logged in simulation that logs a message to stderr if the first two rules above aren't followed. This probably doesn't address every instance of the above problems, but all of the events I was able to hit in simulation pass the check.
2018-06-09 02:11:08 +08:00
TraceEvent("TLogZeroVersion", self->dbgid).detail("LogId", it.first);
Initial repository commit
2017-05-26 04:48:44 +08:00
it.second->queueCommittedVersion.set(it.second->version.get());
}
fix: after recovering from disk, only allow peeking data the was fully recovered
2018-01-07 05:49:13 +08:00
it.second->recoveryComplete.sendError(end_of_stream());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->sharedActors.send(tLogCore(self, it.second, id_interf[it.first], false));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (registerWithMaster.canBeSet())
registerWithMaster.send(Void());
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
bool tlogTerminated(TLogData* self, IKeyValueStore* persistentData, TLogQueue* persistentQueue, Error const& e) {
// Dispose the IKVS (destroying its data permanently) only if this shutdown is definitely permanent. Otherwise just
// close it.
Initial repository commit
2017-05-26 04:48:44 +08:00
if (e.code() == error_code_worker_removed || e.code() == error_code_recruitment_failed) {
persistentData->dispose();
persistentQueue->dispose();
} else {
persistentData->close();
persistentQueue->close();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (e.code() == error_code_worker_removed || e.code() == error_code_recruitment_failed ||
e.code() == error_code_file_not_found) {
Initial repository commit
2017-05-26 04:48:44 +08:00
TraceEvent("TLogTerminated", self->dbgid).error(e, true);
return true;
} else
return false;
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> updateLogSystem(TLogData* self,
Reference<LogData> logData,
LogSystemConfig recoverFrom,
Reference<AsyncVar<Reference<ILogSystem>>> logSystem) {
Initial repository commit
2017-05-26 04:48:44 +08:00
loop {
Small code refactoring
2019-04-30 01:48:09 +08:00
bool found = self->dbInfo->get().logSystemConfig.recruitmentID == logData->recruitmentID;
if (found) {
if (self->dbInfo->get().logSystemConfig.isNextGenerationOf(recoverFrom)) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logSystem->set(ILogSystem::fromOldLogSystemConfig(
logData->logId, self->dbInfo->get().myLocality, self->dbInfo->get().logSystemConfig));
Small code refactoring
2019-04-30 01:48:09 +08:00
} else if (self->dbInfo->get().logSystemConfig.isEqualIds(recoverFrom)) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logSystem->set(ILogSystem::fromLogSystemConfig(
logData->logId, self->dbInfo->get().myLocality, self->dbInfo->get().logSystemConfig, false, true));
Small code refactoring
2019-04-30 01:48:09 +08:00
} else if (self->dbInfo->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logSystem->set(ILogSystem::fromLogSystemConfig(
logData->logId, self->dbInfo->get().myLocality, self->dbInfo->get().logSystemConfig, true));
Small code refactoring
2019-04-30 01:48:09 +08:00
} else {
found = false;
a cleaner solution to preventing tlogs from peeking log routers
2018-04-21 04:25:22 +08:00
}
fix: do not peek from remote logs if they are not fully recovered
2018-02-22 06:06:44 +08:00
}
Small code refactoring
2019-04-30 01:48:09 +08:00
if (!found) {
Initial repository commit
2017-05-26 04:48:44 +08:00
logSystem->set(Reference<ILogSystem>());
fix: pop the log routers again after the log system updates
2018-04-20 05:33:31 +08:00
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->logSystem->get()->pop(logData->logRouterPoppedVersion,
logData->remoteTag,
logData->durableKnownCommittedVersion,
logData->locality);
}
TraceEvent("TLogUpdate", self->dbgid)
.detail("LogId", logData->logId)
.detail("RecruitmentID", logData->recruitmentID)
.detail("DbRecruitmentID", self->dbInfo->get().logSystemConfig.recruitmentID)
.detail("RecoverFrom", recoverFrom.toString())
.detail("DbInfo", self->dbInfo->get().logSystemConfig.toString())
.detail("Found", found)
.detail("LogSystem", (bool)logSystem->get())
.detail("RecoveryState", (int)self->dbInfo->get().recoveryState);
Small code refactoring
2019-04-30 01:48:09 +08:00
for (const auto& it : self->dbInfo->get().logSystemConfig.oldTLogs) {
Attempt to normalize trace events: * Detail names now all start with an uppercase character and contain no underscores. Ideally these should be head-first camel case, though that was harder to check. * Type names have the same rules, except they allow one underscore (to support a usage pattern Context_Type). The first character after the underscore is also uppercase. * Use seconds instead of milliseconds in details. Added a check when events are logged in simulation that logs a message to stderr if the first two rules above aren't followed. This probably doesn't address every instance of the above problems, but all of the events I was able to hit in simulation pass the check.
2018-06-09 02:11:08 +08:00
TraceEvent("TLogUpdateOld", self->dbgid).detail("LogId", logData->logId).detail("DbInfo", it.toString());
fix: the tlog can peek from log systems that have been recovered even if it does not match its recoverFrom set
2018-02-21 06:50:13 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(self->dbInfo->onChange());
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
void stopAllTLogs(TLogData* self, UID newLogId) {
for (auto it : self->id_data) {
if (!it.second->stopped) {
TraceEvent("TLogStoppedByNewRecruitment", self->dbgid)
.detail("LogId", it.second->logId)
.detail("StoppedId", it.first.toString())
.detail("RecruitedId", newLogId)
.detail("EndEpoch", it.second->logSystem->get().getPtr() != 0);
if (!it.second->isPrimary && it.second->logSystem->get()) {
added tag localities fix: remote logs need to stop the master when they are stopped
2017-08-04 07:16:36 +08:00
it.second->removed = it.second->removed && it.second->logSystem->get()->endEpoch();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (it.second->committingQueue.canBeSet()) {
fix: when a tlog is stopped by another recruitment it no longer has the opportunity for commtingQueue to be set
2018-05-07 11:37:44 +08:00
it.second->committingQueue.sendError(worker_removed());
}
added tag localities fix: remote logs need to stop the master when they are stopped
2017-08-04 07:16:36 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
it.second->stopped = true;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (!it.second->recoveryComplete.isSet()) {
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
it.second->recoveryComplete.sendError(end_of_stream());
fix: the tlog did not cancel recovery properly when stopped
2018-01-13 09:18:14 +08:00
}
fix: when a new log is recruited on a shared log, all outstanding commits need to be notified that they are stopped, because there is no longer a guarantee that their queueCommittedVersion will advance
2018-03-20 08:48:28 +08:00
it.second->stopCommit.trigger();
Initial repository commit
2017-05-26 04:48:44 +08:00
}
Fix a bug that would cause active logs to spill aggressively And add some useful logging about when things do or do not spill.
2019-09-11 06:51:30 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
Fix a bug that would cause active logs to spill aggressively And add some useful logging about when things do or do not spill.
2019-09-11 06:51:30 +08:00
// Start the tLog role for a worker
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> tLogStart(TLogData* self, InitializeTLogRequest req, LocalityData locality) {
Fix a bug that would cause active logs to spill aggressively And add some useful logging about when things do or do not spill.
2019-09-11 06:51:30 +08:00
state TLogInterface recruited(self->dbgid, locality);
recruited.initEndpoints();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
DUMPTOKEN(recruited.peekMessages);
DUMPTOKEN(recruited.popMessages);
DUMPTOKEN(recruited.commit);
DUMPTOKEN(recruited.lock);
DUMPTOKEN(recruited.getQueuingMetrics);
DUMPTOKEN(recruited.confirmRunning);
DUMPTOKEN(recruited.waitFailure);
DUMPTOKEN(recruited.recoveryFinished);
DUMPTOKEN(recruited.disablePopRequest);
DUMPTOKEN(recruited.enablePopRequest);
DUMPTOKEN(recruited.snapRequest);
Fix a bug that would cause active logs to spill aggressively And add some useful logging about when things do or do not spill.
2019-09-11 06:51:30 +08:00
stopAllTLogs(self, recruited.id());
Initial repository commit
2017-05-26 04:48:44 +08:00
Improve TLog "Role" event by adding the worker ID, the TLog version, and under what circumstances the TLog is being started (Restored, Recruited, or Recovered). The SharedTLog role was being started and stopped twice, so remove one instance of it.
2020-02-13 07:11:38 +08:00
bool recovering = (req.recoverFrom.logSystemType == LogSystemType::tagPartitioned);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
state Reference<LogData> logData = makeReference<LogData>(self,
recruited,
req.remoteTag,
req.isPrimary,
req.logRouterTags,
req.txsTags,
req.recruitmentID,
g_network->protocolVersion(),
req.spillType,
req.allTags,
recovering ? "Recovered" : "Recruited");
Initial repository commit
2017-05-26 04:48:44 +08:00
self->id_data[recruited.id()] = logData;
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
logData->locality = req.locality;
Initial repository commit
2017-05-26 04:48:44 +08:00
logData->recoveryCount = req.epoch;
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
logData->removed = rejoinMasters(self, recruited, req.epoch, Future<Void>(Void()), req.isPrimary);
Clear versionLocation when spilling, and pop DQ separately. Popping the disk queue now requires potentially recovering the location to which we can pop from the spilled data itself, and for each tag we must maintain the first location with relevant data. The previous queue we had to represent the ordering, queueOrder, was used by spilling, and popped when a TLog had been spilled. This means that as soon as a TLog has been fully spilled, we have no idea how it relates in order to other fully spilled TLogs. Instead, use queueOrder to keep track of all the TLog UIDs until they're removed, and use spillOrder to keep track of the order only for spilling.
2019-03-16 12:01:20 +08:00
self->popOrder.push_back(recruited.id());
self->spillOrder.push_back(recruited.id());
Initial repository commit
2017-05-26 04:48:44 +08:00
add epoch info to trace events when tLog begins
2021-06-10 08:14:36 +08:00
TraceEvent("TLogStart", logData->logId).detail("RecoveryCount", logData->recoveryCount);
code re-orgnaization and address comments
2019-04-21 03:58:24 +08:00
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
state Future<Void> updater;
fix: if a storage server already exists in a remote region after converting to fearless, it did not receive mutations between the known committed version and the recovery version
2018-11-03 05:11:39 +08:00
state bool pulledRecoveryVersions = false;
Initial repository commit
2017-05-26 04:48:44 +08:00
try {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->removed.isReady()) {
Initial repository commit
2017-05-26 04:48:44 +08:00
throw logData->removed.getError();
}
Improve TLog "Role" event by adding the worker ID, the TLog version, and under what circumstances the TLog is being started (Restored, Recruited, or Recovered). The SharedTLog role was being started and stopped twice, so remove one instance of it.
2020-02-13 07:11:38 +08:00
if (recovering) {
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
logData->unrecoveredBefore = req.startVersion;
fix: if we cannot find a tag, it must have been popped at the recovery version.
2018-04-23 06:08:38 +08:00
logData->recoveredAt = req.recoverAt;
fix: initialize the known committed version on the tlog
2018-04-21 15:41:15 +08:00
logData->knownCommittedVersion = req.startVersion - 1;
fix: need to initialize tlog versions to less than the startVersion
2018-04-10 08:17:11 +08:00
logData->persistentDataVersion = logData->unrecoveredBefore - 1;
logData->persistentDataDurableVersion = logData->unrecoveredBefore - 1;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->queueCommittedVersion.set(logData->unrecoveredBefore - 1);
logData->version.set(logData->unrecoveredBefore - 1);
Initial repository commit
2017-05-26 04:48:44 +08:00
fix: pop all tags that did not have data at the recovery version because fully popped tags may come back when pullAsyncData re-indexes the mutations
2018-04-01 07:47:56 +08:00
logData->unpoppedRecoveredTags = req.allTags.size();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(initPersistentState(self, logData) || logData->removed);
TraceEvent("TLogRecover", self->dbgid)
.detail("LogId", logData->logId)
.detail("At", req.recoverAt)
.detail("Known", req.knownCommittedVersion)
.detail("Unrecovered", logData->unrecoveredBefore)
.detail("Tags", describe(req.recoverTags))
.detail("Locality", req.locality)
.detail("LogRouterTags", logData->logRouterTags);
if (logData->recoveryComplete.isSet()) {
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
throw worker_removed();
}
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
updater = updateLogSystem(self, logData, req.recoverFrom, logData->logSystem);
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
fix: tlogs are now initialized immediately, instead of when starting the core, this must be done to pop the log routers during recovery fix: log router start version must be the same as remote log start version
2018-04-16 05:33:07 +08:00
logData->initialized = true;
self->newLogData.trigger();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if ((req.isPrimary || req.recoverFrom.logRouterTags == 0) && !logData->stopped &&
logData->unrecoveredBefore <= req.recoverAt) {
if (req.recoverFrom.logRouterTags > 0 && req.locality != tagLocalitySatellite) {
avoid peeking from logs that do not match the tag’s locality
2018-06-02 09:42:48 +08:00
logData->logRouterPopToVersion = req.recoverAt;
std::vector<Tag> tags;
tags.push_back(logData->remoteTag);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(pullAsyncData(self, logData, tags, logData->unrecoveredBefore, req.recoverAt, true) ||
logData->removed);
} else if (!req.recoverTags.empty()) {
avoid peeking from logs that do not match the tag’s locality
2018-06-02 09:42:48 +08:00
ASSERT(logData->unrecoveredBefore > req.knownCommittedVersion);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(pullAsyncData(
self, logData, req.recoverTags, req.knownCommittedVersion + 1, req.recoverAt, false) ||
logData->removed);
avoid peeking from logs that do not match the tag’s locality
2018-06-02 09:42:48 +08:00
}
fix: if a storage server already exists in a remote region after converting to fearless, it did not receive mutations between the known committed version and the recovery version
2018-11-03 05:11:39 +08:00
pulledRecoveryVersions = true;
logData->knownCommittedVersion = req.recoverAt;
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if ((req.isPrimary || req.recoverFrom.logRouterTags == 0) && logData->version.get() < req.recoverAt &&
!logData->stopped) {
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
// Log the changes to the persistent queue, to be committed by commitQueue()
TLogQueueEntryRef qe;
qe.version = req.recoverAt;
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
qe.knownCommittedVersion = logData->knownCommittedVersion;
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
qe.messages = StringRef();
qe.id = logData->logId;
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self->persistentQueue->push(qe, logData);
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
self->diskQueueCommitBytes += qe.expectedSize();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (self->diskQueueCommitBytes > SERVER_KNOBS->MAX_QUEUE_COMMIT_BYTES) {
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
self->largeDiskQueueCommitBytes.set(true);
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->version.set(req.recoverAt);
first version of non-copying recovery. Upgrades are broken, and it has not been tested using fearless configurations yet
2018-03-30 06:12:38 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->recoveryComplete.isSet()) {
fix: fixed rare bug where a log stopped by a different recruitment would still response successfully to the recruitment message
2018-04-29 04:34:06 +08:00
throw worker_removed();
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
logData->addActor.send(respondToRecovered(recruited, logData->recoveryComplete));
Initial repository commit
2017-05-26 04:48:44 +08:00
} else {
// Brand new tlog, initialization has already been done by caller
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(initPersistentState(self, logData) || logData->removed);
fix: do not use log router tags when configured without remote logs fix: data distribution tracks undesired storage servers re-enabled consistency check
2018-02-14 09:01:34 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (logData->recoveryComplete.isSet()) {
fix: do not use log router tags when configured without remote logs fix: data distribution tracks undesired storage servers re-enabled consistency check
2018-02-14 09:01:34 +08:00
throw worker_removed();
}
fix: tlogs are now initialized immediately, instead of when starting the core, this must be done to pop the log routers during recovery fix: log router start version must be the same as remote log start version
2018-04-16 05:33:07 +08:00
logData->initialized = true;
self->newLogData.trigger();
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
logData->recoveryComplete.send(Void());
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(logData->committingQueue.getFuture() || logData->removed);
} catch (Error& e) {
fix: tlogs must send a recruitment reply even when actor cancelled or the recruitment endpoint will be marked as permanently failed
2020-01-17 09:37:17 +08:00
req.reply.sendError(recruitment_failed());
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (e.code() != error_code_worker_removed) {
Initial repository commit
2017-05-26 04:48:44 +08:00
throw;
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(delay(0.0)); // if multiple recruitment requests were already in the promise stream make sure they are all
// started before any are removed
fix: do not delay before setting logData->version
2017-06-03 02:27:37 +08:00
fix: bytesDurable was not correctly increased when a log was removed re-added many TLogMetrics added a new role for the shared tlog
2017-06-23 08:21:42 +08:00
removeLog(self, logData);
return Void();
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
req.reply.send(recruited);
Initial repository commit
2017-05-26 04:48:44 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TraceEvent("TLogReady", logData->logId)
.detail("AllTags", describe(req.allTags))
.detail("Locality", logData->locality);
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
first working version of non-copying recovery working with fearless configurations
2018-04-09 12:24:05 +08:00
updater = Void();
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(tLogCore(self, logData, recruited, pulledRecoveryVersions));
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
}
Spill SharedTLog when there's more than one. When switching between spill_type or log_version, a new instance of a SharedTLog is created in the transaction log processes. If this is done in a saturated database, then doubling the amount of memory to hold mutations in memory can cause TLogs to be uncomfortably close to the 8GB OOM limit. Instead, we now thread which UID of a SharedTLog is active, and the other TLog spill out the majority of their mutations.
2019-07-30 14:40:28 +08:00
ACTOR Future<Void> startSpillingInTenSeconds(TLogData* self, UID tlogId, Reference<AsyncVar<UID>> activeSharedTLog) {
wait(delay(10));
if (activeSharedTLog->get() != tlogId) {
// TODO: This should fully spill, but currently doing so will cause us to no longer update poppedVersion
// and QuietDatabase will hang thinking our TLog is behind.
Fix a bug that would cause active logs to spill aggressively And add some useful logging about when things do or do not spill.
2019-09-11 06:51:30 +08:00
TraceEvent("SharedTLogBeginSpilling", self->dbgid).detail("NowActive", activeSharedTLog->get());
Spill SharedTLog when there's more than one. When switching between spill_type or log_version, a new instance of a SharedTLog is created in the transaction log processes. If this is done in a saturated database, then doubling the amount of memory to hold mutations in memory can cause TLogs to be uncomfortably close to the 8GB OOM limit. Instead, we now thread which UID of a SharedTLog is active, and the other TLog spill out the majority of their mutations.
2019-07-30 14:40:28 +08:00
self->targetVolatileBytes = SERVER_KNOBS->REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT * 2;
Fix a bug that would cause active logs to spill aggressively And add some useful logging about when things do or do not spill.
2019-09-11 06:51:30 +08:00
} else {
TraceEvent("SharedTLogSkipSpilling", self->dbgid).detail("NowActive", activeSharedTLog->get());
Spill SharedTLog when there's more than one. When switching between spill_type or log_version, a new instance of a SharedTLog is created in the transaction log processes. If this is done in a saturated database, then doubling the amount of memory to hold mutations in memory can cause TLogs to be uncomfortably close to the 8GB OOM limit. Instead, we now thread which UID of a SharedTLog is active, and the other TLog spill out the majority of their mutations.
2019-07-30 14:40:28 +08:00
}
return Void();
}
Initial repository commit
2017-05-26 04:48:44 +08:00
// New tLog (if !recoverFrom.size()) or restore from network
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
IDiskQueue* persistentQueue,
Prevent tLog from modifying ServerDBInfo object
2021-07-12 13:25:40 +08:00
Reference<AsyncVar<ServerDBInfo> const> db,
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
LocalityData locality,
PromiseStream<InitializeTLogRequest> tlogRequests,
UID tlogId,
UID workerID,
bool restoreFromDisk,
Promise<Void> oldLog,
Promise<Void> recovered,
std::string folder,
Reference<AsyncVar<bool>> degraded,
Reference<AsyncVar<UID>> activeSharedTLog) {
state TLogData self(tlogId, workerID, persistentData, persistentQueue, db, degraded, folder);
state Future<Void> error = actorCollection(self.sharedActors.getFuture());
Initial repository commit
2017-05-26 04:48:44 +08:00
TraceEvent("SharedTlog", tlogId);
try {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (restoreFromDisk) {
wait(restorePersistentState(&self, locality, oldLog, recovered, tlogRequests));
Initial repository commit
2017-05-26 04:48:44 +08:00
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
wait(ioTimeoutError(checkEmptyQueue(&self) && checkRecovered(&self),
SERVER_KNOBS->TLOG_MAX_CREATE_DURATION));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// Disk errors need a chance to kill this actor.
Rewrite all `Void _ = wait(...)` -> `wait(...)`. This takes advantage of the new actorcompiler functionality to avoid having duplicate definitions of `Void _` when trying to feed the un-actorompiled source through clang.
2018-08-11 04:57:10 +08:00
wait(delay(0.000001));
fix: allow disk errors to cancel the actor before recruiting logs
2018-06-11 11:27:19 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (recovered.canBeSet())
recovered.send(Void());
do not register a worker with the cluster controller until it has finished recovering all files from disk
2017-09-16 01:57:58 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self.sharedActors.send(commitQueue(&self));
self.sharedActors.send(updateStorageLoop(&self));
self.sharedActors.send(traceRole(Role::SHARED_TRANSACTION_LOG, tlogId));
avoid starting a new startSpillingActor with every TLog recruitment
2019-10-24 02:15:54 +08:00
state Future<Void> activeSharedChange = Void();
Initial repository commit
2017-05-26 04:48:44 +08:00
loop {
choose {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(InitializeTLogRequest req = waitNext(tlogRequests.getFuture())) {
if (!self.tlogCache.exists(req.recruitmentID)) {
self.tlogCache.set(req.recruitmentID, req.reply.getFuture());
self.sharedActors.send(
self.tlogCache.removeOnReady(req.recruitmentID, tLogStart(&self, req, locality)));
Initial repository commit
2017-05-26 04:48:44 +08:00
} else {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
forwardPromise(req.reply, self.tlogCache.get(req.recruitmentID));
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
when(wait(error)) { throw internal_error(); }
when(wait(activeSharedChange)) {
Spill SharedTLog when there's more than one. When switching between spill_type or log_version, a new instance of a SharedTLog is created in the transaction log processes. If this is done in a saturated database, then doubling the amount of memory to hold mutations in memory can cause TLogs to be uncomfortably close to the 8GB OOM limit. Instead, we now thread which UID of a SharedTLog is active, and the other TLog spill out the majority of their mutations.
2019-07-30 14:40:28 +08:00
if (activeSharedTLog->get() == tlogId) {
Fix a bug that would cause active logs to spill aggressively And add some useful logging about when things do or do not spill.
2019-09-11 06:51:30 +08:00
TraceEvent("SharedTLogNowActive", self.dbgid).detail("NowActive", activeSharedTLog->get());
Spill SharedTLog when there's more than one. When switching between spill_type or log_version, a new instance of a SharedTLog is created in the transaction log processes. If this is done in a saturated database, then doubling the amount of memory to hold mutations in memory can cause TLogs to be uncomfortably close to the 8GB OOM limit. Instead, we now thread which UID of a SharedTLog is active, and the other TLog spill out the majority of their mutations.
2019-07-30 14:40:28 +08:00
self.targetVolatileBytes = SERVER_KNOBS->TLOG_SPILL_THRESHOLD;
} else {
Fix a bug that would cause active logs to spill aggressively And add some useful logging about when things do or do not spill.
2019-09-11 06:51:30 +08:00
stopAllTLogs(&self, tlogId);
TraceEvent("SharedTLogQueueSpilling", self.dbgid).detail("NowActive", activeSharedTLog->get());
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
self.sharedActors.send(startSpillingInTenSeconds(&self, tlogId, activeSharedTLog));
Spill SharedTLog when there's more than one. When switching between spill_type or log_version, a new instance of a SharedTLog is created in the transaction log processes. If this is done in a saturated database, then doubling the amount of memory to hold mutations in memory can cause TLogs to be uncomfortably close to the 8GB OOM limit. Instead, we now thread which UID of a SharedTLog is active, and the other TLog spill out the majority of their mutations.
2019-07-30 14:40:28 +08:00
}
avoid starting a new startSpillingActor with every TLog recruitment
2019-10-24 02:15:54 +08:00
activeSharedChange = activeSharedTLog->onChange();
Spill SharedTLog when there's more than one. When switching between spill_type or log_version, a new instance of a SharedTLog is created in the transaction log processes. If this is done in a saturated database, then doubling the amount of memory to hold mutations in memory can cause TLogs to be uncomfortably close to the 8GB OOM limit. Instead, we now thread which UID of a SharedTLog is active, and the other TLog spill out the majority of their mutations.
2019-07-30 14:40:28 +08:00
}
Initial repository commit
2017-05-26 04:48:44 +08:00
}
}
} catch (Error& e) {
fix: during destruction logData could attempt to dereference tLogData after it has been deleted
2018-08-10 03:38:35 +08:00
self.terminated.send(Void());
fix: fixed a rare bug where we do not wait for a file in the process of being deleted to shutdown before rebooting a machine
2017-08-26 01:12:58 +08:00
TraceEvent("TLogError", tlogId).error(e, true);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (recovered.canBeSet())
recovered.send(Void());
do not register a worker with the cluster controller until it has finished recovering all files from disk
2017-09-16 01:57:58 +08:00
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
while (!tlogRequests.isEmpty()) {
Ensured that the circus id is always lowercase
2017-08-24 04:45:00 +08:00
tlogRequests.getFuture().pop().reply.sendError(recruitment_failed());
Initial repository commit
2017-05-26 04:48:44 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (auto& it : self.id_data) {
if (!it.second->recoveryComplete.isSet()) {
re-added support for non-copying tlog recovery
2017-10-25 06:09:31 +08:00
it.second->recoveryComplete.sendError(end_of_stream());
fix: the tlog did not cancel recovery properly when stopped
2018-01-13 09:18:14 +08:00
}
do not continue copying data from prior generations after being locked
2017-07-20 06:11:18 +08:00
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
if (tlogTerminated(&self, persistentData, self.persistentQueue, e)) {
Initial repository commit
2017-05-26 04:48:44 +08:00
return Void();
} else {
throw;
}
}
}
// UNIT TESTS
struct DequeAllocatorStats {
static int64_t allocatedBytes;
};
int64_t DequeAllocatorStats::allocatedBytes = 0;
template <class T>
struct DequeAllocator : std::allocator<T> {
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
template <typename U>
Initial repository commit
2017-05-26 04:48:44 +08:00
struct rebind {
typedef DequeAllocator<U> other;
};
DequeAllocator() {}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
template <typename U>
Initial repository commit
2017-05-26 04:48:44 +08:00
DequeAllocator(DequeAllocator<U> const& u) : std::allocator<T>(u) {}
T* allocate(std::size_t n, std::allocator<void>::const_pointer hint = 0) {
DequeAllocatorStats::allocatedBytes += n * sizeof(T);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// fprintf(stderr, "Allocating %lld objects for %lld bytes (total allocated: %lld)\n", n, n * sizeof(T),
// DequeAllocatorStats::allocatedBytes);
Initial repository commit
2017-05-26 04:48:44 +08:00
return std::allocator<T>::allocate(n, hint);
}
void deallocate(T* p, std::size_t n) {
DequeAllocatorStats::allocatedBytes -= n * sizeof(T);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
// fprintf(stderr, "Deallocating %lld objects for %lld bytes (total allocated: %lld)\n", n, n * sizeof(T),
// DequeAllocatorStats::allocatedBytes);
Initial repository commit
2017-05-26 04:48:44 +08:00
return std::allocator<T>::deallocate(p, n);
}
};
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
TEST_CASE("/fdbserver/tlogserver/VersionMessagesOverheadFactor") {
Initial repository commit
2017-05-26 04:48:44 +08:00
typedef std::pair<Version, LengthPrefixedStringRef> TestType; // type used by versionMessages
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
for (int i = 1; i < 9; ++i) {
for (int j = 0; j < 20; ++j) {
Initial repository commit
2017-05-26 04:48:44 +08:00
DequeAllocatorStats::allocatedBytes = 0;
DequeAllocator<TestType> allocator;
std::deque<TestType, DequeAllocator<TestType>> d(allocator);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
int numElements = deterministicRandom()->randomInt(pow(10, i - 1), pow(10, i));
for (int k = 0; k < numElements; ++k) {
Initial repository commit
2017-05-26 04:48:44 +08:00
d.push_back(TestType());
}
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
int removedElements = 0; // deterministicRandom()->randomInt(0, numElements); // FIXME: the overhead factor
// does not accurately account for removal!
for (int k = 0; k < removedElements; ++k) {
Initial repository commit
2017-05-26 04:48:44 +08:00
d.pop_front();
}
int64_t dequeBytes = DequeAllocatorStats::allocatedBytes + sizeof(std::deque<TestType>);
apply clang-format to *.c, *.cpp, *.h, *.hpp files
2021-03-11 02:06:03 +08:00
int64_t insertedBytes = (numElements - removedElements) * sizeof(TestType);
double overheadFactor =
std::max<double>(insertedBytes, dequeBytes - 10000) /
insertedBytes; // We subtract 10K here as an estimated upper bound for the fixed cost of an std::deque
// fprintf(stderr, "%d elements (%d inserted, %d removed):\n", numElements-removedElements, numElements,
// removedElements); fprintf(stderr, "Allocated %lld bytes to store %lld bytes (%lf overhead factor)\n",
// dequeBytes, insertedBytes, overheadFactor);
Initial repository commit
2017-05-26 04:48:44 +08:00
ASSERT(overheadFactor * 1024 <= SERVER_KNOBS->VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS);
}
}
return Void();
}