wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add debug traces

This commit is contained in:
Xiaoxi Wang 2022-05-18 15:20:23 -07:00
parent 33ae398268
commit 6c11fc74ba
4 changed files with 50 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
fdbserver/LogRouter.actor.cpp
View File

@ -35,7 +35,7 @@
#include "flow/Histogram.h" #include "flow/Histogram.h"
#include "flow/TDMetric.actor.h" #include "flow/TDMetric.actor.h"
#include "flow/network.h" #include "flow/network.h"
#include "flow/DebugTrace.h"
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
struct LogRouterData { struct LogRouterData {
@ -499,9 +499,13 @@ Future<Void> logRouterPeekMessages(PromiseType replyPromise,
} }
} }
//TraceEvent("LogRouterPeek1", self->dbgid).detail("From", replyPromise.getEndpoint().getPrimaryAddress()).detail("Ver", self->version.get()).detail("Begin", reqBegin); DebugLogTraceEvent("LogRouterPeek0", self->dbgid)
.detail("ReturnIfBlocked", reqReturnIfBlocked)
.detail("Tag", reqTag.toString())
.detail("Ver", self->version.get())
.detail("Begin", reqBegin);
if (reqReturnIfBlocked && self->version.get() < reqBegin) { if (reqReturnIfBlocked && self->version.get() < reqBegin) {
//TraceEvent("LogRouterPeek2", self->dbgid);
replyPromise.sendError(end_of_stream()); replyPromise.sendError(end_of_stream());
if (reqSequence.present()) { if (reqSequence.present()) {
auto& trackerData = self->peekTracker[peekId]; auto& trackerData = self->peekTracker[peekId];
@ -603,7 +607,7 @@ Future<Void> logRouterPeekMessages(PromiseType replyPromise,
} }
replyPromise.send(reply); replyPromise.send(reply);
DisabledTraceEvent("LogRouterPeek4", self->dbgid) DebugLogTraceEvent("LogRouterPeek4", self->dbgid)
.detail("Tag", reqTag.toString()) .detail("Tag", reqTag.toString())
.detail("ReqBegin", reqBegin) .detail("ReqBegin", reqBegin)
.detail("End", reply.end) .detail("End", reply.end)
@ -638,8 +642,9 @@ ACTOR Future<Void> logRouterPeekStream(LogRouterData* self, TLogPeekStreamReques
} }
} catch (Error& e) { } catch (Error& e) {
self->activePeekStreams--; self->activePeekStreams--;
TraceEvent(SevDebug, "TLogPeekStreamEnd", self->dbgid) TraceEvent(SevDebug, "LogRouterPeekStreamEnd", self->dbgid)
.errorUnsuppressed(e) .errorUnsuppressed(e)
.detail("Tag", req.tag)
.detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress()); .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
if (e.code() == error_code_end_of_stream || e.code() == error_code_operation_obsolete) { if (e.code() == error_code_end_of_stream || e.code() == error_code_operation_obsolete) {

19
fdbserver/LogSystemPeekCursor.actor.cpp
View File

@ -23,6 +23,7 @@
#include "fdbserver/Knobs.h" #include "fdbserver/Knobs.h"
#include "fdbserver/MutationTracking.h" #include "fdbserver/MutationTracking.h"
#include "fdbrpc/ReplicationUtils.h" #include "fdbrpc/ReplicationUtils.h"
#include "flow/DebugTrace.h"
#include "flow/actorcompiler.h" // has to be last include #include "flow/actorcompiler.h" // has to be last include
// create a peek stream for cursor when it's possible // create a peek stream for cursor when it's possible
@ -35,9 +36,12 @@ ACTOR Future<Void> tryEstablishPeekStream(ILogSystem::ServerPeekCursor* self) {
} }
wait(IFailureMonitor::failureMonitor().onStateEqual(self->interf->get().interf().peekStreamMessages.getEndpoint(), wait(IFailureMonitor::failureMonitor().onStateEqual(self->interf->get().interf().peekStreamMessages.getEndpoint(),
FailureStatus(false))); FailureStatus(false)));
self->peekReplyStream = self->interf->get().interf().peekStreamMessages.getReplyStream(TLogPeekStreamRequest(
self->messageVersion.version, self->tag, self->returnIfBlocked, std::numeric_limits<int>::max())); auto req = TLogPeekStreamRequest(
TraceEvent(SevDebug, "SPC_StreamCreated", self->randomID) self->messageVersion.version, self->tag, self->returnIfBlocked, std::numeric_limits<int>::max());
self->peekReplyStream = self->interf->get().interf().peekStreamMessages.getReplyStream(req);
DebugLogTraceEvent(SevDebug, "SPC_StreamCreated", self->randomID)
.detail("Tag", self->tag)
.detail("PeerAddr", self->interf->get().interf().peekStreamMessages.getEndpoint().getPrimaryAddress()) .detail("PeerAddr", self->interf->get().interf().peekStreamMessages.getEndpoint().getPrimaryAddress())
.detail("PeerToken", self->interf->get().interf().peekStreamMessages.getEndpoint().token); .detail("PeerToken", self->interf->get().interf().peekStreamMessages.getEndpoint().token);
return Void(); return Void();
@ -56,7 +60,7 @@ ILogSystem::ServerPeekCursor::ServerPeekCursor(Reference<AsyncVar<OptionalInterf
fastReplies(0), unknownReplies(0) { fastReplies(0), unknownReplies(0) {
this->results.maxKnownVersion = 0; this->results.maxKnownVersion = 0;
this->results.minKnownCommittedVersion = 0; this->results.minKnownCommittedVersion = 0;
DisabledTraceEvent(SevDebug, "SPC_Starting", randomID) DebugLogTraceEvent(SevDebug, "SPC_Starting", randomID)
.detail("Tag", tag.toString()) .detail("Tag", tag.toString())
.detail("UsePeekStream", usePeekStream) .detail("UsePeekStream", usePeekStream)
.detail("Begin", begin) .detail("Begin", begin)
@ -356,7 +360,8 @@ ACTOR Future<Void> serverPeekStreamGetMore(ILogSystem::ServerPeekCursor* self, T
} }
updateCursorWithReply(self, res); updateCursorWithReply(self, res);
expectedBegin = res.end; expectedBegin = res.end;
DisabledTraceEvent(SevDebug, "SPC_GetMoreB", self->randomID) DebugLogTraceEvent(SevDebug, "SPC_GetMoreB", self->randomID)
.detail("Tag", self->tag)
.detail("Has", self->hasMessage()) .detail("Has", self->hasMessage())
.detail("End", res.end) .detail("End", res.end)
.detail("Popped", res.popped.present() ? res.popped.get() : 0); .detail("Popped", res.popped.present() ? res.popped.get() : 0);
@ -368,7 +373,9 @@ ACTOR Future<Void> serverPeekStreamGetMore(ILogSystem::ServerPeekCursor* self, T
} }
} }
} catch (Error& e) { } catch (Error& e) {
DisabledTraceEvent(SevDebug, "SPC_GetMoreB_Error", self->randomID).errorUnsuppressed(e); DebugLogTraceEvent(SevDebug, "SPC_GetMoreB_Error", self->randomID)
.errorUnsuppressed(e)
.detail("Tag", self->tag);
if (e.code() == error_code_connection_failed || e.code() == error_code_operation_obsolete) { if (e.code() == error_code_connection_failed || e.code() == error_code_operation_obsolete) {
// NOTE: delay in order to avoid the endless retry loop block other tasks // NOTE: delay in order to avoid the endless retry loop block other tasks
self->peekReplyStream.reset(); self->peekReplyStream.reset();

19
fdbserver/TLogServer.actor.cpp
View File

@ -46,6 +46,7 @@
#include "fdbserver/RecoveryState.h" #include "fdbserver/RecoveryState.h"
#include "fdbserver/FDBExecHelper.actor.h" #include "fdbserver/FDBExecHelper.actor.h"
#include "flow/Histogram.h" #include "flow/Histogram.h"
#include "flow/DebugTrace.h"
#include "flow/actorcompiler.h" // This must be the last #include. #include "flow/actorcompiler.h" // This must be the last #include.
struct TLogQueueEntryRef { struct TLogQueueEntryRef {
@ -1675,6 +1676,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
state int sequence = -1; state int sequence = -1;
state UID peekId; state UID peekId;
state double queueStart = now(); state double queueStart = now();
state Future<Void> recoveryTxnReceived = logData->recoveryTxnReceived.getFuture();
if (reqTag.locality == tagLocalityTxs && reqTag.id >= logData->txsTags && logData->txsTags > 0) { if (reqTag.locality == tagLocalityTxs && reqTag.id >= logData->txsTags && logData->txsTags > 0) {
reqTag.id = reqTag.id % logData->txsTags; reqTag.id = reqTag.id % logData->txsTags;
@ -1746,7 +1748,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
return Void(); return Void();
} }
DisabledTraceEvent("TLogPeekMessages0", self->dbgid) DebugLogTraceEvent("TLogPeekMessages0", self->dbgid)
.detail("LogId", logData->logId) .detail("LogId", logData->logId)
.detail("Tag", reqTag.toString()) .detail("Tag", reqTag.toString())
.detail("ReqBegin", reqBegin) .detail("ReqBegin", reqBegin)
@ -1758,11 +1760,17 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
wait(delay(SERVER_KNOBS->TLOG_PEEK_DELAY, g_network->getCurrentTask())); wait(delay(SERVER_KNOBS->TLOG_PEEK_DELAY, g_network->getCurrentTask()));
} }
if (!logData->stopped && reqTag.locality != tagLocalityTxs && reqTag != txsTag) { if (!logData->stopped && reqTag.locality != tagLocalityTxs && reqTag != txsTag) {
DebugLogTraceEvent("TLogPeekMessages1", self->dbgid)
.detail("LogId", logData->logId)
.detail("Tag", reqTag.toString())
.detail("ReqBegin", reqBegin)
.detail("Version", logData->version.get())
.detail("RecoveredAt", logData->recoveredAt);
// Make sure the peek reply has the recovery txn for the current TLog. // Make sure the peek reply has the recovery txn for the current TLog.
// Older generation TLog has been stopped and doesn't wait here. // Older generation TLog has been stopped and doesn't wait here.
// Similarly during recovery, reading transaction state store // Similarly during recovery, reading transaction state store
// doesn't wait here. // doesn't wait here.
wait(logData->recoveryTxnReceived.getFuture()); wait(recoveryTxnReceived);
} }
if (logData->locality != tagLocalitySatellite && reqTag.locality == tagLocalityLogRouter) { if (logData->locality != tagLocalitySatellite && reqTag.locality == tagLocalityLogRouter) {
@ -1803,7 +1811,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
poppedVer = poppedVersion(logData, reqTag); poppedVer = poppedVersion(logData, reqTag);
} }
DisabledTraceEvent("TLogPeekMessages1", self->dbgid) DebugLogTraceEvent("TLogPeekMessages2", self->dbgid)
.detail("LogId", logData->logId) .detail("LogId", logData->logId)
.detail("Tag", reqTag.toString()) .detail("Tag", reqTag.toString())
.detail("ReqBegin", reqBegin) .detail("ReqBegin", reqBegin)
@ -1852,7 +1860,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
onlySpilled = false; onlySpilled = false;
// grab messages from disk // grab messages from disk
DisabledTraceEvent("TLogPeekMessages2", self->dbgid) DebugLogTraceEvent("TLogPeekMessages3", self->dbgid)
.detail("ReqBegin", reqBegin) .detail("ReqBegin", reqBegin)
.detail("Tag", reqTag.toString()); .detail("Tag", reqTag.toString());
if (reqBegin <= logData->persistentDataDurableVersion) { if (reqBegin <= logData->persistentDataDurableVersion) {
@ -2015,7 +2023,7 @@ Future<Void> tLogPeekMessages(PromiseType replyPromise,
reply.end = endVersion; reply.end = endVersion;
reply.onlySpilled = onlySpilled; reply.onlySpilled = onlySpilled;
DisabledTraceEvent("TLogPeekMessages4", self->dbgid) DebugLogTraceEvent("TLogPeekMessages4", self->dbgid)
.detail("LogId", logData->logId) .detail("LogId", logData->logId)
.detail("Tag", reqTag.toString()) .detail("Tag", reqTag.toString())
.detail("ReqBegin", reqBegin) .detail("ReqBegin", reqBegin)
@ -2101,6 +2109,7 @@ ACTOR Future<Void> tLogPeekStream(TLogData* self, TLogPeekStreamRequest req, Ref
self->activePeekStreams--; self->activePeekStreams--;
TraceEvent(SevDebug, "TLogPeekStreamEnd", logData->logId) TraceEvent(SevDebug, "TLogPeekStreamEnd", logData->logId)
.errorUnsuppressed(e) .errorUnsuppressed(e)
.detail("Tag", req.tag)
.detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress()); .detail("PeerAddr", req.reply.getEndpoint().getPrimaryAddress());
if (e.code() == error_code_end_of_stream || e.code() == error_code_operation_obsolete) { if (e.code() == error_code_end_of_stream || e.code() == error_code_operation_obsolete) {

13
flow/DebugTrace.h Normal file
View File

@ -0,0 +1,13 @@
//
// Created by Xiaoxi Wang on 5/18/22.
//
#ifndef FOUNDATIONDB_DEBUGTRACE_H
#define FOUNDATIONDB_DEBUGTRACE_H
#define DebugTraceEvent(enable, ...) enable&& TraceEvent(__VA_ARGS__)
constexpr bool debugLogTraces = true;
#define DebugLogTraceEvent(...) DebugTraceEvent(debugLogTraces, __VA_ARGS__)
#endif // FOUNDATIONDB_DEBUGTRACE_H