diff --git a/fdbrpc/include/fdbrpc/AsyncFileKAIO.actor.h b/fdbrpc/include/fdbrpc/AsyncFileKAIO.actor.h index 09320ae659..8a86415b00 100644 --- a/fdbrpc/include/fdbrpc/AsyncFileKAIO.actor.h +++ b/fdbrpc/include/fdbrpc/AsyncFileKAIO.actor.h @@ -38,7 +38,7 @@ #include #include "fdbrpc/linux_kaio.h" #include "flow/Knobs.h" -#include "flow/Histogram.h" +#include "fdbrpc/Stats.h" #include "flow/UnitTest.h" #include "crc32/crc32c.h" #include "flow/genericactors.actor.h" @@ -48,14 +48,6 @@ // /data/v7/fdb/ #define KAIO_LOGGING 0 -struct AsyncFileKAIOMetrics { - Reference readLatencyDist; - Reference writeLatencyDist; - Reference syncLatencyDist; -} g_asyncFileKAIOMetrics; - -Future g_asyncFileKAIOHistogramLogger; - DESCR struct SlowAioSubmit { int64_t submitDuration; // ns int64_t truncateDuration; // ns @@ -66,6 +58,25 @@ DESCR struct SlowAioSubmit { class AsyncFileKAIO final : public IAsyncFile, public ReferenceCounted { public: + struct AsyncFileKAIOMetrics { + LatencySample readLatencySample = { "AsyncFileKAIOReadLatency", + UID(), + FLOW_KNOBS->KAIO_LATENCY_LOGGING_INTERVAL, + FLOW_KNOBS->KAIO_LATENCY_SAMPLE_SIZE }; + LatencySample writeLatencySample = { "AsyncFileKAIOWriteLatency", + UID(), + FLOW_KNOBS->KAIO_LATENCY_LOGGING_INTERVAL, + FLOW_KNOBS->KAIO_LATENCY_SAMPLE_SIZE }; + LatencySample syncLatencySample = { "AsyncFileKAIOSyncLatency", + UID(), + FLOW_KNOBS->KAIO_LATENCY_LOGGING_INTERVAL, + FLOW_KNOBS->KAIO_LATENCY_SAMPLE_SIZE }; + }; + + static AsyncFileKAIOMetrics& getMetrics() { + static AsyncFileKAIOMetrics metrics; + return metrics; + } #if KAIO_LOGGING private: @@ -365,7 +376,7 @@ public: fsync = map(fsync, [=](Void r) mutable { KAIOLogEvent(logFile, id, OpLogEntry::SYNC, OpLogEntry::COMPLETE); - g_asyncFileKAIOMetrics.syncLatencyDist->sampleSeconds(now() - start_time); + getMetrics().syncLatencySample.addMeasurement(now() - start_time); return r; }); @@ -640,16 +651,6 @@ private: countFileLogicalReads.init(LiteralStringRef("AsyncFile.CountFileLogicalReads"), filename); countLogicalWrites.init(LiteralStringRef("AsyncFile.CountLogicalWrites")); countLogicalReads.init(LiteralStringRef("AsyncFile.CountLogicalReads")); - if (!g_asyncFileKAIOHistogramLogger.isValid()) { - auto& metrics = g_asyncFileKAIOMetrics; - metrics.readLatencyDist = Reference(new Histogram( - Reference(), "AsyncFileKAIO", "ReadLatency", Histogram::Unit::microseconds)); - metrics.writeLatencyDist = Reference(new Histogram( - Reference(), "AsyncFileKAIO", "WriteLatency", Histogram::Unit::microseconds)); - metrics.syncLatencyDist = Reference(new Histogram( - Reference(), "AsyncFileKAIO", "SyncLatency", Histogram::Unit::microseconds)); - g_asyncFileKAIOHistogramLogger = histogramLogger(FLOW_KNOBS->DISK_METRIC_LOGGING_INTERVAL); - } } #if KAIO_LOGGING @@ -769,13 +770,12 @@ private: ctx.removeFromRequestList(iob); } - auto& metrics = g_asyncFileKAIOMetrics; switch (iob->aio_lio_opcode) { case IO_CMD_PREAD: - metrics.readLatencyDist->sampleSeconds(now() - iob->startTime); + getMetrics().readLatencySample.addMeasurement(now() - iob->startTime); break; case IO_CMD_PWRITE: - metrics.writeLatencyDist->sampleSeconds(now() - iob->startTime); + getMetrics().writeLatencySample.addMeasurement(now() - iob->startTime); break; } @@ -783,19 +783,6 @@ private: } } } - - ACTOR static Future histogramLogger(double interval) { - state double currentTime; - loop { - currentTime = now(); - wait(delay(interval)); - double elapsed = now() - currentTime; - auto& metrics = g_asyncFileKAIOMetrics; - metrics.readLatencyDist->writeToLog(elapsed); - metrics.writeLatencyDist->writeToLog(elapsed); - metrics.syncLatencyDist->writeToLog(elapsed); - } - } }; #if KAIO_LOGGING diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp index 123bd04a52..c84424f5e6 100644 --- a/fdbserver/KeyValueStoreSQLite.actor.cpp +++ b/fdbserver/KeyValueStoreSQLite.actor.cpp @@ -1996,7 +1996,7 @@ private: state int64_t lastReadsComplete = 0; state int64_t lastWritesComplete = 0; loop { - wait(delay(FLOW_KNOBS->DISK_METRIC_LOGGING_INTERVAL)); + wait(delay(FLOW_KNOBS->SQLITE_DISK_METRIC_LOGGING_INTERVAL)); int64_t rc = self->readsComplete, wc = self->writesComplete; TraceEvent("DiskMetrics", self->logID) diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 73b98cde6c..197c7a746f 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -163,7 +163,9 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) { //AsyncFileKAIO init( MAX_OUTSTANDING, 64 ); init( MIN_SUBMIT, 10 ); - init( DISK_METRIC_LOGGING_INTERVAL, 5.0 ); + init( SQLITE_DISK_METRIC_LOGGING_INTERVAL, 5.0 ); + init( KAIO_LATENCY_LOGGING_INTERVAL, 30.0 ); + init( KAIO_LATENCY_SAMPLE_SIZE, 30000 ); init( PAGE_WRITE_CHECKSUM_HISTORY, 0 ); if( randomize && BUGGIFY ) PAGE_WRITE_CHECKSUM_HISTORY = 10000000; init( DISABLE_POSIX_KERNEL_AIO, 0 ); diff --git a/flow/include/flow/Knobs.h b/flow/include/flow/Knobs.h index cdc9f2491e..322e3ed6ce 100644 --- a/flow/include/flow/Knobs.h +++ b/flow/include/flow/Knobs.h @@ -227,7 +227,9 @@ public: // AsyncFileKAIO int MAX_OUTSTANDING; int MIN_SUBMIT; - double DISK_METRIC_LOGGING_INTERVAL; + double SQLITE_DISK_METRIC_LOGGING_INTERVAL; + double KAIO_LATENCY_LOGGING_INTERVAL; + int KAIO_LATENCY_SAMPLE_SIZE; int PAGE_WRITE_CHECKSUM_HISTORY; int DISABLE_POSIX_KERNEL_AIO;