diff --git a/documentation/sphinx/source/api-c.rst b/documentation/sphinx/source/api-c.rst index 2c2e256884..2de7056baf 100644 --- a/documentation/sphinx/source/api-c.rst +++ b/documentation/sphinx/source/api-c.rst @@ -481,7 +481,11 @@ An |database-blurb1| Modifications to a database are performed via transactions. |length-of| ``snapshot_command`` .. note:: The function is exposing the functionality of the fdbcli command ``snapshot``. Please take a look at the documentation before using (see :ref:`disk-snapshot-backups`). - + +.. function:: double fdb_database_get_main_thread_busyness(FDBDatabase* database) + + Returns a value between `0` and `1` representing the percentage of time the Network Thread is busy executing work from the TaskQueue. This value is updated every second. + Transaction =========== diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 5666a831c4..1f576d27f8 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -1733,6 +1733,33 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu } } +// update the network busyness on a 1s cadence +ACTOR Future monitorNetworkBusyness() { + state double prevTime = now(); + loop choose { + when(wait(delay(CLIENT_KNOBS->NETWORK_BUSYNESS_MONITOR_INTERVAL, TaskPriority::FlushTrace))) { + double elapsed = now() - prevTime; // get elapsed time from last execution + prevTime = now(); + if (!g_network->isSimulated()) { + struct NetworkMetrics::PriorityStats& itr = + g_network->networkInfo.metrics.starvationTrackerNetworkBusyness; + + if (itr.active) { // update metrics + itr.duration += now() - itr.windowedTimer; + itr.maxDuration = std::max(itr.maxDuration, now() - itr.timer); + itr.windowedTimer = now(); + } + + g_network->networkInfo.metrics.networkBusyness = + std::min(elapsed, itr.duration) / elapsed; // average duration spent doing "work" + + itr.duration = 0; + itr.maxDuration = 0; + } + } + } +} + // Setup g_network and start monitoring for network busyness void setupNetwork(uint64_t transportId, bool useMetrics) { if (g_network) @@ -1748,8 +1775,7 @@ void setupNetwork(uint64_t transportId, bool useMetrics) { FlowTransport::createInstance(true, transportId); Net2FileSystem::newFileSystem(); - monitorNetworkBusyness(); - uncancellable(recurring(&monitorNetworkBusyness, CLIENT_KNOBS->NETWORK_BUSYNESS_MONITOR_INTERVAL, TaskPriority::FlushTrace)); + uncancellable(monitorNetworkBusyness()); } void runNetwork() { diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp index 5a999afb6a..1d07e30782 100644 --- a/fdbclient/ThreadSafeTransaction.cpp +++ b/fdbclient/ThreadSafeTransaction.cpp @@ -93,6 +93,7 @@ ThreadFuture ThreadSafeDatabase::createSnapshot(const StringRef& uid, cons double ThreadSafeDatabase::getMainThreadBusyness() { // Return the main network thread busyness + ASSERT(g_network); return g_network->networkInfo.metrics.networkBusyness; } diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index 28c1ef1e1f..feb6dad8f2 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -1622,13 +1622,8 @@ void Net2::trackAtPriority(TaskPriority priority, double now) { updateStarvationTrackers(binStats, priority, lastPriority, now); } - // Update starvation trackers for 1s measurment interval - for (auto& binStats : networkInfo.metrics.starvationTrackersNetworkBusyness) { - if (binStats.priority > lastPriority && binStats.priority > priority) { - break; - } - updateStarvationTrackers(binStats, priority, lastPriority, now); - } + // Update starvation trackers for network busyness + updateStarvationTrackers(networkInfo.metrics.starvationTrackerNetworkBusyness, priority, lastPriority, now); lastPriorityStats = &activeStatsItr.first->second; } diff --git a/flow/SystemMonitor.cpp b/flow/SystemMonitor.cpp index 76fa78e0b8..37dadf9dc1 100644 --- a/flow/SystemMonitor.cpp +++ b/flow/SystemMonitor.cpp @@ -61,34 +61,6 @@ SystemStatistics getSystemStatistics() { .detail("ApproximateUnusedMemory" #size, FastAllocator::getApproximateMemoryUnused()) \ .detail("ActiveThreads" #size, FastAllocator::getActiveThreads()) -// update the network busyness on a 1s cadence -void monitorNetworkBusyness() { - const IPAddress ipAddr = IPAddress(); - static StatisticsState statState = StatisticsState(); - SystemStatistics currentStats = getSystemStatistics("", &ipAddr, &(&statState)->systemState, true); - if (!g_network->isSimulated() && currentStats.initialized) { - bool firstTracker = true; - // iterate over the starvation trackers which are updated in Net2.actor.cpp - for (auto& itr : g_network->networkInfo.metrics.starvationTrackersNetworkBusyness) { - if (itr.active) { - itr.duration += now() - itr.windowedTimer; - itr.maxDuration = std::max(itr.maxDuration, now() - itr.timer); - itr.windowedTimer = now(); - } - - if (firstTracker) { - g_network->networkInfo.metrics.networkBusyness = - std::min(currentStats.elapsed, itr.duration) / - currentStats.elapsed; // average duration spent doing "work" - firstTracker = false; - } - - itr.duration = 0; - itr.maxDuration = 0; - } - } -} - SystemStatistics customSystemMonitor(std::string eventName, StatisticsState* statState, bool machineMetrics) { const IPAddress ipAddr = machineState.ip.present() ? machineState.ip.get() : IPAddress(); SystemStatistics currentStats = getSystemStatistics( diff --git a/flow/SystemMonitor.h b/flow/SystemMonitor.h index 6e25f2fff9..2d2470bd48 100644 --- a/flow/SystemMonitor.h +++ b/flow/SystemMonitor.h @@ -148,7 +148,6 @@ struct StatisticsState { }; void systemMonitor(); -void monitorNetworkBusyness(); SystemStatistics customSystemMonitor(std::string eventName, StatisticsState* statState, bool machineMetrics = false); SystemStatistics getSystemStatistics(); diff --git a/flow/network.h b/flow/network.h index 04b93653dd..fc641d39a8 100644 --- a/flow/network.h +++ b/flow/network.h @@ -343,14 +343,16 @@ struct NetworkMetrics { std::unordered_map activeTrackers; double lastRunLoopBusyness; std::atomic networkBusyness; - std::vector starvationTrackers, starvationTrackersNetworkBusyness; + std::vector starvationTrackers; + struct PriorityStats starvationTrackerNetworkBusyness; static const std::vector starvationBins; - NetworkMetrics() : lastRunLoopBusyness(0), networkBusyness(0) { + NetworkMetrics() + : lastRunLoopBusyness(0), networkBusyness(0), + starvationTrackerNetworkBusyness(PriorityStats(static_cast(starvationBins.at(0)))) { for (int priority : starvationBins) { starvationTrackers.emplace_back(static_cast(priority)); - starvationTrackersNetworkBusyness.emplace_back(static_cast(priority)); } } @@ -365,7 +367,7 @@ struct NetworkMetrics { lastRunLoopBusyness = rhs.lastRunLoopBusyness; networkBusyness = rhs.networkBusyness.load(); starvationTrackers = rhs.starvationTrackers; - starvationTrackersNetworkBusyness = rhs.starvationTrackersNetworkBusyness; + starvationTrackerNetworkBusyness = rhs.starvationTrackerNetworkBusyness; return *this; } };