2018-05-15 08:42:36 +08:00
|
|
|
//===-- xray_function_call_trie.h ------------------------------*- C++ -*-===//
|
|
|
|
//
|
2019-01-19 16:50:56 +08:00
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
2018-05-15 08:42:36 +08:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// This file is a part of XRay, a dynamic runtime instrumentation system.
|
|
|
|
//
|
|
|
|
// This file defines the interface for a function call trie.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef XRAY_FUNCTION_CALL_TRIE_H
|
|
|
|
#define XRAY_FUNCTION_CALL_TRIE_H
|
|
|
|
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
#include "xray_buffer_queue.h"
|
2018-09-07 18:16:14 +08:00
|
|
|
#include "xray_defs.h"
|
2018-06-12 11:29:39 +08:00
|
|
|
#include "xray_profiling_flags.h"
|
2018-05-15 08:42:36 +08:00
|
|
|
#include "xray_segmented_array.h"
|
2018-11-05 13:43:22 +08:00
|
|
|
#include <limits>
|
2018-07-10 16:25:44 +08:00
|
|
|
#include <memory> // For placement new.
|
2018-05-15 08:42:36 +08:00
|
|
|
#include <utility>
|
|
|
|
|
|
|
|
namespace __xray {
|
|
|
|
|
|
|
|
/// A FunctionCallTrie represents the stack traces of XRay instrumented
|
|
|
|
/// functions that we've encountered, where a node corresponds to a function and
|
|
|
|
/// the path from the root to the node its stack trace. Each node in the trie
|
|
|
|
/// will contain some useful values, including:
|
|
|
|
///
|
|
|
|
/// * The cumulative amount of time spent in this particular node/stack.
|
|
|
|
/// * The number of times this stack has appeared.
|
|
|
|
/// * A histogram of latencies for that particular node.
|
|
|
|
///
|
|
|
|
/// Each node in the trie will also contain a list of callees, represented using
|
|
|
|
/// a Array<NodeIdPair> -- each NodeIdPair instance will contain the function
|
|
|
|
/// ID of the callee, and a pointer to the node.
|
|
|
|
///
|
|
|
|
/// If we visualise this data structure, we'll find the following potential
|
|
|
|
/// representation:
|
|
|
|
///
|
|
|
|
/// [function id node] -> [callees] [cumulative time]
|
|
|
|
/// [call counter] [latency histogram]
|
|
|
|
///
|
|
|
|
/// As an example, when we have a function in this pseudocode:
|
|
|
|
///
|
|
|
|
/// func f(N) {
|
|
|
|
/// g()
|
|
|
|
/// h()
|
|
|
|
/// for i := 1..N { j() }
|
|
|
|
/// }
|
|
|
|
///
|
|
|
|
/// We may end up with a trie of the following form:
|
|
|
|
///
|
|
|
|
/// f -> [ g, h, j ] [...] [1] [...]
|
|
|
|
/// g -> [ ... ] [...] [1] [...]
|
|
|
|
/// h -> [ ... ] [...] [1] [...]
|
|
|
|
/// j -> [ ... ] [...] [N] [...]
|
|
|
|
///
|
|
|
|
/// If for instance the function g() called j() like so:
|
|
|
|
///
|
|
|
|
/// func g() {
|
|
|
|
/// for i := 1..10 { j() }
|
|
|
|
/// }
|
|
|
|
///
|
|
|
|
/// We'll find the following updated trie:
|
|
|
|
///
|
|
|
|
/// f -> [ g, h, j ] [...] [1] [...]
|
|
|
|
/// g -> [ j' ] [...] [1] [...]
|
|
|
|
/// h -> [ ... ] [...] [1] [...]
|
|
|
|
/// j -> [ ... ] [...] [N] [...]
|
|
|
|
/// j' -> [ ... ] [...] [10] [...]
|
|
|
|
///
|
|
|
|
/// Note that we'll have a new node representing the path `f -> g -> j'` with
|
|
|
|
/// isolated data. This isolation gives us a means of representing the stack
|
|
|
|
/// traces as a path, as opposed to a key in a table. The alternative
|
|
|
|
/// implementation here would be to use a separate table for the path, and use
|
|
|
|
/// hashes of the path as an identifier to accumulate the information. We've
|
|
|
|
/// moved away from this approach as it takes a lot of time to compute the hash
|
|
|
|
/// every time we need to update a function's call information as we're handling
|
|
|
|
/// the entry and exit events.
|
|
|
|
///
|
|
|
|
/// This approach allows us to maintain a shadow stack, which represents the
|
|
|
|
/// currently executing path, and on function exits quickly compute the amount
|
|
|
|
/// of time elapsed from the entry, then update the counters for the node
|
|
|
|
/// already represented in the trie. This necessitates an efficient
|
|
|
|
/// representation of the various data structures (the list of callees must be
|
|
|
|
/// cache-aware and efficient to look up, and the histogram must be compact and
|
|
|
|
/// quick to update) to enable us to keep the overheads of this implementation
|
|
|
|
/// to the minimum.
|
|
|
|
class FunctionCallTrie {
|
|
|
|
public:
|
|
|
|
struct Node;
|
|
|
|
|
|
|
|
// We use a NodeIdPair type instead of a std::pair<...> to not rely on the
|
|
|
|
// standard library types in this header.
|
|
|
|
struct NodeIdPair {
|
|
|
|
Node *NodePtr;
|
|
|
|
int32_t FId;
|
|
|
|
};
|
|
|
|
|
|
|
|
using NodeIdPairArray = Array<NodeIdPair>;
|
|
|
|
using NodeIdPairAllocatorType = NodeIdPairArray::AllocatorType;
|
|
|
|
|
|
|
|
// A Node in the FunctionCallTrie gives us a list of callees, the cumulative
|
|
|
|
// number of times this node actually appeared, the cumulative amount of time
|
|
|
|
// for this particular node including its children call times, and just the
|
|
|
|
// local time spent on this node. Each Node will have the ID of the XRay
|
|
|
|
// instrumented function that it is associated to.
|
|
|
|
struct Node {
|
|
|
|
Node *Parent;
|
|
|
|
NodeIdPairArray Callees;
|
2018-11-05 13:43:22 +08:00
|
|
|
uint64_t CallCount;
|
|
|
|
uint64_t CumulativeLocalTime; // Typically in TSC deltas, not wall-time.
|
2018-05-15 08:42:36 +08:00
|
|
|
int32_t FId;
|
|
|
|
|
|
|
|
// TODO: Include the compact histogram.
|
|
|
|
};
|
|
|
|
|
|
|
|
private:
|
|
|
|
struct ShadowStackEntry {
|
|
|
|
uint64_t EntryTSC;
|
|
|
|
Node *NodePtr;
|
2018-11-05 13:43:22 +08:00
|
|
|
uint16_t EntryCPU;
|
2018-05-15 08:42:36 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
using NodeArray = Array<Node>;
|
|
|
|
using RootArray = Array<Node *>;
|
|
|
|
using ShadowStackArray = Array<ShadowStackEntry>;
|
|
|
|
|
|
|
|
public:
|
|
|
|
// We collate the allocators we need into a single struct, as a convenience to
|
|
|
|
// allow us to initialize these as a group.
|
|
|
|
struct Allocators {
|
|
|
|
using NodeAllocatorType = NodeArray::AllocatorType;
|
|
|
|
using RootAllocatorType = RootArray::AllocatorType;
|
|
|
|
using ShadowStackAllocatorType = ShadowStackArray::AllocatorType;
|
|
|
|
|
2018-12-07 11:19:13 +08:00
|
|
|
// Use hosted aligned storage members to allow for trivial move and init.
|
|
|
|
// This also allows us to sidestep the potential-failing allocation issue.
|
|
|
|
typename std::aligned_storage<sizeof(NodeAllocatorType),
|
|
|
|
alignof(NodeAllocatorType)>::type
|
|
|
|
NodeAllocatorStorage;
|
|
|
|
typename std::aligned_storage<sizeof(RootAllocatorType),
|
|
|
|
alignof(RootAllocatorType)>::type
|
|
|
|
RootAllocatorStorage;
|
|
|
|
typename std::aligned_storage<sizeof(ShadowStackAllocatorType),
|
|
|
|
alignof(ShadowStackAllocatorType)>::type
|
|
|
|
ShadowStackAllocatorStorage;
|
|
|
|
typename std::aligned_storage<sizeof(NodeIdPairAllocatorType),
|
|
|
|
alignof(NodeIdPairAllocatorType)>::type
|
|
|
|
NodeIdPairAllocatorStorage;
|
|
|
|
|
2018-05-15 08:42:36 +08:00
|
|
|
NodeAllocatorType *NodeAllocator = nullptr;
|
|
|
|
RootAllocatorType *RootAllocator = nullptr;
|
|
|
|
ShadowStackAllocatorType *ShadowStackAllocator = nullptr;
|
|
|
|
NodeIdPairAllocatorType *NodeIdPairAllocator = nullptr;
|
|
|
|
|
2018-12-07 11:19:13 +08:00
|
|
|
Allocators() = default;
|
2018-05-15 08:42:36 +08:00
|
|
|
Allocators(const Allocators &) = delete;
|
|
|
|
Allocators &operator=(const Allocators &) = delete;
|
|
|
|
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
struct Buffers {
|
|
|
|
BufferQueue::Buffer NodeBuffer;
|
|
|
|
BufferQueue::Buffer RootsBuffer;
|
|
|
|
BufferQueue::Buffer ShadowStackBuffer;
|
|
|
|
BufferQueue::Buffer NodeIdPairBuffer;
|
|
|
|
};
|
|
|
|
|
|
|
|
explicit Allocators(Buffers &B) XRAY_NEVER_INSTRUMENT {
|
|
|
|
new (&NodeAllocatorStorage)
|
|
|
|
NodeAllocatorType(B.NodeBuffer.Data, B.NodeBuffer.Size);
|
|
|
|
NodeAllocator =
|
|
|
|
reinterpret_cast<NodeAllocatorType *>(&NodeAllocatorStorage);
|
|
|
|
|
|
|
|
new (&RootAllocatorStorage)
|
|
|
|
RootAllocatorType(B.RootsBuffer.Data, B.RootsBuffer.Size);
|
|
|
|
RootAllocator =
|
|
|
|
reinterpret_cast<RootAllocatorType *>(&RootAllocatorStorage);
|
|
|
|
|
|
|
|
new (&ShadowStackAllocatorStorage) ShadowStackAllocatorType(
|
|
|
|
B.ShadowStackBuffer.Data, B.ShadowStackBuffer.Size);
|
|
|
|
ShadowStackAllocator = reinterpret_cast<ShadowStackAllocatorType *>(
|
|
|
|
&ShadowStackAllocatorStorage);
|
|
|
|
|
|
|
|
new (&NodeIdPairAllocatorStorage) NodeIdPairAllocatorType(
|
|
|
|
B.NodeIdPairBuffer.Data, B.NodeIdPairBuffer.Size);
|
|
|
|
NodeIdPairAllocator = reinterpret_cast<NodeIdPairAllocatorType *>(
|
|
|
|
&NodeIdPairAllocatorStorage);
|
|
|
|
}
|
|
|
|
|
2018-12-07 11:19:13 +08:00
|
|
|
explicit Allocators(uptr Max) XRAY_NEVER_INSTRUMENT {
|
|
|
|
new (&NodeAllocatorStorage) NodeAllocatorType(Max);
|
|
|
|
NodeAllocator =
|
|
|
|
reinterpret_cast<NodeAllocatorType *>(&NodeAllocatorStorage);
|
|
|
|
|
|
|
|
new (&RootAllocatorStorage) RootAllocatorType(Max);
|
|
|
|
RootAllocator =
|
|
|
|
reinterpret_cast<RootAllocatorType *>(&RootAllocatorStorage);
|
|
|
|
|
|
|
|
new (&ShadowStackAllocatorStorage) ShadowStackAllocatorType(Max);
|
|
|
|
ShadowStackAllocator = reinterpret_cast<ShadowStackAllocatorType *>(
|
|
|
|
&ShadowStackAllocatorStorage);
|
|
|
|
|
|
|
|
new (&NodeIdPairAllocatorStorage) NodeIdPairAllocatorType(Max);
|
|
|
|
NodeIdPairAllocator = reinterpret_cast<NodeIdPairAllocatorType *>(
|
|
|
|
&NodeIdPairAllocatorStorage);
|
|
|
|
}
|
|
|
|
|
|
|
|
Allocators(Allocators &&O) XRAY_NEVER_INSTRUMENT {
|
|
|
|
// Here we rely on the safety of memcpy'ing contents of the storage
|
|
|
|
// members, and then pointing the source pointers to nullptr.
|
|
|
|
internal_memcpy(&NodeAllocatorStorage, &O.NodeAllocatorStorage,
|
|
|
|
sizeof(NodeAllocatorType));
|
|
|
|
internal_memcpy(&RootAllocatorStorage, &O.RootAllocatorStorage,
|
|
|
|
sizeof(RootAllocatorType));
|
|
|
|
internal_memcpy(&ShadowStackAllocatorStorage,
|
|
|
|
&O.ShadowStackAllocatorStorage,
|
|
|
|
sizeof(ShadowStackAllocatorType));
|
|
|
|
internal_memcpy(&NodeIdPairAllocatorStorage,
|
|
|
|
&O.NodeIdPairAllocatorStorage,
|
|
|
|
sizeof(NodeIdPairAllocatorType));
|
|
|
|
|
|
|
|
NodeAllocator =
|
|
|
|
reinterpret_cast<NodeAllocatorType *>(&NodeAllocatorStorage);
|
|
|
|
RootAllocator =
|
|
|
|
reinterpret_cast<RootAllocatorType *>(&RootAllocatorStorage);
|
|
|
|
ShadowStackAllocator = reinterpret_cast<ShadowStackAllocatorType *>(
|
|
|
|
&ShadowStackAllocatorStorage);
|
|
|
|
NodeIdPairAllocator = reinterpret_cast<NodeIdPairAllocatorType *>(
|
|
|
|
&NodeIdPairAllocatorStorage);
|
|
|
|
|
2018-05-15 08:42:36 +08:00
|
|
|
O.NodeAllocator = nullptr;
|
|
|
|
O.RootAllocator = nullptr;
|
|
|
|
O.ShadowStackAllocator = nullptr;
|
|
|
|
O.NodeIdPairAllocator = nullptr;
|
|
|
|
}
|
|
|
|
|
2018-09-07 18:16:14 +08:00
|
|
|
Allocators &operator=(Allocators &&O) XRAY_NEVER_INSTRUMENT {
|
2018-12-07 11:19:13 +08:00
|
|
|
// When moving into an existing instance, we ensure that we clean up the
|
|
|
|
// current allocators.
|
|
|
|
if (NodeAllocator)
|
2018-12-06 08:25:56 +08:00
|
|
|
NodeAllocator->~NodeAllocatorType();
|
2018-12-07 11:19:13 +08:00
|
|
|
if (O.NodeAllocator) {
|
|
|
|
new (&NodeAllocatorStorage)
|
|
|
|
NodeAllocatorType(std::move(*O.NodeAllocator));
|
|
|
|
NodeAllocator =
|
|
|
|
reinterpret_cast<NodeAllocatorType *>(&NodeAllocatorStorage);
|
|
|
|
O.NodeAllocator = nullptr;
|
|
|
|
} else {
|
2018-12-06 11:28:57 +08:00
|
|
|
NodeAllocator = nullptr;
|
|
|
|
}
|
2018-12-07 11:19:13 +08:00
|
|
|
|
|
|
|
if (RootAllocator)
|
2018-12-06 08:25:56 +08:00
|
|
|
RootAllocator->~RootAllocatorType();
|
2018-12-07 11:19:13 +08:00
|
|
|
if (O.RootAllocator) {
|
|
|
|
new (&RootAllocatorStorage)
|
|
|
|
RootAllocatorType(std::move(*O.RootAllocator));
|
|
|
|
RootAllocator =
|
|
|
|
reinterpret_cast<RootAllocatorType *>(&RootAllocatorStorage);
|
|
|
|
O.RootAllocator = nullptr;
|
|
|
|
} else {
|
2018-12-06 11:28:57 +08:00
|
|
|
RootAllocator = nullptr;
|
|
|
|
}
|
2018-12-07 11:19:13 +08:00
|
|
|
|
|
|
|
if (ShadowStackAllocator)
|
2018-12-06 08:25:56 +08:00
|
|
|
ShadowStackAllocator->~ShadowStackAllocatorType();
|
2018-12-07 11:19:13 +08:00
|
|
|
if (O.ShadowStackAllocator) {
|
|
|
|
new (&ShadowStackAllocatorStorage)
|
|
|
|
ShadowStackAllocatorType(std::move(*O.ShadowStackAllocator));
|
|
|
|
ShadowStackAllocator = reinterpret_cast<ShadowStackAllocatorType *>(
|
|
|
|
&ShadowStackAllocatorStorage);
|
|
|
|
O.ShadowStackAllocator = nullptr;
|
|
|
|
} else {
|
2018-12-06 11:28:57 +08:00
|
|
|
ShadowStackAllocator = nullptr;
|
|
|
|
}
|
2018-12-07 11:19:13 +08:00
|
|
|
|
|
|
|
if (NodeIdPairAllocator)
|
2018-12-06 08:25:56 +08:00
|
|
|
NodeIdPairAllocator->~NodeIdPairAllocatorType();
|
2018-12-07 11:19:13 +08:00
|
|
|
if (O.NodeIdPairAllocator) {
|
|
|
|
new (&NodeIdPairAllocatorStorage)
|
|
|
|
NodeIdPairAllocatorType(std::move(*O.NodeIdPairAllocator));
|
|
|
|
NodeIdPairAllocator = reinterpret_cast<NodeIdPairAllocatorType *>(
|
|
|
|
&NodeIdPairAllocatorStorage);
|
|
|
|
O.NodeIdPairAllocator = nullptr;
|
|
|
|
} else {
|
2018-12-06 11:28:57 +08:00
|
|
|
NodeIdPairAllocator = nullptr;
|
|
|
|
}
|
2018-12-07 11:19:13 +08:00
|
|
|
|
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
|
|
|
|
~Allocators() XRAY_NEVER_INSTRUMENT {
|
|
|
|
if (NodeAllocator != nullptr)
|
|
|
|
NodeAllocator->~NodeAllocatorType();
|
|
|
|
if (RootAllocator != nullptr)
|
|
|
|
RootAllocator->~RootAllocatorType();
|
|
|
|
if (ShadowStackAllocator != nullptr)
|
|
|
|
ShadowStackAllocator->~ShadowStackAllocatorType();
|
|
|
|
if (NodeIdPairAllocator != nullptr)
|
|
|
|
NodeIdPairAllocator->~NodeIdPairAllocatorType();
|
2018-05-15 08:42:36 +08:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2018-09-07 18:16:14 +08:00
|
|
|
static Allocators InitAllocators() XRAY_NEVER_INSTRUMENT {
|
2018-07-10 16:25:44 +08:00
|
|
|
return InitAllocatorsCustom(profilingFlags()->per_thread_allocator_max);
|
|
|
|
}
|
|
|
|
|
2018-09-07 18:16:14 +08:00
|
|
|
static Allocators InitAllocatorsCustom(uptr Max) XRAY_NEVER_INSTRUMENT {
|
2018-12-07 11:19:13 +08:00
|
|
|
Allocators A(Max);
|
2018-05-15 08:42:36 +08:00
|
|
|
return A;
|
|
|
|
}
|
|
|
|
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
static Allocators
|
|
|
|
InitAllocatorsFromBuffers(Allocators::Buffers &Bufs) XRAY_NEVER_INSTRUMENT {
|
|
|
|
Allocators A(Bufs);
|
|
|
|
return A;
|
|
|
|
}
|
|
|
|
|
2018-05-15 08:42:36 +08:00
|
|
|
private:
|
|
|
|
NodeArray Nodes;
|
|
|
|
RootArray Roots;
|
|
|
|
ShadowStackArray ShadowStack;
|
2018-12-07 11:19:13 +08:00
|
|
|
NodeIdPairAllocatorType *NodeIdPairAllocator;
|
|
|
|
uint32_t OverflowedFunctions;
|
2018-05-15 08:42:36 +08:00
|
|
|
|
|
|
|
public:
|
2018-09-07 18:16:14 +08:00
|
|
|
explicit FunctionCallTrie(const Allocators &A) XRAY_NEVER_INSTRUMENT
|
|
|
|
: Nodes(*A.NodeAllocator),
|
|
|
|
Roots(*A.RootAllocator),
|
2018-07-18 10:08:39 +08:00
|
|
|
ShadowStack(*A.ShadowStackAllocator),
|
2018-12-07 11:19:13 +08:00
|
|
|
NodeIdPairAllocator(A.NodeIdPairAllocator),
|
|
|
|
OverflowedFunctions(0) {}
|
|
|
|
|
|
|
|
FunctionCallTrie() = delete;
|
|
|
|
FunctionCallTrie(const FunctionCallTrie &) = delete;
|
|
|
|
FunctionCallTrie &operator=(const FunctionCallTrie &) = delete;
|
|
|
|
|
|
|
|
FunctionCallTrie(FunctionCallTrie &&O) XRAY_NEVER_INSTRUMENT
|
|
|
|
: Nodes(std::move(O.Nodes)),
|
|
|
|
Roots(std::move(O.Roots)),
|
|
|
|
ShadowStack(std::move(O.ShadowStack)),
|
|
|
|
NodeIdPairAllocator(O.NodeIdPairAllocator),
|
|
|
|
OverflowedFunctions(O.OverflowedFunctions) {}
|
|
|
|
|
|
|
|
FunctionCallTrie &operator=(FunctionCallTrie &&O) XRAY_NEVER_INSTRUMENT {
|
|
|
|
Nodes = std::move(O.Nodes);
|
|
|
|
Roots = std::move(O.Roots);
|
|
|
|
ShadowStack = std::move(O.ShadowStack);
|
|
|
|
NodeIdPairAllocator = O.NodeIdPairAllocator;
|
|
|
|
OverflowedFunctions = O.OverflowedFunctions;
|
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
|
|
|
|
~FunctionCallTrie() XRAY_NEVER_INSTRUMENT {}
|
2018-05-15 08:42:36 +08:00
|
|
|
|
2018-11-05 13:43:22 +08:00
|
|
|
void enterFunction(const int32_t FId, uint64_t TSC,
|
|
|
|
uint16_t CPU) XRAY_NEVER_INSTRUMENT {
|
2018-07-10 16:25:44 +08:00
|
|
|
DCHECK_NE(FId, 0);
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
|
|
|
|
// If we're already overflowed the function call stack, do not bother
|
|
|
|
// attempting to record any more function entries.
|
|
|
|
if (UNLIKELY(OverflowedFunctions)) {
|
|
|
|
++OverflowedFunctions;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// If this is the first function we've encountered, we want to set up the
|
|
|
|
// node(s) and treat it as a root.
|
2018-05-15 08:42:36 +08:00
|
|
|
if (UNLIKELY(ShadowStack.empty())) {
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
auto *NewRoot = Nodes.AppendEmplace(
|
|
|
|
nullptr, NodeIdPairArray(*NodeIdPairAllocator), 0u, 0u, FId);
|
2018-05-15 08:42:36 +08:00
|
|
|
if (UNLIKELY(NewRoot == nullptr))
|
|
|
|
return;
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
if (Roots.AppendEmplace(NewRoot) == nullptr) {
|
|
|
|
Nodes.trim(1);
|
2018-12-07 11:19:13 +08:00
|
|
|
return;
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
}
|
2018-12-07 11:19:13 +08:00
|
|
|
if (ShadowStack.AppendEmplace(TSC, NewRoot, CPU) == nullptr) {
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
Nodes.trim(1);
|
2018-12-07 11:19:13 +08:00
|
|
|
Roots.trim(1);
|
|
|
|
++OverflowedFunctions;
|
|
|
|
return;
|
|
|
|
}
|
2018-05-15 08:42:36 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
// From this point on, we require that the stack is not empty.
|
|
|
|
DCHECK(!ShadowStack.empty());
|
|
|
|
auto TopNode = ShadowStack.back().NodePtr;
|
2018-07-10 16:25:44 +08:00
|
|
|
DCHECK_NE(TopNode, nullptr);
|
2018-05-15 08:42:36 +08:00
|
|
|
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
// If we've seen this callee before, then we access that node and place that
|
|
|
|
// on the top of the stack.
|
|
|
|
auto* Callee = TopNode->Callees.find_element(
|
2018-05-15 08:42:36 +08:00
|
|
|
[FId](const NodeIdPair &NR) { return NR.FId == FId; });
|
|
|
|
if (Callee != nullptr) {
|
|
|
|
CHECK_NE(Callee->NodePtr, nullptr);
|
2018-12-07 11:19:13 +08:00
|
|
|
if (ShadowStack.AppendEmplace(TSC, Callee->NodePtr, CPU) == nullptr)
|
|
|
|
++OverflowedFunctions;
|
2018-05-15 08:42:36 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// This means we've never seen this stack before, create a new node here.
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
auto* NewNode = Nodes.AppendEmplace(
|
2018-12-07 11:19:13 +08:00
|
|
|
TopNode, NodeIdPairArray(*NodeIdPairAllocator), 0u, 0u, FId);
|
2018-05-15 08:42:36 +08:00
|
|
|
if (UNLIKELY(NewNode == nullptr))
|
|
|
|
return;
|
2018-07-10 16:25:44 +08:00
|
|
|
DCHECK_NE(NewNode, nullptr);
|
2018-05-15 08:42:36 +08:00
|
|
|
TopNode->Callees.AppendEmplace(NewNode, FId);
|
2018-12-07 11:19:13 +08:00
|
|
|
if (ShadowStack.AppendEmplace(TSC, NewNode, CPU) == nullptr)
|
|
|
|
++OverflowedFunctions;
|
2018-05-15 08:42:36 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-11-05 13:43:22 +08:00
|
|
|
void exitFunction(int32_t FId, uint64_t TSC,
|
|
|
|
uint16_t CPU) XRAY_NEVER_INSTRUMENT {
|
2018-12-07 11:19:13 +08:00
|
|
|
// If we're exiting functions that have "overflowed" or don't fit into the
|
|
|
|
// stack due to allocator constraints, we then decrement that count first.
|
|
|
|
if (OverflowedFunctions) {
|
|
|
|
--OverflowedFunctions;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-05-15 08:42:36 +08:00
|
|
|
// When we exit a function, we look up the ShadowStack to see whether we've
|
|
|
|
// entered this function before. We do as little processing here as we can,
|
|
|
|
// since most of the hard work would have already been done at function
|
|
|
|
// entry.
|
|
|
|
uint64_t CumulativeTreeTime = 0;
|
2018-12-07 11:19:13 +08:00
|
|
|
|
2018-05-15 08:42:36 +08:00
|
|
|
while (!ShadowStack.empty()) {
|
2018-07-10 16:25:44 +08:00
|
|
|
const auto &Top = ShadowStack.back();
|
2018-05-15 08:42:36 +08:00
|
|
|
auto TopNode = Top.NodePtr;
|
2018-07-10 16:25:44 +08:00
|
|
|
DCHECK_NE(TopNode, nullptr);
|
2018-11-05 13:43:22 +08:00
|
|
|
|
|
|
|
// We may encounter overflow on the TSC we're provided, which may end up
|
|
|
|
// being less than the TSC when we first entered the function.
|
|
|
|
//
|
|
|
|
// To get the accurate measurement of cycles, we need to check whether
|
|
|
|
// we've overflowed (TSC < Top.EntryTSC) and then account the difference
|
|
|
|
// between the entry TSC and the max for the TSC counter (max of uint64_t)
|
|
|
|
// then add the value of TSC. We can prove that the maximum delta we will
|
|
|
|
// get is at most the 64-bit unsigned value, since the difference between
|
|
|
|
// a TSC of 0 and a Top.EntryTSC of 1 is (numeric_limits<uint64_t>::max()
|
|
|
|
// - 1) + 1.
|
|
|
|
//
|
|
|
|
// NOTE: This assumes that TSCs are synchronised across CPUs.
|
|
|
|
// TODO: Count the number of times we've seen CPU migrations.
|
|
|
|
uint64_t LocalTime =
|
|
|
|
Top.EntryTSC > TSC
|
|
|
|
? (std::numeric_limits<uint64_t>::max() - Top.EntryTSC) + TSC
|
|
|
|
: TSC - Top.EntryTSC;
|
2018-05-15 08:42:36 +08:00
|
|
|
TopNode->CallCount++;
|
|
|
|
TopNode->CumulativeLocalTime += LocalTime - CumulativeTreeTime;
|
|
|
|
CumulativeTreeTime += LocalTime;
|
|
|
|
ShadowStack.trim(1);
|
|
|
|
|
|
|
|
// TODO: Update the histogram for the node.
|
2018-07-10 16:25:44 +08:00
|
|
|
if (TopNode->FId == FId)
|
2018-05-15 08:42:36 +08:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-09-07 18:16:14 +08:00
|
|
|
const RootArray &getRoots() const XRAY_NEVER_INSTRUMENT { return Roots; }
|
2018-05-15 08:42:36 +08:00
|
|
|
|
|
|
|
// The deepCopyInto operation will update the provided FunctionCallTrie by
|
|
|
|
// re-creating the contents of this particular FunctionCallTrie in the other
|
|
|
|
// FunctionCallTrie. It will do this using a Depth First Traversal from the
|
|
|
|
// roots, and while doing so recreating the traversal in the provided
|
|
|
|
// FunctionCallTrie.
|
|
|
|
//
|
|
|
|
// This operation will *not* destroy the state in `O`, and thus may cause some
|
|
|
|
// duplicate entries in `O` if it is not empty.
|
|
|
|
//
|
|
|
|
// This function is *not* thread-safe, and may require external
|
|
|
|
// synchronisation of both "this" and |O|.
|
|
|
|
//
|
|
|
|
// This function must *not* be called with a non-empty FunctionCallTrie |O|.
|
2018-09-07 18:16:14 +08:00
|
|
|
void deepCopyInto(FunctionCallTrie &O) const XRAY_NEVER_INSTRUMENT {
|
2018-05-15 08:42:36 +08:00
|
|
|
DCHECK(O.getRoots().empty());
|
2018-07-10 16:25:44 +08:00
|
|
|
|
|
|
|
// We then push the root into a stack, to use as the parent marker for new
|
|
|
|
// nodes we push in as we're traversing depth-first down the call tree.
|
|
|
|
struct NodeAndParent {
|
|
|
|
FunctionCallTrie::Node *Node;
|
|
|
|
FunctionCallTrie::Node *NewNode;
|
|
|
|
};
|
|
|
|
using Stack = Array<NodeAndParent>;
|
|
|
|
|
|
|
|
typename Stack::AllocatorType StackAllocator(
|
2018-07-18 09:53:39 +08:00
|
|
|
profilingFlags()->stack_allocator_max);
|
2018-07-18 10:08:39 +08:00
|
|
|
Stack DFSStack(StackAllocator);
|
2018-07-10 16:25:44 +08:00
|
|
|
|
2018-05-15 08:42:36 +08:00
|
|
|
for (const auto Root : getRoots()) {
|
|
|
|
// Add a node in O for this root.
|
|
|
|
auto NewRoot = O.Nodes.AppendEmplace(
|
2018-12-07 11:19:13 +08:00
|
|
|
nullptr, NodeIdPairArray(*O.NodeIdPairAllocator), Root->CallCount,
|
2018-05-15 08:42:36 +08:00
|
|
|
Root->CumulativeLocalTime, Root->FId);
|
|
|
|
|
2018-07-10 16:25:44 +08:00
|
|
|
// Because we cannot allocate more memory we should bail out right away.
|
|
|
|
if (UNLIKELY(NewRoot == nullptr))
|
|
|
|
return;
|
2018-05-15 08:42:36 +08:00
|
|
|
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
if (UNLIKELY(O.Roots.Append(NewRoot) == nullptr))
|
|
|
|
return;
|
2018-05-15 08:42:36 +08:00
|
|
|
|
|
|
|
// TODO: Figure out what to do if we fail to allocate any more stack
|
|
|
|
// space. Maybe warn or report once?
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
if (DFSStack.AppendEmplace(Root, NewRoot) == nullptr)
|
|
|
|
return;
|
2018-05-15 08:42:36 +08:00
|
|
|
while (!DFSStack.empty()) {
|
|
|
|
NodeAndParent NP = DFSStack.back();
|
|
|
|
DCHECK_NE(NP.Node, nullptr);
|
|
|
|
DCHECK_NE(NP.NewNode, nullptr);
|
|
|
|
DFSStack.trim(1);
|
|
|
|
for (const auto Callee : NP.Node->Callees) {
|
|
|
|
auto NewNode = O.Nodes.AppendEmplace(
|
2018-12-07 11:19:13 +08:00
|
|
|
NP.NewNode, NodeIdPairArray(*O.NodeIdPairAllocator),
|
|
|
|
Callee.NodePtr->CallCount, Callee.NodePtr->CumulativeLocalTime,
|
|
|
|
Callee.FId);
|
2018-07-10 16:25:44 +08:00
|
|
|
if (UNLIKELY(NewNode == nullptr))
|
|
|
|
return;
|
[XRay] Use preallocated memory for XRay profiling
Summary:
This change builds upon D54989, which removes memory allocation from the
critical path of the profiling implementation. This also changes the API
for the profile collection service, to take ownership of the memory and
associated data structures per-thread.
The consolidation of the memory allocation allows us to do two things:
- Limits the amount of memory used by the profiling implementation,
associating preallocated buffers instead of allocating memory
on-demand.
- Consolidate the memory initialisation and cleanup by relying on the
buffer queue's reference counting implementation.
We find a number of places which also display some problematic
behaviour, including:
- Off-by-factor bug in the allocator implementation.
- Unrolling semantics in cases of "memory exhausted" situations, when
managing the state of the function call trie.
We also add a few test cases which verify our understanding of the
behaviour of the system, with important edge-cases (especially for
memory-exhausted cases) in the segmented array and profile collector
unit tests.
Depends on D54989.
Reviewers: mboerger
Subscribers: dschuff, mgorny, dmgreen, jfb, llvm-commits
Differential Revision: https://reviews.llvm.org/D55249
llvm-svn: 348568
2018-12-07 14:23:06 +08:00
|
|
|
if (UNLIKELY(NP.NewNode->Callees.AppendEmplace(NewNode, Callee.FId) ==
|
|
|
|
nullptr))
|
|
|
|
return;
|
|
|
|
if (UNLIKELY(DFSStack.AppendEmplace(Callee.NodePtr, NewNode) ==
|
|
|
|
nullptr))
|
|
|
|
return;
|
2018-05-15 08:42:36 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// The mergeInto operation will update the provided FunctionCallTrie by
|
|
|
|
// traversing the current trie's roots and updating (i.e. merging) the data in
|
|
|
|
// the nodes with the data in the target's nodes. If the node doesn't exist in
|
|
|
|
// the provided trie, we add a new one in the right position, and inherit the
|
|
|
|
// data from the original (current) trie, along with all its callees.
|
|
|
|
//
|
|
|
|
// This function is *not* thread-safe, and may require external
|
|
|
|
// synchronisation of both "this" and |O|.
|
2018-09-07 18:16:14 +08:00
|
|
|
void mergeInto(FunctionCallTrie &O) const XRAY_NEVER_INSTRUMENT {
|
2018-05-15 08:42:36 +08:00
|
|
|
struct NodeAndTarget {
|
|
|
|
FunctionCallTrie::Node *OrigNode;
|
|
|
|
FunctionCallTrie::Node *TargetNode;
|
|
|
|
};
|
|
|
|
using Stack = Array<NodeAndTarget>;
|
|
|
|
typename Stack::AllocatorType StackAllocator(
|
2018-07-18 09:53:39 +08:00
|
|
|
profilingFlags()->stack_allocator_max);
|
2018-07-18 10:08:39 +08:00
|
|
|
Stack DFSStack(StackAllocator);
|
2018-05-15 08:42:36 +08:00
|
|
|
|
|
|
|
for (const auto Root : getRoots()) {
|
|
|
|
Node *TargetRoot = nullptr;
|
|
|
|
auto R = O.Roots.find_element(
|
|
|
|
[&](const Node *Node) { return Node->FId == Root->FId; });
|
|
|
|
if (R == nullptr) {
|
2018-12-07 11:19:13 +08:00
|
|
|
TargetRoot = O.Nodes.AppendEmplace(
|
|
|
|
nullptr, NodeIdPairArray(*O.NodeIdPairAllocator), 0u, 0u,
|
|
|
|
Root->FId);
|
2018-07-10 16:25:44 +08:00
|
|
|
if (UNLIKELY(TargetRoot == nullptr))
|
|
|
|
return;
|
|
|
|
|
2018-05-15 08:42:36 +08:00
|
|
|
O.Roots.Append(TargetRoot);
|
|
|
|
} else {
|
|
|
|
TargetRoot = *R;
|
|
|
|
}
|
|
|
|
|
2018-12-07 11:19:13 +08:00
|
|
|
DFSStack.AppendEmplace(Root, TargetRoot);
|
2018-05-15 08:42:36 +08:00
|
|
|
while (!DFSStack.empty()) {
|
|
|
|
NodeAndTarget NT = DFSStack.back();
|
|
|
|
DCHECK_NE(NT.OrigNode, nullptr);
|
|
|
|
DCHECK_NE(NT.TargetNode, nullptr);
|
|
|
|
DFSStack.trim(1);
|
|
|
|
// TODO: Update the histogram as well when we have it ready.
|
|
|
|
NT.TargetNode->CallCount += NT.OrigNode->CallCount;
|
|
|
|
NT.TargetNode->CumulativeLocalTime += NT.OrigNode->CumulativeLocalTime;
|
|
|
|
for (const auto Callee : NT.OrigNode->Callees) {
|
|
|
|
auto TargetCallee = NT.TargetNode->Callees.find_element(
|
|
|
|
[&](const FunctionCallTrie::NodeIdPair &C) {
|
|
|
|
return C.FId == Callee.FId;
|
|
|
|
});
|
|
|
|
if (TargetCallee == nullptr) {
|
2018-07-18 10:08:39 +08:00
|
|
|
auto NewTargetNode = O.Nodes.AppendEmplace(
|
2018-12-07 11:19:13 +08:00
|
|
|
NT.TargetNode, NodeIdPairArray(*O.NodeIdPairAllocator), 0u, 0u,
|
|
|
|
Callee.FId);
|
2018-07-10 16:25:44 +08:00
|
|
|
|
|
|
|
if (UNLIKELY(NewTargetNode == nullptr))
|
|
|
|
return;
|
|
|
|
|
2018-05-15 08:42:36 +08:00
|
|
|
TargetCallee =
|
|
|
|
NT.TargetNode->Callees.AppendEmplace(NewTargetNode, Callee.FId);
|
|
|
|
}
|
2018-07-10 16:25:44 +08:00
|
|
|
DFSStack.AppendEmplace(Callee.NodePtr, TargetCallee->NodePtr);
|
2018-05-15 08:42:36 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
} // namespace __xray
|
|
|
|
|
|
|
|
#endif // XRAY_FUNCTION_CALL_TRIE_H
|