forked from OSchip/llvm-project
[BOLT] Introduce non-LBR mode
Summary: Add support to read profiles collected without LBR. This involves adapting our data aggregator perf2bolt and adding support in llvm-bolt itself to read this data. This patch also introduces different options to convert basic block execution count to edge count, so BOLT can operate with its regular algorithms to perform basic block layout. The most successful approach is the default one. (cherry picked from FBD5664735)
This commit is contained in:
Rafael Auler
Maksim Panchenko
parent
29d4f4cfac
commit
9df155ce11
|
|
@ -373,6 +373,16 @@ void BinaryBasicBlock::addTailCallInstruction(const MCSymbol *Target) {
|
|||
Instructions.emplace_back(std::move(NewInst));
|
||||
}
|
||||
|
||||
uint32_t BinaryBasicBlock::getNumCalls() const {
|
||||
uint32_t N{0};
|
||||
auto &BC = Function->getBinaryContext();
|
||||
for (auto &Instr : Instructions) {
|
||||
if (BC.MIA->isCall(Instr))
|
||||
++N;
|
||||
}
|
||||
return N;
|
||||
}
|
||||
|
||||
uint32_t BinaryBasicBlock::getNumPseudos() const {
|
||||
#ifndef NDEBUG
|
||||
auto &BC = Function->getBinaryContext();
|
||||
|
|
|
|||
|
|
@ -349,6 +349,17 @@ public:
|
|||
return BranchInfo[Condition == true ? 0 : 1];
|
||||
};
|
||||
|
||||
BinaryBranchInfo &getBranchInfo(const BinaryBasicBlock &Succ) {
|
||||
auto BI = branch_info_begin();
|
||||
for (auto BB : successors()) {
|
||||
if (&Succ == BB)
|
||||
return *BI;
|
||||
++BI;
|
||||
}
|
||||
llvm_unreachable("Invalid successor");
|
||||
return *BI;
|
||||
}
|
||||
|
||||
/// Try to compute the taken and misprediction frequencies for the given
|
||||
/// successor. The result is an error if no information can be found.
|
||||
ErrorOr<std::pair<double, double>>
|
||||
|
|
@ -368,6 +379,9 @@ public:
|
|||
/// to the end of this basic block.
|
||||
void addTailCallInstruction(const MCSymbol *Target);
|
||||
|
||||
/// Return the number of call instructions in this basic block.
|
||||
uint32_t getNumCalls() const;
|
||||
|
||||
/// Get landing pad with given label. Returns nullptr if no such
|
||||
/// landing pad is found.
|
||||
BinaryBasicBlock *getLandingPad(const MCSymbol *Label) const;
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
#include "BinaryBasicBlock.h"
|
||||
#include "BinaryFunction.h"
|
||||
#include "DataReader.h"
|
||||
#include "Passes/MCF.h"
|
||||
#include "Passes/ReorderAlgorithm.h"
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
#include "llvm/DebugInfo/DWARF/DWARFContext.h"
|
||||
|
|
@ -129,6 +130,27 @@ SplitEH("split-eh",
|
|||
cl::Hidden,
|
||||
cl::cat(BoltOptCategory));
|
||||
|
||||
cl::opt<MCFCostFunction>
|
||||
DoMCF("mcf",
|
||||
cl::desc("solve a min cost flow problem on the CFG to fix edge counts "
|
||||
"(default=disable)"),
|
||||
cl::init(MCF_DISABLE),
|
||||
cl::values(
|
||||
clEnumValN(MCF_DISABLE, "none",
|
||||
"disable MCF"),
|
||||
clEnumValN(MCF_LINEAR, "linear",
|
||||
"cost function is inversely proportional to edge count"),
|
||||
clEnumValN(MCF_QUADRATIC, "quadratic",
|
||||
"cost function is inversely proportional to edge count squared"),
|
||||
clEnumValN(MCF_LOG, "log",
|
||||
"cost function is inversely proportional to log of edge count"),
|
||||
clEnumValN(MCF_BLAMEFTS, "blamefts",
|
||||
"tune cost to blame fall-through edges for surplus flow"),
|
||||
clEnumValEnd),
|
||||
cl::ZeroOrMore,
|
||||
cl::Hidden,
|
||||
cl::cat(BoltOptCategory));
|
||||
|
||||
bool shouldPrint(const BinaryFunction &Function) {
|
||||
if (PrintOnly.empty())
|
||||
return true;
|
||||
|
|
@ -1839,10 +1861,15 @@ bool BinaryFunction::buildCFG() {
|
|||
addLandingPads(0, BasicBlocks.size());
|
||||
|
||||
// Infer frequency for non-taken branches
|
||||
if (hasValidProfile())
|
||||
if (hasValidProfile() && opts::DoMCF != MCF_DISABLE) {
|
||||
// Convert COUNT_NO_PROFILE to 0
|
||||
removeTagsFromProfile();
|
||||
solveMCF(*this, opts::DoMCF);
|
||||
} else if (hasValidProfile()) {
|
||||
inferFallThroughCounts();
|
||||
else
|
||||
} else {
|
||||
clearProfile();
|
||||
}
|
||||
|
||||
// Assign CFI information to each BB entry.
|
||||
annotateCFIState();
|
||||
|
|
@ -1875,6 +1902,14 @@ bool BinaryFunction::buildCFG() {
|
|||
// Eliminate inconsistencies between branch instructions and CFG.
|
||||
postProcessBranches();
|
||||
|
||||
// If our profiling data comes from samples instead of LBR entries,
|
||||
// now is the time to read this data and attach it to BBs. At this point,
|
||||
// conditional tail calls are converted into a branch and a new basic block,
|
||||
// making it slightly different than the original binary where profiled data
|
||||
// was collected. However, this shouldn't matter for plain sampling events.
|
||||
if (!BC.DR.hasLBR())
|
||||
readSampleData();
|
||||
|
||||
// Clean-up memory taken by instructions and labels.
|
||||
//
|
||||
// NB: don't clear Labels list as we may need them if we mark the function
|
||||
|
|
@ -1900,6 +1935,71 @@ bool BinaryFunction::buildCFG() {
|
|||
return true;
|
||||
}
|
||||
|
||||
void BinaryFunction::removeTagsFromProfile() {
|
||||
for (auto *BB : BasicBlocks) {
|
||||
if (BB->ExecutionCount == BinaryBasicBlock::COUNT_NO_PROFILE)
|
||||
BB->ExecutionCount = 0;
|
||||
for (auto &BI : BB->branch_info()) {
|
||||
if (BI.Count != BinaryBasicBlock::COUNT_NO_PROFILE &&
|
||||
BI.MispredictedCount != BinaryBasicBlock::COUNT_NO_PROFILE)
|
||||
continue;
|
||||
BI.Count = 0;
|
||||
BI.MispredictedCount = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void BinaryFunction::readSampleData() {
|
||||
auto SampleDataOrErr = BC.DR.getFuncSampleData(getNames());
|
||||
|
||||
if (!SampleDataOrErr)
|
||||
return;
|
||||
|
||||
// Non-LBR mode territory
|
||||
// First step is to assign BB execution count based on samples from perf
|
||||
ProfileMatchRatio = 1.0f;
|
||||
removeTagsFromProfile();
|
||||
bool NormalizeByInsnCount =
|
||||
BC.DR.usesEvent("cycles") || BC.DR.usesEvent("instructions");
|
||||
bool NormalizeByCalls = BC.DR.usesEvent("branches");
|
||||
static bool NagUser{true};
|
||||
if (NagUser) {
|
||||
outs() << "BOLT-INFO: operating with non-LBR profiling data.\n";
|
||||
if (NormalizeByInsnCount) {
|
||||
outs() << "BOLT-INFO: normalizing samples by instruction count.\n";
|
||||
} else if (NormalizeByCalls) {
|
||||
outs() << "BOLT-INFO: normalizing samples by branches.\n";
|
||||
}
|
||||
NagUser = false;
|
||||
}
|
||||
uint64_t LastOffset = getSize();
|
||||
uint64_t TotalEntryCount{0};
|
||||
for (auto I = BasicBlockOffsets.rbegin(), E = BasicBlockOffsets.rend();
|
||||
I != E; ++I) {
|
||||
uint64_t CurOffset = I->first;
|
||||
// Always work with samples multiplied by 1000 to avoid losing them if we
|
||||
// later need to normalize numbers
|
||||
uint64_t NumSamples =
|
||||
SampleDataOrErr->getSamples(CurOffset, LastOffset) * 1000;
|
||||
if (NormalizeByInsnCount && I->second->getNumNonPseudos())
|
||||
NumSamples /= I->second->getNumNonPseudos();
|
||||
else if (NormalizeByCalls) {
|
||||
uint32_t NumCalls = I->second->getNumCalls();
|
||||
NumSamples /= NumCalls + 1;
|
||||
}
|
||||
I->second->setExecutionCount(NumSamples);
|
||||
if (I->second->isEntryPoint())
|
||||
TotalEntryCount += NumSamples;
|
||||
LastOffset = CurOffset;
|
||||
}
|
||||
ExecutionCount = TotalEntryCount;
|
||||
|
||||
estimateEdgeCounts(BC, *this);
|
||||
|
||||
if (opts::DoMCF != MCF_DISABLE)
|
||||
solveMCF(*this, opts::DoMCF);
|
||||
}
|
||||
|
||||
void BinaryFunction::addEntryPoint(uint64_t Address) {
|
||||
assert(containsAddress(Address) && "address does not belong to the function");
|
||||
|
||||
|
|
@ -1987,6 +2087,12 @@ bool BinaryFunction::fetchProfileForOtherEntryPoints() {
|
|||
}
|
||||
|
||||
void BinaryFunction::matchProfileData() {
|
||||
// This functionality is available for LBR-mode only
|
||||
// TODO: Implement evaluateProfileData() for samples, checking whether
|
||||
// sample addresses match instruction addresses in the function
|
||||
if (!BC.DR.hasLBR())
|
||||
return;
|
||||
|
||||
if (BranchData) {
|
||||
ProfileMatchRatio = evaluateProfileData(*BranchData);
|
||||
if (ProfileMatchRatio == 1.0f) {
|
||||
|
|
|
|||
|
|
@ -1763,6 +1763,18 @@ public:
|
|||
/// using profile information.
|
||||
void removeConditionalTailCalls();
|
||||
|
||||
// Convert COUNT_NO_PROFILE to 0
|
||||
void removeTagsFromProfile();
|
||||
|
||||
/// If our profile data comes from sample addresses instead of LBR entries,
|
||||
/// collect sample count for all addresses in this function address space,
|
||||
/// aggregating them per basic block and assigning an execution count to each
|
||||
/// basic block based on the number of samples recorded at those addresses.
|
||||
/// The last step is to infer edge counts based on BB execution count. Note
|
||||
/// this is the opposite of the LBR way, where we infer BB execution count
|
||||
/// based on edge counts.
|
||||
void readSampleData();
|
||||
|
||||
/// Computes a function hotness score: the sum of the products of BB frequency
|
||||
/// and size.
|
||||
uint64_t getFunctionScore();
|
||||
|
|
@ -2038,9 +2050,11 @@ template <> struct GraphTraits<bolt::BinaryFunction *> :
|
|||
|
||||
typedef bolt::BinaryBasicBlock * nodes_iterator;
|
||||
static nodes_iterator nodes_begin(bolt::BinaryFunction *F) {
|
||||
llvm_unreachable("Not implemented");
|
||||
return &(*F->begin());
|
||||
}
|
||||
static nodes_iterator nodes_end(bolt::BinaryFunction *F) {
|
||||
llvm_unreachable("Not implemented");
|
||||
return &(*F->end());
|
||||
}
|
||||
static size_t size(bolt::BinaryFunction *F) {
|
||||
|
|
@ -2056,9 +2070,11 @@ template <> struct GraphTraits<const bolt::BinaryFunction *> :
|
|||
|
||||
typedef const bolt::BinaryBasicBlock * nodes_iterator;
|
||||
static nodes_iterator nodes_begin(const bolt::BinaryFunction *F) {
|
||||
llvm_unreachable("Not implemented");
|
||||
return &(*F->begin());
|
||||
}
|
||||
static nodes_iterator nodes_end(const bolt::BinaryFunction *F) {
|
||||
llvm_unreachable("Not implemented");
|
||||
return &(*F->end());
|
||||
}
|
||||
static size_t size(const bolt::BinaryFunction *F) {
|
||||
|
|
|
|||
|
|
@ -31,9 +31,10 @@ private:
|
|||
std::set<uint64_t> &LargeFunctions;
|
||||
std::vector<std::pair<const bool,
|
||||
std::unique_ptr<BinaryFunctionPass>>> Passes;
|
||||
static const char TimerGroupName[];
|
||||
|
||||
public:
|
||||
static const char TimerGroupName[];
|
||||
|
||||
BinaryFunctionPassManager(BinaryContext &BC,
|
||||
std::map<uint64_t, BinaryFunction> &BFs,
|
||||
std::set<uint64_t> &LargeFunctions)
|
||||
|
|
|
|||
|
|
@ -78,6 +78,36 @@ void FuncBranchData::appendFrom(const FuncBranchData &FBD, uint64_t Offset) {
|
|||
}
|
||||
}
|
||||
|
||||
void SampleInfo::mergeWith(const SampleInfo &SI) {
|
||||
Occurrences += SI.Occurrences;
|
||||
}
|
||||
|
||||
void SampleInfo::print(raw_ostream &OS) const {
|
||||
OS << Address.IsSymbol << " " << Address.Name << " "
|
||||
<< Twine::utohexstr(Address.Offset) << " "
|
||||
<< Occurrences << "\n";
|
||||
}
|
||||
|
||||
uint64_t
|
||||
FuncSampleData::getSamples(uint64_t Start, uint64_t End) const {
|
||||
assert(std::is_sorted(Data.begin(), Data.end()));
|
||||
struct Compare {
|
||||
bool operator()(const SampleInfo &SI, const uint64_t Val) const {
|
||||
return SI.Address.Offset < Val;
|
||||
}
|
||||
bool operator()(const uint64_t Val, const SampleInfo &SI) const {
|
||||
return Val < SI.Address.Offset;
|
||||
}
|
||||
};
|
||||
uint64_t Result{0};
|
||||
for (auto I = std::lower_bound(Data.begin(), Data.end(), Start, Compare()),
|
||||
E = std::lower_bound(Data.begin(), Data.end(), End, Compare());
|
||||
I != E; ++I) {
|
||||
Result += I->Occurrences;
|
||||
}
|
||||
return Result;
|
||||
}
|
||||
|
||||
void BranchInfo::mergeWith(const BranchInfo &BI) {
|
||||
|
||||
// Merge branch and misprediction counts.
|
||||
|
|
@ -406,6 +436,48 @@ ErrorOr<BranchInfo> DataReader::parseBranchInfo() {
|
|||
std::move(Histories));
|
||||
}
|
||||
|
||||
ErrorOr<SampleInfo> DataReader::parseSampleInfo() {
|
||||
auto Res = parseLocation(FieldSeparator);
|
||||
if (std::error_code EC = Res.getError())
|
||||
return EC;
|
||||
Location Address = Res.get();
|
||||
|
||||
auto BRes = parseNumberField(FieldSeparator, /* EndNl = */ true);
|
||||
if (std::error_code EC = BRes.getError())
|
||||
return EC;
|
||||
int64_t Occurrences = BRes.get();
|
||||
|
||||
if (!checkAndConsumeNewLine()) {
|
||||
reportError("expected end of line");
|
||||
return make_error_code(llvm::errc::io_error);
|
||||
}
|
||||
|
||||
return SampleInfo(std::move(Address), Occurrences);
|
||||
}
|
||||
|
||||
ErrorOr<bool> DataReader::maybeParseNoLBRFlag() {
|
||||
if (ParsingBuf.size() < 6 || ParsingBuf.substr(0, 6) != "no_lbr")
|
||||
return false;
|
||||
ParsingBuf = ParsingBuf.drop_front(6);
|
||||
Col += 6;
|
||||
|
||||
if (ParsingBuf.size() > 0 && ParsingBuf[0] == ' ')
|
||||
ParsingBuf = ParsingBuf.drop_front(1);
|
||||
|
||||
while (ParsingBuf.size() > 0 && ParsingBuf[0] != '\n') {
|
||||
auto EventName = parseString(' ', true);
|
||||
if (!EventName)
|
||||
return make_error_code(llvm::errc::io_error);
|
||||
EventNames.insert(EventName.get());
|
||||
}
|
||||
|
||||
if (!checkAndConsumeNewLine()) {
|
||||
reportError("malformed no_lbr line");
|
||||
return make_error_code(llvm::errc::io_error);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DataReader::hasData() {
|
||||
if (ParsingBuf.size() == 0)
|
||||
return false;
|
||||
|
|
@ -415,12 +487,48 @@ bool DataReader::hasData() {
|
|||
return false;
|
||||
}
|
||||
|
||||
std::error_code DataReader::parseInNoLBRMode() {
|
||||
auto GetOrCreateFuncEntry = [&](StringRef Name) {
|
||||
auto I = FuncsToSamples.find(Name);
|
||||
if (I == FuncsToSamples.end()) {
|
||||
bool success;
|
||||
std::tie(I, success) = FuncsToSamples.insert(std::make_pair(
|
||||
Name, FuncSampleData(Name, FuncSampleData::ContainerTy())));
|
||||
|
||||
assert(success && "unexpected result of insert");
|
||||
}
|
||||
return I;
|
||||
};
|
||||
|
||||
while (hasData()) {
|
||||
auto Res = parseSampleInfo();
|
||||
if (std::error_code EC = Res.getError())
|
||||
return EC;
|
||||
|
||||
SampleInfo SI = Res.get();
|
||||
|
||||
// Ignore samples not involving known locations
|
||||
if (!SI.Address.IsSymbol)
|
||||
continue;
|
||||
|
||||
auto I = GetOrCreateFuncEntry(SI.Address.Name);
|
||||
I->getValue().Data.emplace_back(std::move(SI));
|
||||
}
|
||||
|
||||
for (auto &FuncSamples : FuncsToSamples) {
|
||||
std::stable_sort(FuncSamples.second.Data.begin(),
|
||||
FuncSamples.second.Data.end());
|
||||
}
|
||||
|
||||
return std::error_code();
|
||||
}
|
||||
|
||||
std::error_code DataReader::parse() {
|
||||
auto GetOrCreateFuncEntry = [&](StringRef Name) {
|
||||
auto I = FuncsMap.find(Name);
|
||||
if (I == FuncsMap.end()) {
|
||||
auto I = FuncsToBranches.find(Name);
|
||||
if (I == FuncsToBranches.end()) {
|
||||
bool success;
|
||||
std::tie(I, success) = FuncsMap.insert(
|
||||
std::tie(I, success) = FuncsToBranches.insert(
|
||||
std::make_pair(Name, FuncBranchData(Name,
|
||||
FuncBranchData::ContainerTy(),
|
||||
FuncBranchData::ContainerTy())));
|
||||
|
|
@ -431,6 +539,13 @@ std::error_code DataReader::parse() {
|
|||
|
||||
Col = 0;
|
||||
Line = 1;
|
||||
auto FlagOrErr = maybeParseNoLBRFlag();
|
||||
if (!FlagOrErr)
|
||||
return FlagOrErr.getError();
|
||||
NoLBRMode = *FlagOrErr;
|
||||
if (NoLBRMode)
|
||||
return parseInNoLBRMode();
|
||||
|
||||
while (hasData()) {
|
||||
auto Res = parseBranchInfo();
|
||||
if (std::error_code EC = Res.getError())
|
||||
|
|
@ -462,7 +577,7 @@ std::error_code DataReader::parse() {
|
|||
}
|
||||
}
|
||||
|
||||
for (auto &FuncBranches : FuncsMap) {
|
||||
for (auto &FuncBranches : FuncsToBranches) {
|
||||
std::stable_sort(FuncBranches.second.Data.begin(),
|
||||
FuncBranches.second.Data.end());
|
||||
}
|
||||
|
|
@ -471,7 +586,7 @@ std::error_code DataReader::parse() {
|
|||
}
|
||||
|
||||
void DataReader::buildLTONameMap() {
|
||||
for (auto &FuncData : FuncsMap) {
|
||||
for (auto &FuncData : FuncsToBranches) {
|
||||
const auto FuncName = FuncData.getKey();
|
||||
const auto CommonName = getLTOCommonName(FuncName);
|
||||
if (CommonName)
|
||||
|
|
@ -479,17 +594,30 @@ void DataReader::buildLTONameMap() {
|
|||
}
|
||||
}
|
||||
|
||||
FuncBranchData *
|
||||
DataReader::getFuncBranchData(const std::vector<std::string> &FuncNames) {
|
||||
namespace {
|
||||
template <typename MapTy>
|
||||
decltype(MapTy::MapEntryTy::second) *
|
||||
fetchMapEntry(MapTy &Map, const std::vector<std::string> &FuncNames) {
|
||||
// Do a reverse order iteration since the name in profile has a higher chance
|
||||
// of matching a name at the end of the list.
|
||||
for (auto FI = FuncNames.rbegin(), FE = FuncNames.rend(); FI != FE; ++FI) {
|
||||
auto I = FuncsMap.find(normalizeName(*FI));
|
||||
if (I != FuncsMap.end())
|
||||
auto I = Map.find(normalizeName(*FI));
|
||||
if (I != Map.end())
|
||||
return &I->getValue();
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
FuncBranchData *
|
||||
DataReader::getFuncBranchData(const std::vector<std::string> &FuncNames) {
|
||||
return fetchMapEntry<FuncsToBranchesMapTy>(FuncsToBranches, FuncNames);
|
||||
}
|
||||
|
||||
FuncSampleData *
|
||||
DataReader::getFuncSampleData(const std::vector<std::string> &FuncNames) {
|
||||
return fetchMapEntry<FuncsToSamplesMapTy>(FuncsToSamples, FuncNames);
|
||||
}
|
||||
|
||||
std::vector<FuncBranchData *>
|
||||
DataReader::getFuncBranchDataRegex(const std::vector<std::string> &FuncNames) {
|
||||
|
|
@ -507,8 +635,8 @@ DataReader::getFuncBranchDataRegex(const std::vector<std::string> &FuncNames) {
|
|||
AllData.insert(AllData.end(), CommonData.begin(), CommonData.end());
|
||||
}
|
||||
} else {
|
||||
auto I = FuncsMap.find(Name);
|
||||
if (I != FuncsMap.end()) {
|
||||
auto I = FuncsToBranches.find(Name);
|
||||
if (I != FuncsToBranches.end()) {
|
||||
return {&I->getValue()};
|
||||
}
|
||||
}
|
||||
|
|
@ -517,7 +645,7 @@ DataReader::getFuncBranchDataRegex(const std::vector<std::string> &FuncNames) {
|
|||
}
|
||||
|
||||
bool DataReader::hasLocalsWithFileName() const {
|
||||
for (const auto &Func : FuncsMap) {
|
||||
for (const auto &Func : FuncsToBranches) {
|
||||
const auto &FuncName = Func.getKey();
|
||||
if (FuncName.count('/') == 2 && FuncName[0] != '/')
|
||||
return true;
|
||||
|
|
@ -526,7 +654,7 @@ bool DataReader::hasLocalsWithFileName() const {
|
|||
}
|
||||
|
||||
void DataReader::dump() const {
|
||||
for (const auto &Func : FuncsMap) {
|
||||
for (const auto &Func : FuncsToBranches) {
|
||||
Diag << Func.getKey() << " branches:\n";
|
||||
for (const auto &BI : Func.getValue().Data) {
|
||||
Diag << BI.From.Name << " " << BI.From.Offset << " " << BI.To.Name << " "
|
||||
|
|
@ -552,6 +680,18 @@ void DataReader::dump() const {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (auto I = EventNames.begin(), E = EventNames.end(); I != E; ++I) {
|
||||
StringRef Event = I->getKey();
|
||||
Diag << "Data was collected with event: " << Event << "\n";
|
||||
}
|
||||
for (const auto &Func : FuncsToSamples) {
|
||||
Diag << Func.getKey() << " samples:\n";
|
||||
for (const auto &SI : Func.getValue().Data) {
|
||||
Diag << SI.Address.Name << " " << SI.Address.Offset << " "
|
||||
<< SI.Occurrences << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace bolt
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
#include "llvm/ADT/Optional.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/StringMap.h"
|
||||
#include "llvm/ADT/StringSet.h"
|
||||
#include "llvm/Support/Allocator.h"
|
||||
#include "llvm/Support/Errc.h"
|
||||
#include "llvm/Support/ErrorOr.h"
|
||||
|
|
@ -163,6 +164,47 @@ struct FuncBranchData {
|
|||
void appendFrom(const FuncBranchData &FBD, uint64_t Offset);
|
||||
};
|
||||
|
||||
/// Similar to BranchInfo, but instead of recording from-to address (an edge),
|
||||
/// it records the address of a perf event and the number of times samples hit
|
||||
/// this address.
|
||||
struct SampleInfo {
|
||||
Location Address; // FIXME: Change this name to Loc
|
||||
int64_t Occurrences; // FIXME: Variable name is horrible
|
||||
|
||||
SampleInfo(Location Address, int64_t Occurrences)
|
||||
: Address(std::move(Address)), Occurrences(Occurrences) {}
|
||||
|
||||
bool operator==(const SampleInfo &RHS) const {
|
||||
return Address == RHS.Address;
|
||||
}
|
||||
|
||||
bool operator<(const SampleInfo &RHS) const {
|
||||
if (Address < RHS.Address)
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void print(raw_ostream &OS) const;
|
||||
|
||||
void mergeWith(const SampleInfo &SI);
|
||||
};
|
||||
|
||||
/// Helper class to store samples recorded in the address space of a given
|
||||
/// function, analogous to FuncBranchData but for samples instead of branches.
|
||||
struct FuncSampleData {
|
||||
typedef std::vector<SampleInfo> ContainerTy;
|
||||
|
||||
StringRef Name;
|
||||
ContainerTy Data;
|
||||
|
||||
FuncSampleData(StringRef Name, ContainerTy Data)
|
||||
: Name(Name), Data(std::move(Data)) {}
|
||||
|
||||
/// Get the number of samples recorded in [Start, End)
|
||||
uint64_t getSamples(uint64_t Start, uint64_t End) const;
|
||||
};
|
||||
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
/// DataReader Class
|
||||
|
|
@ -222,21 +264,45 @@ public:
|
|||
/// offset d.
|
||||
std::error_code parse();
|
||||
|
||||
/// When no_lbr is the first line of the file, activate No LBR mode. In this
|
||||
/// mode we read the addresses where samples were recorded directly instead of
|
||||
/// LBR entries. The line format is almost the same, except for a missing <to>
|
||||
/// triple and a missing mispredictions field:
|
||||
///
|
||||
/// no_lbr
|
||||
/// <is symbol?> <closest elf symbol or DSO name> <relative address> <count>
|
||||
/// ...
|
||||
///
|
||||
/// Example:
|
||||
///
|
||||
/// no_lbr # First line of fdata file
|
||||
/// 1 BZ2_compressBlock 466c 3
|
||||
/// 1 BZ2_hbMakeCodeLengths 29c 1
|
||||
///
|
||||
std::error_code parseInNoLBRMode();
|
||||
|
||||
/// Return branch data matching one of the names in \p FuncNames.
|
||||
FuncBranchData *
|
||||
getFuncBranchData(const std::vector<std::string> &FuncNames);
|
||||
|
||||
FuncSampleData *
|
||||
getFuncSampleData(const std::vector<std::string> &FuncNames);
|
||||
|
||||
/// Return a vector of all FuncBranchData matching the list of names.
|
||||
/// Internally use fuzzy matching to match special names like LTO-generated
|
||||
/// function names.
|
||||
std::vector<FuncBranchData *>
|
||||
getFuncBranchDataRegex(const std::vector<std::string> &FuncNames);
|
||||
|
||||
using FuncsMapType = StringMap<FuncBranchData>;
|
||||
using FuncsToBranchesMapTy = StringMap<FuncBranchData>;
|
||||
using FuncsToSamplesMapTy = StringMap<FuncSampleData>;
|
||||
|
||||
FuncsMapType &getAllFuncsData() { return FuncsMap; }
|
||||
FuncsToBranchesMapTy &getAllFuncsBranchData() { return FuncsToBranches; }
|
||||
FuncsToSamplesMapTy &getAllFuncsSampleData() { return FuncsToSamples; }
|
||||
|
||||
const FuncsMapType &getAllFuncsData() const { return FuncsMap; }
|
||||
const FuncsToBranchesMapTy &getAllFuncsData() const {
|
||||
return FuncsToBranches;
|
||||
}
|
||||
|
||||
/// Return true if profile contains an entry for a local function
|
||||
/// that has a non-empty associated file name.
|
||||
|
|
@ -245,6 +311,24 @@ public:
|
|||
/// Dumps the entire data structures parsed. Used for debugging.
|
||||
void dump() const;
|
||||
|
||||
/// Return false only if we are running with profiling data that lacks LBR.
|
||||
bool hasLBR() const { return !NoLBRMode; }
|
||||
|
||||
/// Return true if event named \p Name was used to collect this profile data.
|
||||
bool usesEvent(StringRef Name) const {
|
||||
for (auto I = EventNames.begin(), E = EventNames.end(); I != E; ++I) {
|
||||
StringRef Event = I->getKey();
|
||||
if (Event.find(Name) != StringRef::npos)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Return all event names used to collect this profile
|
||||
const StringSet<> &getEventNames() const {
|
||||
return EventNames;
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
void reportError(StringRef ErrorMsg);
|
||||
|
|
@ -255,6 +339,8 @@ private:
|
|||
ErrorOr<Location> parseLocation(char EndChar, bool EndNl=false);
|
||||
ErrorOr<BranchHistory> parseBranchHistory();
|
||||
ErrorOr<BranchInfo> parseBranchInfo();
|
||||
ErrorOr<SampleInfo> parseSampleInfo();
|
||||
ErrorOr<bool> maybeParseNoLBRFlag();
|
||||
bool hasData();
|
||||
|
||||
/// Build suffix map once the profile data is parsed.
|
||||
|
|
@ -266,7 +352,10 @@ private:
|
|||
StringRef ParsingBuf;
|
||||
unsigned Line;
|
||||
unsigned Col;
|
||||
FuncsMapType FuncsMap;
|
||||
FuncsToBranchesMapTy FuncsToBranches;
|
||||
FuncsToSamplesMapTy FuncsToSamples;
|
||||
bool NoLBRMode;
|
||||
StringSet<> EventNames;
|
||||
static const char FieldSeparator = ' ';
|
||||
|
||||
/// Map of common LTO names to possible matching profiles.
|
||||
|
|
|
|||
|
|
@ -13,8 +13,9 @@ add_llvm_library(LLVMBOLTPasses
|
|||
IndirectCallPromotion.cpp
|
||||
Inliner.cpp
|
||||
LivenessAnalysis.cpp
|
||||
PLTCall.cpp
|
||||
MCF.cpp
|
||||
PettisAndHansen.cpp
|
||||
PLTCall.cpp
|
||||
RegAnalysis.cpp
|
||||
ReorderAlgorithm.cpp
|
||||
ReorderFunctions.cpp
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "../DataReader.h"
|
||||
#include "llvm/ADT/StringSet.h"
|
||||
#include "llvm/Object/Binary.h"
|
||||
#include "llvm/Support/CommandLine.h"
|
||||
#include "llvm/Support/PrettyStackTrace.h"
|
||||
|
|
@ -91,7 +92,9 @@ int main(int argc, char **argv) {
|
|||
ToolName = argv[0];
|
||||
|
||||
// All merged data.
|
||||
DataReader::FuncsMapType MergedFunctionsData;
|
||||
DataReader::FuncsToBranchesMapTy MergedFunctionsBranchData;
|
||||
DataReader::FuncsToSamplesMapTy MergedFunctionsSampleData;
|
||||
StringSet<> EventNames;
|
||||
|
||||
// Merged functions data has to replace strings refs with strings from the
|
||||
// pool.
|
||||
|
|
@ -140,9 +143,20 @@ int main(int argc, char **argv) {
|
|||
AllStrings.emplace_back(ToNamePtr); // keep the reference
|
||||
};
|
||||
|
||||
auto CopySampleInfo = [&](const SampleInfo &SI,
|
||||
std::vector<SampleInfo> &SIData) {
|
||||
auto NamePtr = MergedStringPool.intern(SI.Address.Name);
|
||||
BranchHistories Histories;
|
||||
SIData.emplace_back(SampleInfo(Location(SI.Address.IsSymbol,
|
||||
*NamePtr,
|
||||
SI.Address.Offset),
|
||||
SI.Occurrences));
|
||||
AllStrings.emplace_back(NamePtr); // keep the reference
|
||||
};
|
||||
|
||||
// Simply replace string references in BranchInfo with internal storage
|
||||
// references.
|
||||
auto replaceStringRefs = [&] (BranchInfo &BI) {
|
||||
auto replaceBIStringRefs = [&] (BranchInfo &BI) {
|
||||
auto FromNamePtr = MergedStringPool.intern(BI.From.Name);
|
||||
BI.From.Name = *FromNamePtr;
|
||||
AllStrings.emplace_back(FromNamePtr); // keep the reference
|
||||
|
|
@ -163,6 +177,12 @@ int main(int argc, char **argv) {
|
|||
}
|
||||
};
|
||||
|
||||
auto replaceSIStringRefs = [&] (SampleInfo &SI) {
|
||||
auto NamePtr = MergedStringPool.intern(SI.Address.Name);
|
||||
SI.Address.Name = *NamePtr;
|
||||
AllStrings.emplace_back(NamePtr); // keep the reference
|
||||
};
|
||||
|
||||
for (auto &InputDataFilename : opts::InputDataFilenames) {
|
||||
if (!sys::fs::exists(InputDataFilename))
|
||||
report_error(InputDataFilename, errc::no_such_file_or_directory);
|
||||
|
|
@ -175,9 +195,17 @@ int main(int argc, char **argv) {
|
|||
if (std::error_code EC = ReaderOrErr.getError())
|
||||
report_error(InputDataFilename, EC);
|
||||
|
||||
for (auto &FI : ReaderOrErr.get()->getAllFuncsData()) {
|
||||
auto MI = MergedFunctionsData.find(FI.second.Name);
|
||||
if (MI != MergedFunctionsData.end()) {
|
||||
if ((ReaderOrErr.get()->hasLBR() && MergedFunctionsSampleData.size() > 0) ||
|
||||
(!ReaderOrErr.get()->hasLBR() &&
|
||||
MergedFunctionsBranchData.size() > 0)) {
|
||||
errs() << "Cannot merge LBR profile with non-LBR "
|
||||
"profile\n";
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
for (auto &FI : ReaderOrErr.get()->getAllFuncsBranchData()) {
|
||||
auto MI = MergedFunctionsBranchData.find(FI.second.Name);
|
||||
if (MI != MergedFunctionsBranchData.end()) {
|
||||
MI->second.ExecutionCount += FI.second.ExecutionCount;
|
||||
std::vector<BranchInfo> TmpBI;
|
||||
for (auto &BI : FI.second.Data) {
|
||||
|
|
@ -186,7 +214,7 @@ int main(int argc, char **argv) {
|
|||
MI->second.Data.end(),
|
||||
BI);
|
||||
if (TI != MI->second.Data.end() && *TI == BI) {
|
||||
replaceStringRefs(BI);
|
||||
replaceBIStringRefs(BI);
|
||||
TI->mergeWith(BI);
|
||||
} else {
|
||||
CopyBranchInfo(BI, TmpBI);
|
||||
|
|
@ -208,7 +236,7 @@ int main(int argc, char **argv) {
|
|||
auto NamePtr = MergedStringPool.intern(FI.second.Name);
|
||||
AllStrings.emplace_back(NamePtr); // keep the ref
|
||||
bool Success;
|
||||
std::tie(MI, Success) = MergedFunctionsData.insert(
|
||||
std::tie(MI, Success) = MergedFunctionsBranchData.insert(
|
||||
std::make_pair(*NamePtr,
|
||||
FuncBranchData(*NamePtr,
|
||||
FuncBranchData::ContainerTy())));
|
||||
|
|
@ -218,7 +246,7 @@ int main(int argc, char **argv) {
|
|||
BranchInfo *PrevBI = nullptr;
|
||||
for (auto &BI : FI.second.Data) {
|
||||
if (PrevBI && *PrevBI == BI) {
|
||||
replaceStringRefs(BI);
|
||||
replaceBIStringRefs(BI);
|
||||
PrevBI->mergeWith(BI);
|
||||
} else {
|
||||
CopyBranchInfo(BI, MI->second.Data);
|
||||
|
|
@ -227,24 +255,98 @@ int main(int argc, char **argv) {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!opts::SuppressMergedDataOutput) {
|
||||
// Print all the data in the original format
|
||||
for (const auto &FDI : MergedFunctionsData) {
|
||||
for (const auto &BD : FDI.second.Data) {
|
||||
BD.print(outs());
|
||||
for (auto NameIter = ReaderOrErr.get()->getEventNames().begin(),
|
||||
End = ReaderOrErr.get()->getEventNames().end();
|
||||
NameIter != End; ++NameIter) {
|
||||
auto NamePtr = MergedStringPool.intern(NameIter->getKey());
|
||||
EventNames.insert(*NamePtr);
|
||||
}
|
||||
|
||||
for (auto &FI : ReaderOrErr.get()->getAllFuncsSampleData()) {
|
||||
auto MI = MergedFunctionsSampleData.find(FI.second.Name);
|
||||
if (MI != MergedFunctionsSampleData.end()) {
|
||||
std::vector<SampleInfo> TmpSI;
|
||||
for (auto &SI : FI.second.Data) {
|
||||
// Find and merge a corresponding entry or copy data.
|
||||
auto TI = std::lower_bound(MI->second.Data.begin(),
|
||||
MI->second.Data.end(),
|
||||
SI);
|
||||
if (TI != MI->second.Data.end() && *TI == SI) {
|
||||
replaceSIStringRefs(SI);
|
||||
TI->mergeWith(SI);
|
||||
} else {
|
||||
CopySampleInfo(SI, TmpSI);
|
||||
}
|
||||
}
|
||||
// Merge in the temp vector making sure it doesn't contain duplicates.
|
||||
std::sort(TmpSI.begin(), TmpSI.end());
|
||||
SampleInfo *PrevSI = nullptr;
|
||||
for (auto &SI : TmpSI) {
|
||||
if (PrevSI && *PrevSI == SI) {
|
||||
PrevSI->mergeWith(SI);
|
||||
} else {
|
||||
MI->second.Data.emplace_back(SI);
|
||||
PrevSI = &MI->second.Data.back();
|
||||
}
|
||||
}
|
||||
std::sort(MI->second.Data.begin(), MI->second.Data.end());
|
||||
} else {
|
||||
auto NamePtr = MergedStringPool.intern(FI.second.Name);
|
||||
AllStrings.emplace_back(NamePtr); // keep the ref
|
||||
bool Success;
|
||||
std::tie(MI, Success) = MergedFunctionsSampleData.insert(
|
||||
std::make_pair(*NamePtr,
|
||||
FuncSampleData(*NamePtr,
|
||||
FuncSampleData::ContainerTy())));
|
||||
// Copy with string conversion while eliminating duplicates.
|
||||
std::sort(FI.second.Data.begin(), FI.second.Data.end());
|
||||
SampleInfo *PrevSI = nullptr;
|
||||
for (auto &SI : FI.second.Data) {
|
||||
if (PrevSI && *PrevSI == SI) {
|
||||
replaceSIStringRefs(SI);
|
||||
PrevSI->mergeWith(SI);
|
||||
} else {
|
||||
CopySampleInfo(SI, MI->second.Data);
|
||||
PrevSI = &MI->second.Data.back();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
errs() << "Data for " << MergedFunctionsData.size()
|
||||
if (!opts::SuppressMergedDataOutput) {
|
||||
// Print all the data in the original format
|
||||
// Print mode
|
||||
if (MergedFunctionsSampleData.size() > 0) {
|
||||
outs() << "no_lbr";
|
||||
for (auto NameIter = EventNames.begin(), End = EventNames.end();
|
||||
NameIter != End; ++NameIter) {
|
||||
outs() << " " << NameIter->getKey();
|
||||
}
|
||||
outs() << "\n";
|
||||
}
|
||||
for (const auto &FDI : MergedFunctionsBranchData) {
|
||||
for (const auto &BD : FDI.second.Data) {
|
||||
BD.print(outs());
|
||||
}
|
||||
}
|
||||
for (const auto &FDI : MergedFunctionsSampleData) {
|
||||
for (const auto &SD : FDI.second.Data) {
|
||||
SD.print(outs());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
errs() << "Data for "
|
||||
<< (MergedFunctionsBranchData.size() +
|
||||
MergedFunctionsSampleData.size())
|
||||
<< " unique objects successfully merged.\n";
|
||||
|
||||
if (opts::PrintFunctionList != opts::ST_NONE) {
|
||||
// List of function names with execution count.
|
||||
std::vector<std::pair<uint64_t, StringRef>>
|
||||
FunctionList(MergedFunctionsData.size());
|
||||
FunctionList(MergedFunctionsBranchData.size());
|
||||
using CountFuncType =
|
||||
std::function<std::pair<uint64_t,StringRef>(
|
||||
const StringMapEntry<FuncBranchData>&)>;
|
||||
|
|
@ -264,8 +366,8 @@ int main(int argc, char **argv) {
|
|||
CountFuncType CountFunc = (opts::PrintFunctionList == opts::ST_EXEC_COUNT)
|
||||
? ExecCountFunc
|
||||
: BranchCountFunc;
|
||||
std::transform(MergedFunctionsData.begin(),
|
||||
MergedFunctionsData.end(),
|
||||
std::transform(MergedFunctionsBranchData.begin(),
|
||||
MergedFunctionsBranchData.end(),
|
||||
FunctionList.begin(),
|
||||
CountFunc);
|
||||
std::stable_sort(FunctionList.rbegin(), FunctionList.rend());
|
||||
|
|
|
|||
Loading…
Reference in New Issue