maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Refactor runtime library 

Summary:
As we are adding more types of runtime libraries, it would be better to move the runtime library out of RewriteInstance so that it could grow separately. This also requires splitting the current implementation of Instrumentation.cpp to two separate pieces, one as normal Pass, one as the runtime library. The Instrumentation Pass would pass over the generated data to the runtime library, which will use to emit binary and perform linking.

This patch does the following:
1. Turn Instrumentation class into an optimization pass. Register the pass in the pass manager instead of in RewriteInstance.
2. Split all the data that are generated by Instrumentation that's needed by runtime library into a separate data structure called InstrumentationSummary. At the creation of Instrumentation pass, we create an instance of such data structure, which will be moved over to the runtime at the end of the pass.
3. Added a runtime library member to BinaryContext. Set the member at the end of Instrumentation pass.
4. In BinaryEmitter, make BinaryContext to also emit runtime library binary.
5. Created a base class RuntimeLibrary, that defines the interface of a runtime library, along with a few common helper functions.
6. Created InstrumentationRuntimeLibrary which inherits from RuntimeLibrary, that does all the work (mostly copied over) for emit and linking.
7. Added a new directory called RuntimeLibs, and put all the runtime library related files into it.

(cherry picked from FBD21694762)
This commit is contained in:
Xun Li 2020-05-21 14:28:47 -07:00 committed by Maksim Panchenko
parent cd067ae1e8
commit 00892a5fd0
14 changed files with 689 additions and 449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bolt/src/BinaryContext.h
View File

@ -19,10 +19,11 @@
#include "DebugData.h"
#include "JumpTable.h"
#include "MCPlusBuilder.h"
#include "llvm/ADT/iterator.h"
#include "RuntimeLibs/RuntimeLibrary.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/iterator.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/BinaryFormat/MachO.h"
#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
@ -43,8 +44,8 @@
#include "llvm/MC/MCSymbol.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/raw_ostream.h"
#include <functional>
#include <map>
#include <set>
@ -238,6 +239,12 @@ public:
NumUnusedProfiledObjects = N;
}
RuntimeLibrary *getRuntimeLibrary() { return RtLibrary.get(); }
void setRuntimeLibrary(std::unique_ptr<RuntimeLibrary> Lib) {
assert(!RtLibrary && "Cannot set runtime library twice.");
RtLibrary = std::move(Lib);
}
/// Return BinaryFunction containing a given \p Address or nullptr if
/// no registered function has it.
///
@ -541,6 +548,9 @@ public:
/// Map SDT locations to SDT markers info
std::unordered_map<uint64_t, SDTMarkerInfo> SDTMarkers;
/// The runtime library.
std::unique_ptr<RuntimeLibrary> RtLibrary;
BinaryContext(std::unique_ptr<MCContext> Ctx,
std::unique_ptr<DWARFContext> DwCtx,
std::unique_ptr<Triple> TheTriple,

4
bolt/src/BinaryEmitter.cpp
View File

@ -193,6 +193,10 @@ private:
void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
Streamer.InitSections(false);
if (auto *RtLibrary = BC.getRuntimeLibrary()) {
RtLibrary->emitBinary(BC, Streamer);
}
BC.getTextSection()->setAlignment(opts::AlignText);
emitFunctions();

13
bolt/src/BinaryPassManager.cpp
View File

@ -16,15 +16,16 @@
#include "Passes/IdenticalCodeFolding.h"
#include "Passes/IndirectCallPromotion.h"
#include "Passes/Inliner.h"
#include "Passes/LongJmp.h"
#include "Passes/Instrumentation.h"
#include "Passes/JTFootprintReduction.h"
#include "Passes/LongJmp.h"
#include "Passes/PLTCall.h"
#include "Passes/RegReAssign.h"
#include "Passes/ReorderFunctions.h"
#include "Passes/ReorderData.h"
#include "Passes/ReorderFunctions.h"
#include "Passes/RetpolineInsertion.h"
#include "Passes/SplitFunctions.h"
#include "Passes/StokeInfo.h"
#include "Passes/RetpolineInsertion.h"
#include "Passes/ValidateInternalCalls.h"
#include "Passes/VeneerElimination.h"
#include "llvm/Support/Timer.h"
@ -38,6 +39,8 @@ namespace opts {
extern cl::OptionCategory BoltOptCategory;
extern cl::OptionCategory BoltCategory;
extern cl::opt<bool> Instrument;
extern cl::opt<unsigned> Verbosity;
extern cl::opt<bool> PrintAll;
extern cl::opt<bool> PrintDynoStats;
@ -378,6 +381,10 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
const auto InitialDynoStats = getDynoStats(BC.getBinaryFunctions());
if (opts::Instrument) {
Manager.registerPass(llvm::make_unique<Instrumentation>(NeverPrint));
}
// Here we manage dependencies/order manually, since passes are run in the
// order they're registered.

2
bolt/src/CMakeLists.txt
View File

@ -1,5 +1,6 @@
add_subdirectory(merge-fdata)
add_subdirectory(Passes)
add_subdirectory(RuntimeLibs)
add_subdirectory(Target)
# Get the current git revision for BOLT.
@ -48,6 +49,7 @@ add_public_gen_version_target(GenBoltRevision)
set(LLVM_LINK_COMPONENTS
${LLVM_TARGETS_TO_BUILD}
BOLTPasses
BOLTRuntimeLibs
CodeGen
Core
DebugInfoDWARF

268
bolt/src/Passes/Instrumentation.cpp
View File

@ -71,10 +71,10 @@ uint32_t Instrumentation::getFunctionNameIndex(const BinaryFunction &Function) {
auto Iter = FuncToStringIdx.find(&Function);
if (Iter != FuncToStringIdx.end())
return Iter->second;
auto Idx = StringTable.size();
auto Idx = Summary->StringTable.size();
FuncToStringIdx.emplace(std::make_pair(&Function, Idx));
StringTable.append(Function.getOneName());
StringTable.append(1, '\0');
Summary->StringTable.append(Function.getOneName());
Summary->StringTable.append(1, '\0');
return Idx;
}
@ -95,7 +95,7 @@ bool Instrumentation::createCallDescription(FunctionDescription &FuncDesc,
CD.Target = &ToFunction;
CD.ToLoc.FuncString = getFunctionNameIndex(ToFunction);
CD.ToLoc.Offset = To;
CD.Counter = ForceInstrumentation ? Counters.size() : 0xffffffff;
CD.Counter = ForceInstrumentation ? Summary->Counters.size() : 0xffffffff;
if (ForceInstrumentation)
++DirectCallCounters;
FuncDesc.Calls.emplace_back(CD);
@ -107,7 +107,7 @@ void Instrumentation::createIndCallDescription(
IndCallDescription ICD;
ICD.FromLoc.FuncString = getFunctionNameIndex(FromFunction);
ICD.FromLoc.Offset = From;
IndCallDescriptions.emplace_back(ICD);
Summary->IndCallDescriptions.emplace_back(ICD);
}
void Instrumentation::createIndCallTargetDescription(
@ -116,7 +116,7 @@ void Instrumentation::createIndCallTargetDescription(
ICD.ToLoc.FuncString = getFunctionNameIndex(ToFunction);
ICD.ToLoc.Offset = To;
ICD.Target = &ToFunction;
IndCallTargetDescriptions.emplace_back(ICD);
Summary->IndCallTargetDescriptions.emplace_back(ICD);
}
bool Instrumentation::createEdgeDescription(
@ -137,7 +137,7 @@ bool Instrumentation::createEdgeDescription(
ED.ToLoc.FuncString = getFunctionNameIndex(ToFunction);
ED.ToLoc.Offset = To;
ED.ToNode = ToNodeID;
ED.Counter = Instrumented ? Counters.size() : 0xffffffff;
ED.Counter = Instrumented ? Summary->Counters.size() : 0xffffffff;
if (Instrumented)
++BranchCounters;
FuncDesc.Edges.emplace_back(ED);
@ -148,7 +148,7 @@ void Instrumentation::createLeafNodeDescription(FunctionDescription &FuncDesc,
uint32_t Node) {
InstrumentedNode IN;
IN.Node = Node;
IN.Counter = Counters.size();
IN.Counter = Summary->Counters.size();
++LeafNodeCounters;
FuncDesc.LeafNodes.emplace_back(IN);
}
@ -158,7 +158,7 @@ Instrumentation::createInstrumentationSnippet(BinaryContext &BC, bool IsLeaf) {
auto L = BC.scopeLock();
MCSymbol *Label;
Label = BC.Ctx->createTempSymbol("InstrEntry", true);
Counters.emplace_back(Label);
Summary->Counters.emplace_back(Label);
std::vector<MCInst> CounterInstrs(5);
// Don't clobber application red zone (ABI dependent)
if (IsLeaf)
@ -193,15 +193,16 @@ void Instrumentation::instrumentIndirectTarget(BinaryBasicBlock &BB,
BinaryFunction &FromFunction,
uint32_t From) {
auto L = FromFunction.getBinaryContext().scopeLock();
const auto IndCallSiteID = IndCallDescriptions.size();
const auto IndCallSiteID = Summary->IndCallDescriptions.size();
createIndCallDescription(FromFunction, From);
BinaryContext &BC = FromFunction.getBinaryContext();
bool IsTailCall = BC.MIB->isTailCall(*Iter);
std::vector<MCInst> CounterInstrs = BC.MIB->createInstrumentedIndirectCall(
*Iter, IsTailCall,
IsTailCall ? IndTailCallHandlerFunc : IndCallHandlerFunc, IndCallSiteID,
&*BC.Ctx);
IsTailCall ? Summary->IndTailCallHandlerFunc
: Summary->IndCallHandlerFunc,
IndCallSiteID, &*BC.Ctx);
Iter = BB.eraseInstruction(Iter);
for (auto &NewInst : CounterInstrs) {
@ -283,8 +284,8 @@ void Instrumentation::instrumentFunction(BinaryContext &BC,
FunctionDescription *FuncDesc = nullptr;
{
std::unique_lock<std::shared_timed_mutex> L(FDMutex);
FunctionDescriptions.emplace_back();
FuncDesc = &FunctionDescriptions.back();
Summary->FunctionDescriptions.emplace_back();
FuncDesc = &Summary->FunctionDescriptions.back();
}
FuncDesc->Function = &Function;
@ -507,8 +508,9 @@ void Instrumentation::runOnFunctions(BinaryContext &BC) {
/*Alignment=*/1,
/*IsReadOnly=*/true, ELF::SHT_NOTE);
IndCallHandlerFunc = BC.Ctx->getOrCreateSymbol("__bolt_trampoline_ind_call");
IndTailCallHandlerFunc =
Summary->IndCallHandlerFunc =
BC.Ctx->getOrCreateSymbol("__bolt_trampoline_ind_call");
Summary->IndTailCallHandlerFunc =
BC.Ctx->getOrCreateSymbol("__bolt_trampoline_ind_tailcall");
ParallelUtilities::PredicateTy SkipPredicate = [&](const BinaryFunction &BF) {
@ -526,6 +528,8 @@ void Instrumentation::runOnFunctions(BinaryContext &BC) {
SkipPredicate, "instrumentation", /* ForceSequential=*/true);
createAuxiliaryFunctions(BC);
setupRuntimeLibrary(BC);
}
void Instrumentation::createAuxiliaryFunctions(BinaryContext &BC) {
@ -545,247 +549,47 @@ void Instrumentation::createAuxiliaryFunctions(BinaryContext &BC) {
return Func;
};
InitialIndCallHandlerFunction =
Summary->InitialIndCallHandlerFunction =
createSimpleFunction("__bolt_instr_default_ind_call_handler",
BC.MIB->createInstrumentedNoopIndCallHandler());
InitialIndTailCallHandlerFunction =
Summary->InitialIndTailCallHandlerFunction =
createSimpleFunction("__bolt_instr_default_ind_tailcall_handler",
BC.MIB->createInstrumentedNoopIndTailCallHandler());
}
uint32_t Instrumentation::getFDSize() const {
uint32_t FuncDescSize = 0;
for (const auto &Func : FunctionDescriptions) {
FuncDescSize += 16 + Func.Edges.size() * sizeof(EdgeDescription) +
Func.LeafNodes.size() * sizeof(InstrumentedNode) +
Func.Calls.size() * sizeof(CallDescription) +
Func.EntryNodes.size() * sizeof(EntryNode);
}
return FuncDescSize;
}
void Instrumentation::setupRuntimeLibrary(BinaryContext &BC) {
auto FuncDescSize = Summary->getFDSize();
void Instrumentation::emitTablesAsELFNote(BinaryContext &BC) {
std::string TablesStr;
raw_string_ostream OS(TablesStr);
// This is sync'ed with runtime/instr.cpp:readDescriptions()
auto getOutputAddress = [](const BinaryFunction &Func,
uint64_t Offset) -> uint64_t {
return Offset == 0
? Func.getOutputAddress()
: Func.translateInputToOutputAddress(Func.getAddress() + Offset);
};
// Indirect targets need to be sorted for fast lookup during runtime
std::sort(IndCallTargetDescriptions.begin(), IndCallTargetDescriptions.end(),
[&](const IndCallTargetDescription &A,
const IndCallTargetDescription &B) {
return getOutputAddress(*A.Target, A.ToLoc.Offset) <
getOutputAddress(*B.Target, B.ToLoc.Offset);
});
// Start of the vector with descriptions (one CounterDescription for each
// counter), vector size is Counters.size() CounterDescription-sized elmts
const auto IDSize = IndCallDescriptions.size() * sizeof(IndCallDescription);
OS.write(reinterpret_cast<const char *>(&IDSize), 4);
for (const auto &Desc : IndCallDescriptions) {
OS.write(reinterpret_cast<const char *>(&Desc.FromLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Desc.FromLoc.Offset), 4);
}
const auto ITDSize =
IndCallTargetDescriptions.size() * sizeof(IndCallTargetDescription);
OS.write(reinterpret_cast<const char *>(&ITDSize), 4);
for (const auto &Desc : IndCallTargetDescriptions) {
OS.write(reinterpret_cast<const char *>(&Desc.ToLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Desc.ToLoc.Offset), 4);
uint64_t TargetFuncAddress =
getOutputAddress(*Desc.Target, Desc.ToLoc.Offset);
OS.write(reinterpret_cast<const char *>(&TargetFuncAddress), 8);
}
const auto FDSize = getFDSize();
OS.write(reinterpret_cast<const char *>(&FDSize), 4);
for (const auto &Desc : FunctionDescriptions) {
const auto LeafNum = Desc.LeafNodes.size();
OS.write(reinterpret_cast<const char *>(&LeafNum), 4);
for (const auto &LeafNode : Desc.LeafNodes) {
OS.write(reinterpret_cast<const char *>(&LeafNode.Node), 4);
OS.write(reinterpret_cast<const char *>(&LeafNode.Counter), 4);
}
const auto EdgesNum = Desc.Edges.size();
OS.write(reinterpret_cast<const char *>(&EdgesNum), 4);
for (const auto &Edge : Desc.Edges) {
OS.write(reinterpret_cast<const char *>(&Edge.FromLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Edge.FromLoc.Offset), 4);
OS.write(reinterpret_cast<const char *>(&Edge.FromNode), 4);
OS.write(reinterpret_cast<const char *>(&Edge.ToLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Edge.ToLoc.Offset), 4);
OS.write(reinterpret_cast<const char *>(&Edge.ToNode), 4);
OS.write(reinterpret_cast<const char *>(&Edge.Counter), 4);
}
const auto CallsNum = Desc.Calls.size();
OS.write(reinterpret_cast<const char *>(&CallsNum), 4);
for (const auto &Call : Desc.Calls) {
OS.write(reinterpret_cast<const char *>(&Call.FromLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Call.FromLoc.Offset), 4);
OS.write(reinterpret_cast<const char *>(&Call.FromNode), 4);
OS.write(reinterpret_cast<const char *>(&Call.ToLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Call.ToLoc.Offset), 4);
OS.write(reinterpret_cast<const char *>(&Call.Counter), 4);
uint64_t TargetFuncAddress =
getOutputAddress(*Call.Target, Call.ToLoc.Offset);
OS.write(reinterpret_cast<const char *>(&TargetFuncAddress), 8);
}
const auto EntryNum = Desc.EntryNodes.size();
OS.write(reinterpret_cast<const char *>(&EntryNum), 4);
for (const auto &EntryNode : Desc.EntryNodes) {
OS.write(reinterpret_cast<const char *>(&EntryNode.Node), 8);
uint64_t TargetFuncAddress =
getOutputAddress(*Desc.Function, EntryNode.Address);
OS.write(reinterpret_cast<const char *>(&TargetFuncAddress), 8);
}
}
// Our string table lives immediately after descriptions vector
OS << StringTable;
OS.flush();
const auto BoltInfo = BinarySection::encodeELFNote(
"BOLT", TablesStr, BinarySection::NT_BOLT_INSTRUMENTATION_TABLES);
BC.registerOrUpdateNoteSection(".bolt.instr.tables", copyByteArray(BoltInfo),
BoltInfo.size(),
/*Alignment=*/1,
/*IsReadOnly=*/true, ELF::SHT_NOTE);
}
void Instrumentation::emit(BinaryContext &BC, MCStreamer &Streamer) {
const auto *StartFunction =
BC.getBinaryFunctionAtAddress(*BC.StartFunctionAddress);
if (!StartFunction) {
errs() << "BOLT-ERROR: failed to locate function at binary start address\n";
exit(1);
}
const auto *FiniFunction =
BC.getBinaryFunctionAtAddress(*BC.FiniFunctionAddress);
if (!FiniFunction) {
errs() << "BOLT-ERROR: failed to locate function at binary fini address\n";
exit(1);
}
const auto Flags = BinarySection::getFlags(/*IsReadOnly=*/false,
/*IsText=*/false,
/*IsAllocatable=*/true);
auto *Section = BC.Ctx->getELFSection(".bolt.instr.counters",
ELF::SHT_PROGBITS,
Flags);
// All of the following symbols will be exported as globals to be used by the
// instrumentation runtime library to dump the instrumentation data to disk.
// Label marking start of the memory region containing instrumentation
// counters, total vector size is Counters.size() 8-byte counters
MCSymbol *Locs = BC.Ctx->getOrCreateSymbol("__bolt_instr_locations");
MCSymbol *NumLocs = BC.Ctx->getOrCreateSymbol("__bolt_num_counters");
MCSymbol *NumIndCalls =
BC.Ctx->getOrCreateSymbol("__bolt_instr_num_ind_calls");
MCSymbol *NumIndCallTargets =
BC.Ctx->getOrCreateSymbol("__bolt_instr_num_ind_targets");
MCSymbol *NumFuncs = BC.Ctx->getOrCreateSymbol("__bolt_instr_num_funcs");
/// File name where profile is going to written to after target binary
/// finishes a run
MCSymbol *FilenameSym = BC.Ctx->getOrCreateSymbol("__bolt_instr_filename");
MCSymbol *UsePIDSym = BC.Ctx->getOrCreateSymbol("__bolt_instr_use_pid");
MCSymbol *InitPtr = BC.Ctx->getOrCreateSymbol("__bolt_instr_init_ptr");
MCSymbol *FiniPtr = BC.Ctx->getOrCreateSymbol("__bolt_instr_fini_ptr");
MCSymbol *SleepSym = BC.Ctx->getOrCreateSymbol("__bolt_instr_sleep_time");
Section->setAlignment(BC.RegularPageSize);
Streamer.SwitchSection(Section);
Streamer.EmitLabel(Locs);
Streamer.EmitSymbolAttribute(Locs,
MCSymbolAttr::MCSA_Global);
for (const auto &Label : Counters) {
Streamer.EmitLabel(Label);
Streamer.emitFill(8, 0);
}
const uint64_t Padding =
alignTo(8 * Counters.size(), BC.RegularPageSize) - 8 * Counters.size();
if (Padding)
Streamer.emitFill(Padding, 0);
Streamer.EmitLabel(SleepSym);
Streamer.EmitSymbolAttribute(SleepSym,
MCSymbolAttr::MCSA_Global);
Streamer.EmitIntValue(opts::InstrumentationSleepTime, /*Size=*/4);
Streamer.EmitLabel(NumLocs);
Streamer.EmitSymbolAttribute(NumLocs,
MCSymbolAttr::MCSA_Global);
Streamer.EmitIntValue(Counters.size(), /*Size=*/4);
Streamer.EmitLabel(IndCallHandlerFunc);
Streamer.EmitSymbolAttribute(IndCallHandlerFunc,
MCSymbolAttr::MCSA_Global);
Streamer.EmitValue(MCSymbolRefExpr::create(
InitialIndCallHandlerFunction->getSymbol(), *BC.Ctx),
/*Size=*/8);
Streamer.EmitLabel(IndTailCallHandlerFunc);
Streamer.EmitSymbolAttribute(IndTailCallHandlerFunc,
MCSymbolAttr::MCSA_Global);
Streamer.EmitValue(
MCSymbolRefExpr::create(InitialIndTailCallHandlerFunction->getSymbol(),
*BC.Ctx),
/*Size=*/8);
Streamer.EmitLabel(NumIndCalls);
Streamer.EmitSymbolAttribute(NumIndCalls,
MCSymbolAttr::MCSA_Global);
Streamer.EmitIntValue(IndCallDescriptions.size(), /*Size=*/4);
Streamer.EmitLabel(NumIndCallTargets);
Streamer.EmitSymbolAttribute(NumIndCallTargets,
MCSymbolAttr::MCSA_Global);
Streamer.EmitIntValue(IndCallTargetDescriptions.size(), /*Size=*/4);
Streamer.EmitLabel(NumFuncs);
Streamer.EmitSymbolAttribute(NumFuncs,
MCSymbolAttr::MCSA_Global);
Streamer.EmitIntValue(FunctionDescriptions.size(), /*Size=*/4);
Streamer.EmitLabel(FilenameSym);
Streamer.EmitBytes(opts::InstrumentationFilename);
Streamer.emitFill(1, 0);
Streamer.EmitLabel(UsePIDSym);
Streamer.EmitIntValue(opts::InstrumentationFileAppendPID ? 1 : 0, /*Size=*/1);
Streamer.EmitLabel(InitPtr);
Streamer.EmitSymbolAttribute(InitPtr,
MCSymbolAttr::MCSA_Global);
Streamer.EmitValue(
MCSymbolRefExpr::create(StartFunction->getSymbol(), *BC.Ctx), /*Size=*/8);
Streamer.EmitLabel(FiniPtr);
Streamer.EmitSymbolAttribute(FiniPtr, MCSymbolAttr::MCSA_Global);
Streamer.EmitValue(
MCSymbolRefExpr::create(FiniFunction->getSymbol(), *BC.Ctx), /*Size=*/8);
uint32_t FuncDescSize = getFDSize();
outs() << "BOLT-INSTRUMENTER: Number of indirect call site descriptors: "
<< IndCallDescriptions.size() << "\n";
<< Summary->IndCallDescriptions.size() << "\n";
outs() << "BOLT-INSTRUMENTER: Number of indirect call target descriptors: "
<< IndCallTargetDescriptions.size() << "\n";
<< Summary->IndCallTargetDescriptions.size() << "\n";
outs() << "BOLT-INSTRUMENTER: Number of function descriptors: "
<< FunctionDescriptions.size() << "\n";
<< Summary->FunctionDescriptions.size() << "\n";
outs() << "BOLT-INSTRUMENTER: Number of branch counters: " << BranchCounters
<< "\n";
outs() << "BOLT-INSTRUMENTER: Number of ST leaf node counters: "
<< LeafNodeCounters << "\n";
outs() << "BOLT-INSTRUMENTER: Number of direct call counters: "
<< DirectCallCounters << "\n";
outs() << "BOLT-INSTRUMENTER: Total number of counters: " << Counters.size()
<< "\n";
outs() << "BOLT-INSTRUMENTER: Total number of counters: "
<< Summary->Counters.size() << "\n";
outs() << "BOLT-INSTRUMENTER: Total size of counters: "
<< (Counters.size() * 8) << " bytes (static alloc memory)\n";
<< (Summary->Counters.size() * 8) << " bytes (static alloc memory)\n";
outs() << "BOLT-INSTRUMENTER: Total size of string table emitted: "
<< StringTable.size() << " bytes in file\n";
<< Summary->StringTable.size() << " bytes in file\n";
outs() << "BOLT-INSTRUMENTER: Total size of descriptors: "
<< (FuncDescSize +
IndCallDescriptions.size() * sizeof(IndCallDescription) +
IndCallTargetDescriptions.size() *
Summary->IndCallDescriptions.size() * sizeof(IndCallDescription) +
Summary->IndCallTargetDescriptions.size() *
sizeof(IndCallTargetDescription))
<< " bytes in file\n";
outs() << "BOLT-INSTRUMENTER: Profile will be saved to file "
<< opts::InstrumentationFilename << "\n";
}
BC.setRuntimeLibrary(
llvm::make_unique<InstrumentationRuntimeLibrary>(std::move(Summary)));
}
}
}

118
bolt/src/Passes/Instrumentation.h
View File

@ -18,6 +18,7 @@
#define LLVM_TOOLS_LLVM_BOLT_PASSES_INSTRUMENTATION_H
#include "BinaryPasses.h"
#include "RuntimeLibs/InstrumentationRuntimeLibrary.h"
#include "llvm/MC/MCSection.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSymbol.h"
@ -25,93 +26,18 @@
namespace llvm {
namespace bolt {
class Instrumentation {
class Instrumentation : public BinaryFunctionPass {
public:
Instrumentation() {}
Instrumentation(const cl::opt<bool> &PrintPass)
: BinaryFunctionPass(PrintPass),
Summary(llvm::make_unique<InstrumentationSummary>()) {}
/// Modifies all functions by inserting instrumentation code (first step)
void runOnFunctions(BinaryContext &BC);
void runOnFunctions(BinaryContext &BC) override;
/// Emit data structures that will be necessary during runtime (second step)
void emit(BinaryContext &BC, MCStreamer &Streamer);
/// Create a non-allocatable ELF section with read-only tables necessary for
/// writing the instrumented data profile during program finish. The runtime
/// library needs to open the program executable file and read this data from
/// disk, this is not loaded by the system.
void emitTablesAsELFNote(BinaryContext &BC);
const char *getName() const override { return "instrumentation"; }
private:
// All structs here are part of the program metadata serialization format and
// consist of POD types or array of POD types that are trivially mapped from
// disk to memory. This provides the runtime library with a basic
// understanding of the program structure, so it can build a CFG for each
// function and deduce execution counts for edges that don't require explicit
// counters. It also provides function names and offsets used when writing the
// fdata file.
// Location information -- analoguous to the concept of the same name in fdata
// writing/reading. The difference is that the name is stored as an index to a
// string table written separately.
struct LocDescription {
uint32_t FuncString;
uint32_t Offset;
};
// Inter-function control flow transfer instrumentation
struct CallDescription {
LocDescription FromLoc;
uint32_t FromNode; // Node refers to the CFG node index of the call site
LocDescription ToLoc;
uint32_t Counter;
const BinaryFunction *Target;
};
// Spans multiple counters during runtime - this is an indirect call site
struct IndCallDescription {
LocDescription FromLoc;
};
// This is an indirect call target (any entry point from any function). This
// is stored sorted in the binary for fast lookups during data writing.
struct IndCallTargetDescription {
LocDescription ToLoc;
const BinaryFunction *Target;
};
// Intra-function control flow transfer instrumentation
struct EdgeDescription {
LocDescription FromLoc;
uint32_t FromNode;
LocDescription ToLoc;
uint32_t ToNode;
uint32_t Counter;
};
// Basic block frequency (CFG node) instrumentation - only used for spanning
// tree leaf nodes.
struct InstrumentedNode {
uint32_t Node;
uint32_t Counter;
};
// Entry basic blocks for a function. We record their output addresses to
// check frequency of this address (via node number) against all tracked calls
// to this address and discover traffic coming from uninstrumented code.
struct EntryNode {
uint64_t Node;
uint64_t Address;
};
// Base struct organizing all metadata pertaining to a single function
struct FunctionDescription {
const BinaryFunction *Function;
std::vector<InstrumentedNode> LeafNodes;
std::vector<EdgeDescription> Edges;
DenseSet<std::pair<uint32_t, uint32_t>> EdgesSet;
std::vector<CallDescription> Calls;
std::vector<EntryNode> EntryNodes;
};
void instrumentFunction(BinaryContext &BC, BinaryFunction &Function,
MCPlusBuilder::AllocatorIdTy = 0);
@ -180,42 +106,24 @@ private:
uint32_t getFDSize() const;
/// Stores function names, to be emitted to the runtime
std::string StringTable;
/// Create a runtime library, pass the BinData over, and register it
/// under \p BC.
void setupRuntimeLibrary(BinaryContext &BC);
/// strtab indices in StringTable for each function name
std::unordered_map<const BinaryFunction *, uint32_t> FuncToStringIdx;
/// Intra-function control flow and direct calls
std::vector<FunctionDescription> FunctionDescriptions;
mutable std::shared_timed_mutex FDMutex;
/// Inter-function control flow via indirect calls
std::vector<IndCallDescription> IndCallDescriptions;
std::vector<IndCallTargetDescription> IndCallTargetDescriptions;
/// Identify all counters used in runtime while instrumentation is running
std::vector<MCSymbol *> Counters;
/// Our runtime indirect call instrumenter function
MCSymbol *IndCallHandlerFunc;
MCSymbol *IndTailCallHandlerFunc;
/// Our generated initial indirect call handler function that does nothing
/// except calling the indirect call target. The target program starts
/// using this no-op instrumentation function until our runtime library
/// setup runs and installs the correct handler. We need something before
/// our setup runs in case dyld starts running init code for other libs when
/// we did not have time to set up our indirect call counters yet.
BinaryFunction *InitialIndCallHandlerFunction;
BinaryFunction *InitialIndTailCallHandlerFunction;
/// The data generated during Instrumentation pass that needs to
/// be passed to the Instrument runtime library.
std::unique_ptr<InstrumentationSummary> Summary;
/// Statistics on counters
uint32_t DirectCallCounters{0};
uint32_t BranchCounters{0};
uint32_t LeafNodeCounters{0};
};
}
}

152
bolt/src/Passes/InstrumentationSummary.h Normal file
View File

@ -0,0 +1,152 @@
//===--- InstrumentationSummary.h -------------------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
// InstrumentationSummary holds all the data generated during
// the Instrumentation pass, which will be needed latter for runtime library
// binary emit and linking.
//===----------------------------------------------------------------------===//
#ifndef LLVM_TOOLS_LLVM_BOLT_PASSES_INSTRUMENTATION_SUMMARY_H
#define LLVM_TOOLS_LLVM_BOLT_PASSES_INSTRUMENTATION_SUMMARY_H
#include "llvm/ADT/DenseSet.h"
namespace llvm {
class MCSymbol;
namespace bolt {
class BinaryFunction;
// All structs here are part of the program metadata serialization format and
// consist of POD types or array of POD types that are trivially mapped from
// disk to memory. This provides the runtime library with a basic
// understanding of the program structure, so it can build a CFG for each
// function and deduce execution counts for edges that don't require explicit
// counters. It also provides function names and offsets used when writing the
// fdata file.
// Location information -- analoguous to the concept of the same name in fdata
// writing/reading. The difference is that the name is stored as an index to a
// string table written separately.
struct LocDescription {
uint32_t FuncString;
uint32_t Offset;
};
// Inter-function control flow transfer instrumentation
struct CallDescription {
LocDescription FromLoc;
uint32_t FromNode; // Node refers to the CFG node index of the call site
LocDescription ToLoc;
uint32_t Counter;
const BinaryFunction *Target;
};
// Spans multiple counters during runtime - this is an indirect call site
struct IndCallDescription {
LocDescription FromLoc;
};
// This is an indirect call target (any entry point from any function). This
// is stored sorted in the binary for fast lookups during data writing.
struct IndCallTargetDescription {
LocDescription ToLoc;
const BinaryFunction *Target;
};
// Intra-function control flow transfer instrumentation
struct EdgeDescription {
LocDescription FromLoc;
uint32_t FromNode;
LocDescription ToLoc;
uint32_t ToNode;
uint32_t Counter;
};
// Basic block frequency (CFG node) instrumentation - only used for spanning
// tree leaf nodes.
struct InstrumentedNode {
uint32_t Node;
uint32_t Counter;
};
// Entry basic blocks for a function. We record their output addresses to
// check frequency of this address (via node number) against all tracked calls
// to this address and discover traffic coming from uninstrumented code.
struct EntryNode {
uint64_t Node;
uint64_t Address;
};
// Base struct organizing all metadata pertaining to a single function
struct FunctionDescription {
const BinaryFunction *Function;
std::vector<InstrumentedNode> LeafNodes;
std::vector<EdgeDescription> Edges;
DenseSet<std::pair<uint32_t, uint32_t>> EdgesSet;
std::vector<CallDescription> Calls;
std::vector<EntryNode> EntryNodes;
};
/// Holds the summary of the data generated by the Instrumentation Pass.
/// These information will be needed for binary emit.
struct InstrumentationSummary {
/// Identify all counters used in runtime while instrumentation is running
std::vector<MCSymbol *> Counters;
/// Stores function names, to be emitted to the runtime
std::string StringTable;
/// Our runtime indirect call instrumenter function
MCSymbol *IndCallHandlerFunc;
MCSymbol *IndTailCallHandlerFunc;
/// Intra-function control flow and direct calls
std::vector<FunctionDescription> FunctionDescriptions;
/// Inter-function control flow via indirect calls
std::vector<IndCallDescription> IndCallDescriptions;
std::vector<IndCallTargetDescription> IndCallTargetDescriptions;
/// Our generated initial indirect call handler function that does nothing
/// except calling the indirect call target. The target program starts
/// using this no-op instrumentation function until our runtime library
/// setup runs and installs the correct handler. We need something before
/// our setup runs in case dyld starts running init code for other libs when
/// we did not have time to set up our indirect call counters yet.
BinaryFunction *InitialIndCallHandlerFunction;
BinaryFunction *InitialIndTailCallHandlerFunction;
static constexpr uint64_t NUM_SERIALIZED_CONTAINERS = 4;
static constexpr uint64_t SERIALIZED_CONTAINER_SIZE =
sizeof(uint32_t) * NUM_SERIALIZED_CONTAINERS;
uint32_t getFDSize() const {
uint32_t FuncDescSize = 0;
for (const auto &Func : FunctionDescriptions) {
// A function description consists of containers of different
// descriptions. We use vectors to store them and when serializing them,
// we first output a uint32_t-sized field for the number of elements of
// the vector and then we write each element, so a simple parser know
// where to stop.
FuncDescSize += SERIALIZED_CONTAINER_SIZE +
Func.Edges.size() * sizeof(EdgeDescription) +
Func.LeafNodes.size() * sizeof(InstrumentedNode) +
Func.Calls.size() * sizeof(CallDescription) +
Func.EntryNodes.size() * sizeof(EntryNode);
}
return FuncDescSize;
}
};
} // namespace bolt
} // namespace llvm
#endif

116
bolt/src/RewriteInstance.cpp
View File

@ -91,6 +91,7 @@ extern cl::OptionCategory BoltOutputCategory;
extern cl::OptionCategory AggregatorCategory;
extern cl::opt<MacroFusionType> AlignMacroOpFusion;
extern cl::opt<bool> Instrument;
extern cl::opt<JumpTableSupportLevel> JumpTables;
extern cl::list<std::string> ReorderData;
extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions;
@ -103,19 +104,6 @@ AlignText("align-text",
cl::Hidden,
cl::cat(BoltCategory));
cl::opt<bool>
Instrument("instrument",
cl::desc("instrument code to generate accurate profile data"),
cl::ZeroOrMore,
cl::cat(BoltOptCategory));
static cl::opt<std::string>
RuntimeInstrumentationLib("runtime-instrumentation-lib",
cl::desc("specify file name of the runtime instrumentation library"),
cl::ZeroOrMore,
cl::init("libbolt_rt.a"),
cl::cat(BoltOptCategory));
static cl::opt<bool>
ForceToDataRelocations("force-data-relocations",
cl::desc("force relocations to data sections to always be processed"),
@ -2529,10 +2517,6 @@ void RewriteInstance::postProcessFunctions() {
void RewriteInstance::runOptimizationPasses() {
NamedRegionTimer T("runOptimizationPasses", "run optimization passes",
TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
if (opts::Instrument) {
Instrumenter = llvm::make_unique<Instrumentation>();
Instrumenter->runOnFunctions(*BC);
}
BinaryFunctionPassManager::runAllPasses(*BC);
}
@ -2587,11 +2571,6 @@ void RewriteInstance::emitAndLink() {
}
}
// Emit contents outside of BinaryContext.
if (opts::Instrument) {
Instrumenter->emit(*BC, *Streamer.get());
}
emitBinaryContext(*Streamer, *BC, getOrgSecPrefix());
Streamer->Finish();
@ -2684,10 +2663,8 @@ void RewriteInstance::emitAndLink() {
cantFail(OLT->addObject(K, std::move(ObjectMemBuffer)));
cantFail(OLT->emitAndFinalize(K));
// Link instrumentation runtime library
if (opts::Instrument) {
linkRuntime();
Instrumenter->emitTablesAsELFNote(*BC);
if (auto *RtLibrary = BC->getRuntimeLibrary()) {
RtLibrary->link(*BC, ToolPath, *ES, *OLT);
}
// Once the code is emitted, we can rename function sections to actual
@ -2715,79 +2692,6 @@ void RewriteInstance::emitAndLink() {
TempOut->keep();
}
void RewriteInstance::linkRuntime() {
OLT->setProcessAllSections(false);
std::string Dir = llvm::sys::path::parent_path(ToolPath);
SmallString<128> P(Dir);
P = llvm::sys::path::parent_path(Dir);
llvm::sys::path::append(P, "lib", opts::RuntimeInstrumentationLib);
std::string LibPath = P.str();
if (!llvm::sys::fs::exists(LibPath)) {
errs() << "BOLT-ERROR: instrumentation library not found: " << LibPath
<< "\n";
exit(1);
}
ErrorOr<std::unique_ptr<MemoryBuffer>> MaybeBuf =
MemoryBuffer::getFile(LibPath, -1, false);
check_error(MaybeBuf.getError(), LibPath);
std::unique_ptr<MemoryBuffer> B = std::move(MaybeBuf.get());
file_magic Magic = identify_magic(B->getBuffer());
if (Magic == file_magic::archive) {
Error Err = Error::success();
object::Archive Archive(B.get()->getMemBufferRef(), Err);
for (auto &C : Archive.children(Err)) {
auto ChildKey = ES->allocateVModule();
auto ChildBuf =
MemoryBuffer::getMemBuffer(cantFail(C.getMemoryBufferRef()));
auto ChildMagic = identify_magic(ChildBuf->getBuffer());
if (ChildMagic != file_magic::elf_relocatable &&
ChildMagic != file_magic::elf_shared_object) {
errs() << "BOLT-ERROR: unrecognized instrumentation library format "
<< "inside the archiver: "
<< LibPath << "\n";
exit(1);
}
cantFail(OLT->addObject(ChildKey, std::move(ChildBuf)));
cantFail(OLT->emitAndFinalize(ChildKey));
}
check_error(std::move(Err), B->getBufferIdentifier());
} else if (Magic == file_magic::elf_relocatable ||
Magic == file_magic::elf_shared_object) {
auto K2 = ES->allocateVModule();
cantFail(OLT->addObject(K2, std::move(B)));
cantFail(OLT->emitAndFinalize(K2));
} else {
errs() << "BOLT-ERROR: unrecognized instrumentation library format: "
<< LibPath << "\n";
exit(1);
}
InstrumentationRuntimeFiniAddress =
cantFail(OLT->findSymbol("__bolt_instr_fini", false).getAddress());
if (!InstrumentationRuntimeFiniAddress) {
errs() << "BOLT-ERROR: instrumentation library does not define "
"__bolt_instr_fini: "
<< LibPath << "\n";
exit(1);
}
InstrumentationRuntimeStartAddress =
cantFail(OLT->findSymbol("__bolt_instr_start", false).getAddress());
if (!InstrumentationRuntimeStartAddress) {
errs() << "BOLT-ERROR: instrumentation library does not define "
"__bolt_instr_start: "
<< LibPath << "\n";
exit(1);
}
outs() << "BOLT-INFO: output linked against instrumentation runtime "
"library, lib entry point is 0x"
<< Twine::utohexstr(InstrumentationRuntimeFiniAddress) << "\n";
outs() << "BOLT-INFO: clear procedure is 0x"
<< Twine::utohexstr(
cantFail(OLT->findSymbol("__bolt_instr_clear_counters", false)
.getAddress()))
<< "\n";
}
void RewriteInstance::updateMetadata() {
updateSDTMarkers();
@ -3676,8 +3580,11 @@ void RewriteInstance::patchELFSectionHeaderTable(ELFObjectFile<ELFT> *File) {
auto NewEhdr = *Obj->getHeader();
if (BC->HasRelocations) {
NewEhdr.e_entry = opts::Instrument ? InstrumentationRuntimeStartAddress
: getNewFunctionAddress(NewEhdr.e_entry);
if (auto *RtLibrary = BC->getRuntimeLibrary()) {
NewEhdr.e_entry = RtLibrary->getRuntimeStartAddress();
} else {
NewEhdr.e_entry = getNewFunctionAddress(NewEhdr.e_entry);
}
assert(NewEhdr.e_entry && "cannot find new address for entry point");
}
NewEhdr.e_phoff = PHDRTableOffset;
@ -4224,8 +4131,11 @@ void RewriteInstance::patchELFDynamic(ELFObjectFile<ELFT> *File) {
NewDE.d_un.d_ptr = NewAddress;
}
}
if (opts::Instrument && DE->getTag() == ELF::DT_FINI)
NewDE.d_un.d_ptr = InstrumentationRuntimeFiniAddress;
if (DE->getTag() == ELF::DT_FINI) {
if (auto *RtLibrary = BC->getRuntimeLibrary()) {
NewDE.d_un.d_ptr = RtLibrary->getRuntimeFiniAddress();
}
}
break;
case ELF::DT_FLAGS:
if (BC->RequiresZNow) {

4
bolt/src/RewriteInstance.h
View File

@ -409,10 +409,6 @@ private:
uint64_t NewTextSegmentOffset{0};
uint64_t NewTextSegmentSize{0};
/// Extra linking
uint64_t InstrumentationRuntimeFiniAddress{0};
uint64_t InstrumentationRuntimeStartAddress{0};
/// Track next available address for new allocatable sections.
uint64_t NextAvailableAddress{0};

9
bolt/src/RuntimeLibs/CMakeLists.txt Normal file
View File

@ -0,0 +1,9 @@
add_llvm_library(LLVMBOLTRuntimeLibs
RuntimeLibrary.cpp
InstrumentationRuntimeLibrary.cpp
DEPENDS
intrinsics_gen
)
include_directories( ${LLVM_MAIN_SRC_DIR}/tools/llvm-bolt/src )

261
bolt/src/RuntimeLibs/InstrumentationRuntimeLibrary.cpp Normal file
View File

@ -0,0 +1,261 @@
//===InstrumentationRuntimeLibrary.cpp - The Instrumentation Runtime Library =//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//===----------------------------------------------------------------------===//
#include "InstrumentationRuntimeLibrary.h"
#include "BinaryFunction.h"
using namespace llvm;
using namespace bolt;
namespace opts {
extern cl::OptionCategory BoltOptCategory;
extern cl::opt<bool> InstrumentationFileAppendPID;
extern cl::opt<std::string> InstrumentationFilename;
extern cl::opt<uint32_t> InstrumentationSleepTime;
cl::opt<bool>
Instrument("instrument",
cl::desc("instrument code to generate accurate profile data"),
cl::ZeroOrMore, cl::cat(BoltOptCategory));
static cl::opt<std::string> RuntimeInstrumentationLib(
"runtime-instrumentation-lib",
cl::desc("specify file name of the runtime instrumentation library"),
cl::ZeroOrMore, cl::init("libbolt_rt.a"), cl::cat(BoltOptCategory));
} // namespace opts
void InstrumentationRuntimeLibrary::emitBinary(BinaryContext &BC,
MCStreamer &Streamer) {
const auto *StartFunction =
BC.getBinaryFunctionAtAddress(*BC.StartFunctionAddress);
if (!StartFunction) {
errs() << "BOLT-ERROR: failed to locate function at binary start address\n";
exit(1);
}
const auto *FiniFunction =
BC.getBinaryFunctionAtAddress(*BC.FiniFunctionAddress);
if (!FiniFunction) {
errs() << "BOLT-ERROR: failed to locate function at binary fini address\n";
exit(1);
}
const auto Flags = BinarySection::getFlags(/*IsReadOnly=*/false,
/*IsText=*/false,
/*IsAllocatable=*/true);
auto *Section =
BC.Ctx->getELFSection(".bolt.instr.counters", ELF::SHT_PROGBITS, Flags);
// All of the following symbols will be exported as globals to be used by the
// instrumentation runtime library to dump the instrumentation data to disk.
// Label marking start of the memory region containing instrumentation
// counters, total vector size is Counters.size() 8-byte counters
MCSymbol *Locs = BC.Ctx->getOrCreateSymbol("__bolt_instr_locations");
MCSymbol *NumLocs = BC.Ctx->getOrCreateSymbol("__bolt_num_counters");
MCSymbol *NumIndCalls =
BC.Ctx->getOrCreateSymbol("__bolt_instr_num_ind_calls");
MCSymbol *NumIndCallTargets =
BC.Ctx->getOrCreateSymbol("__bolt_instr_num_ind_targets");
MCSymbol *NumFuncs = BC.Ctx->getOrCreateSymbol("__bolt_instr_num_funcs");
/// File name where profile is going to written to after target binary
/// finishes a run
MCSymbol *FilenameSym = BC.Ctx->getOrCreateSymbol("__bolt_instr_filename");
MCSymbol *UsePIDSym = BC.Ctx->getOrCreateSymbol("__bolt_instr_use_pid");
MCSymbol *InitPtr = BC.Ctx->getOrCreateSymbol("__bolt_instr_init_ptr");
MCSymbol *FiniPtr = BC.Ctx->getOrCreateSymbol("__bolt_instr_fini_ptr");
MCSymbol *SleepSym = BC.Ctx->getOrCreateSymbol("__bolt_instr_sleep_time");
Section->setAlignment(BC.RegularPageSize);
Streamer.SwitchSection(Section);
Streamer.EmitLabel(Locs);
Streamer.EmitSymbolAttribute(Locs, MCSymbolAttr::MCSA_Global);
for (const auto &Label : Summary->Counters) {
Streamer.EmitLabel(Label);
Streamer.emitFill(8, 0);
}
const uint64_t Padding =
alignTo(8 * Summary->Counters.size(), BC.RegularPageSize) -
8 * Summary->Counters.size();
if (Padding)
Streamer.emitFill(Padding, 0);
Streamer.EmitLabel(SleepSym);
Streamer.EmitSymbolAttribute(SleepSym, MCSymbolAttr::MCSA_Global);
Streamer.EmitIntValue(opts::InstrumentationSleepTime, /*Size=*/4);
Streamer.EmitLabel(NumLocs);
Streamer.EmitSymbolAttribute(NumLocs, MCSymbolAttr::MCSA_Global);
Streamer.EmitIntValue(Summary->Counters.size(), /*Size=*/4);
Streamer.EmitLabel(Summary->IndCallHandlerFunc);
Streamer.EmitSymbolAttribute(Summary->IndCallHandlerFunc,
MCSymbolAttr::MCSA_Global);
Streamer.EmitValue(
MCSymbolRefExpr::create(
Summary->InitialIndCallHandlerFunction->getSymbol(), *BC.Ctx),
/*Size=*/8);
Streamer.EmitLabel(Summary->IndTailCallHandlerFunc);
Streamer.EmitSymbolAttribute(Summary->IndTailCallHandlerFunc,
MCSymbolAttr::MCSA_Global);
Streamer.EmitValue(
MCSymbolRefExpr::create(
Summary->InitialIndTailCallHandlerFunction->getSymbol(), *BC.Ctx),
/*Size=*/8);
Streamer.EmitLabel(NumIndCalls);
Streamer.EmitSymbolAttribute(NumIndCalls, MCSymbolAttr::MCSA_Global);
Streamer.EmitIntValue(Summary->IndCallDescriptions.size(), /*Size=*/4);
Streamer.EmitLabel(NumIndCallTargets);
Streamer.EmitSymbolAttribute(NumIndCallTargets, MCSymbolAttr::MCSA_Global);
Streamer.EmitIntValue(Summary->IndCallTargetDescriptions.size(), /*Size=*/4);
Streamer.EmitLabel(NumFuncs);
Streamer.EmitSymbolAttribute(NumFuncs, MCSymbolAttr::MCSA_Global);
Streamer.EmitIntValue(Summary->FunctionDescriptions.size(), /*Size=*/4);
Streamer.EmitLabel(FilenameSym);
Streamer.EmitBytes(opts::InstrumentationFilename);
Streamer.emitFill(1, 0);
Streamer.EmitLabel(UsePIDSym);
Streamer.EmitIntValue(opts::InstrumentationFileAppendPID ? 1 : 0, /*Size=*/1);
Streamer.EmitLabel(InitPtr);
Streamer.EmitSymbolAttribute(InitPtr, MCSymbolAttr::MCSA_Global);
Streamer.EmitValue(
MCSymbolRefExpr::create(StartFunction->getSymbol(), *BC.Ctx), /*Size=*/8);
Streamer.EmitLabel(FiniPtr);
Streamer.EmitSymbolAttribute(FiniPtr, MCSymbolAttr::MCSA_Global);
Streamer.EmitValue(
MCSymbolRefExpr::create(FiniFunction->getSymbol(), *BC.Ctx), /*Size=*/8);
}
void InstrumentationRuntimeLibrary::link(BinaryContext &BC, StringRef ToolPath,
orc::ExecutionSession &ES,
orc::RTDyldObjectLinkingLayer &OLT) {
auto LibPath = getLibPath(ToolPath, opts::RuntimeInstrumentationLib);
loadLibraryToOLT(LibPath, ES, OLT);
RuntimeFiniAddress =
cantFail(OLT.findSymbol("__bolt_instr_fini", false).getAddress());
if (!RuntimeFiniAddress) {
errs() << "BOLT-ERROR: instrumentation library does not define "
"__bolt_instr_fini: "
<< LibPath << "\n";
exit(1);
}
RuntimeStartAddress =
cantFail(OLT.findSymbol("__bolt_instr_start", false).getAddress());
if (!RuntimeStartAddress) {
errs() << "BOLT-ERROR: instrumentation library does not define "
"__bolt_instr_start: "
<< LibPath << "\n";
exit(1);
}
outs() << "BOLT-INFO: output linked against instrumentation runtime "
"library, lib entry point is 0x"
<< Twine::utohexstr(RuntimeFiniAddress) << "\n";
outs()
<< "BOLT-INFO: clear procedure is 0x"
<< Twine::utohexstr(cantFail(
OLT.findSymbol("__bolt_instr_clear_counters", false).getAddress()))
<< "\n";
emitTablesAsELFNote(BC);
}
void InstrumentationRuntimeLibrary::emitTablesAsELFNote(BinaryContext &BC) {
std::string TablesStr;
raw_string_ostream OS(TablesStr);
// This is sync'ed with runtime/instr.cpp:readDescriptions()
auto getOutputAddress = [](const BinaryFunction &Func,
uint64_t Offset) -> uint64_t {
return Offset == 0
? Func.getOutputAddress()
: Func.translateInputToOutputAddress(Func.getAddress() + Offset);
};
// Indirect targets need to be sorted for fast lookup during runtime
std::sort(Summary->IndCallTargetDescriptions.begin(),
Summary->IndCallTargetDescriptions.end(),
[&](const IndCallTargetDescription &A,
const IndCallTargetDescription &B) {
return getOutputAddress(*A.Target, A.ToLoc.Offset) <
getOutputAddress(*B.Target, B.ToLoc.Offset);
});
// Start of the vector with descriptions (one CounterDescription for each
// counter), vector size is Counters.size() CounterDescription-sized elmts
const auto IDSize =
Summary->IndCallDescriptions.size() * sizeof(IndCallDescription);
OS.write(reinterpret_cast<const char *>(&IDSize), 4);
for (const auto &Desc : Summary->IndCallDescriptions) {
OS.write(reinterpret_cast<const char *>(&Desc.FromLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Desc.FromLoc.Offset), 4);
}
const auto ITDSize = Summary->IndCallTargetDescriptions.size() *
sizeof(IndCallTargetDescription);
OS.write(reinterpret_cast<const char *>(&ITDSize), 4);
for (const auto &Desc : Summary->IndCallTargetDescriptions) {
OS.write(reinterpret_cast<const char *>(&Desc.ToLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Desc.ToLoc.Offset), 4);
uint64_t TargetFuncAddress =
getOutputAddress(*Desc.Target, Desc.ToLoc.Offset);
OS.write(reinterpret_cast<const char *>(&TargetFuncAddress), 8);
}
auto FuncDescSize = Summary->getFDSize();
OS.write(reinterpret_cast<const char *>(&FuncDescSize), 4);
for (const auto &Desc : Summary->FunctionDescriptions) {
const auto LeafNum = Desc.LeafNodes.size();
OS.write(reinterpret_cast<const char *>(&LeafNum), 4);
for (const auto &LeafNode : Desc.LeafNodes) {
OS.write(reinterpret_cast<const char *>(&LeafNode.Node), 4);
OS.write(reinterpret_cast<const char *>(&LeafNode.Counter), 4);
}
const auto EdgesNum = Desc.Edges.size();
OS.write(reinterpret_cast<const char *>(&EdgesNum), 4);
for (const auto &Edge : Desc.Edges) {
OS.write(reinterpret_cast<const char *>(&Edge.FromLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Edge.FromLoc.Offset), 4);
OS.write(reinterpret_cast<const char *>(&Edge.FromNode), 4);
OS.write(reinterpret_cast<const char *>(&Edge.ToLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Edge.ToLoc.Offset), 4);
OS.write(reinterpret_cast<const char *>(&Edge.ToNode), 4);
OS.write(reinterpret_cast<const char *>(&Edge.Counter), 4);
}
const auto CallsNum = Desc.Calls.size();
OS.write(reinterpret_cast<const char *>(&CallsNum), 4);
for (const auto &Call : Desc.Calls) {
OS.write(reinterpret_cast<const char *>(&Call.FromLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Call.FromLoc.Offset), 4);
OS.write(reinterpret_cast<const char *>(&Call.FromNode), 4);
OS.write(reinterpret_cast<const char *>(&Call.ToLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Call.ToLoc.Offset), 4);
OS.write(reinterpret_cast<const char *>(&Call.Counter), 4);
uint64_t TargetFuncAddress =
getOutputAddress(*Call.Target, Call.ToLoc.Offset);
OS.write(reinterpret_cast<const char *>(&TargetFuncAddress), 8);
}
const auto EntryNum = Desc.EntryNodes.size();
OS.write(reinterpret_cast<const char *>(&EntryNum), 4);
for (const auto &EntryNode : Desc.EntryNodes) {
OS.write(reinterpret_cast<const char *>(&EntryNode.Node), 8);
uint64_t TargetFuncAddress =
getOutputAddress(*Desc.Function, EntryNode.Address);
OS.write(reinterpret_cast<const char *>(&TargetFuncAddress), 8);
}
}
// Our string table lives immediately after descriptions vector
OS << Summary->StringTable;
OS.flush();
const auto BoltInfo = BinarySection::encodeELFNote(
"BOLT", TablesStr, BinarySection::NT_BOLT_INSTRUMENTATION_TABLES);
BC.registerOrUpdateNoteSection(".bolt.instr.tables", copyByteArray(BoltInfo),
BoltInfo.size(),
/*Alignment=*/1,
/*IsReadOnly=*/true, ELF::SHT_NOTE);
}

49
bolt/src/RuntimeLibs/InstrumentationRuntimeLibrary.h Normal file
View File

@ -0,0 +1,49 @@
//===--- InstrumentationRuntimeLibrary.h - The Instrument Runtime Library -===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_TOOLS_LLVM_BOLT_INSTRUMENTATION_RUNTIME_LIBRARY_H
#define LLVM_TOOLS_LLVM_BOLT_INSTRUMENTATION_RUNTIME_LIBRARY_H
#include "Passes/InstrumentationSummary.h"
#include "RuntimeLibs/RuntimeLibrary.h"
namespace llvm {
namespace bolt {
class InstrumentationRuntimeLibrary : public RuntimeLibrary {
public:
InstrumentationRuntimeLibrary(std::unique_ptr<InstrumentationSummary> Summary)
: Summary(std::move(Summary)) {}
/// Add custom section names generated by the runtime libraries to \p
/// SecNames.
void
addRuntimeLibSections(std::vector<std::string> &SecNames) const override {
SecNames.push_back(".bolt.instr.counters");
}
void emitBinary(BinaryContext &BC, MCStreamer &Streamer) override;
void link(BinaryContext &BC, StringRef ToolPath, orc::ExecutionSession &ES,
orc::RTDyldObjectLinkingLayer &OLT) override;
private:
/// Create a non-allocatable ELF section with read-only tables necessary for
/// writing the instrumented data profile during program finish. The runtime
/// library needs to open the program executable file and read this data from
/// disk, this is not loaded by the system.
void emitTablesAsELFNote(BinaryContext &BC);
std::unique_ptr<InstrumentationSummary> Summary;
};
} // namespace bolt
} // namespace llvm
#endif

65
bolt/src/RuntimeLibs/RuntimeLibrary.cpp Normal file
View File

@ -0,0 +1,65 @@
//===--- RuntimeLibrary.cpp - The Runtime Library -------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//===----------------------------------------------------------------------===//
#include "RuntimeLibrary.h"
#include "Utils.h"
#include "llvm/Object/Archive.h"
#include "llvm/Support/Path.h"
#undef DEBUG_TYPE
#define DEBUG_TYPE "bolt-rtlib"
using namespace llvm;
using namespace bolt;
std::string RuntimeLibrary::getLibPath(StringRef ToolPath,
StringRef LibFileName) {
auto Dir = llvm::sys::path::parent_path(ToolPath);
SmallString<128> LibPath = llvm::sys::path::parent_path(Dir);
llvm::sys::path::append(LibPath, "lib", LibFileName);
if (!llvm::sys::fs::exists(LibPath)) {
errs() << "BOLT-ERROR: library not found: " << LibPath << "\n";
exit(1);
}
return LibPath.str();
}
void RuntimeLibrary::loadLibraryToOLT(StringRef LibPath,
orc::ExecutionSession &ES,
orc::RTDyldObjectLinkingLayer &OLT) {
OLT.setProcessAllSections(false);
ErrorOr<std::unique_ptr<MemoryBuffer>> MaybeBuf =
MemoryBuffer::getFile(LibPath, -1, false);
check_error(MaybeBuf.getError(), LibPath);
std::unique_ptr<MemoryBuffer> B = std::move(MaybeBuf.get());
file_magic Magic = identify_magic(B->getBuffer());
if (Magic == file_magic::archive) {
Error Err = Error::success();
object::Archive Archive(B.get()->getMemBufferRef(), Err);
for (auto &C : Archive.children(Err)) {
auto ChildKey = ES.allocateVModule();
auto ChildBuf =
MemoryBuffer::getMemBuffer(cantFail(C.getMemoryBufferRef()));
cantFail(OLT.addObject(ChildKey, std::move(ChildBuf)));
cantFail(OLT.emitAndFinalize(ChildKey));
}
check_error(std::move(Err), B->getBufferIdentifier());
} else if (Magic == file_magic::elf_relocatable ||
Magic == file_magic::elf_shared_object) {
auto K2 = ES.allocateVModule();
cantFail(OLT.addObject(K2, std::move(B)));
cantFail(OLT.emitAndFinalize(K2));
} else {
errs() << "BOLT-ERROR: unrecognized library format: " << LibPath << "\n";
exit(1);
}
}

63
bolt/src/RuntimeLibs/RuntimeLibrary.h Normal file
View File

@ -0,0 +1,63 @@
//===--- RuntimeLibrary.h - The Runtime Library ---------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// Provides all the necessary utilities to link runtime libraries during
// binary rewriting, such as the instrumentation runtime library.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_TOOLS_LLVM_BOLT_LINKRUNTIME_H
#define LLVM_TOOLS_LLVM_BOLT_LINKRUNTIME_H
#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
namespace llvm {
class MCStreamer;
namespace bolt {
class BinaryContext;
class RuntimeLibrary {
public:
virtual ~RuntimeLibrary() = default;
uint64_t getRuntimeFiniAddress() const { return RuntimeFiniAddress; }
uint64_t getRuntimeStartAddress() const { return RuntimeStartAddress; }
/// Add custom sections added by the runtime libraries.
virtual void
addRuntimeLibSections(std::vector<std::string> &SecNames) const = 0;
/// Emit data structures that will be necessary during runtime.
virtual void emitBinary(BinaryContext &BC, MCStreamer &Streamer) = 0;
/// Link with the library code.
virtual void link(BinaryContext &BC, StringRef ToolPath,
orc::ExecutionSession &ES,
orc::RTDyldObjectLinkingLayer &OLT) = 0;
protected:
/// The fini and init address set by the runtime library.
uint64_t RuntimeFiniAddress{0};
uint64_t RuntimeStartAddress{0};
/// Get the full path to a runtime library specified by \p LibFileName.
static std::string getLibPath(StringRef ToolPath, StringRef LibFileName);
/// Load a static runtime library specified by \p LibPath to OLT.
static void loadLibraryToOLT(StringRef LibPath, orc::ExecutionSession &ES,
orc::RTDyldObjectLinkingLayer &OLT);
};
} // namespace bolt
} // namespace llvm
#endif