maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[BOLT] Refactor code and data emission code 

Summary:
Consolidate code and data emission code in ELF-independent
BinaryEmitter. The high-level interface includes only two
functions emitBinaryContext() and emitFunctionBody() used
by RewriteInstance and BinaryContext respectively.

(cherry picked from FBD20332901)
This commit is contained in:
Maksim Panchenko 2020-03-06 15:06:37 -08:00
parent 74a2777c54
commit 1f3e351a9c
14 changed files with 1250 additions and 1076 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
bolt/src/BinaryContext.cpp
View File

@ -10,6 +10,7 @@
//===----------------------------------------------------------------------===//
#include "BinaryContext.h"
#include "BinaryEmitter.h"
#include "BinaryFunction.h"
#include "DataReader.h"
#include "ParallelUtilities.h"
@ -402,8 +403,8 @@ BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF,
/// a reference to its constant island. When emitting this function,
/// we will also emit IslandIter->second's constants. This only
/// happens in custom AArch64 assembly code.
BF.IslandDependency.insert(IslandIter->second);
BF.ProxyIslandSymbols[IslandSym] = IslandIter->second;
BF.Islands.Dependency.insert(IslandIter->second);
BF.Islands.ProxySymbols[IslandSym] = IslandIter->second;
return std::make_pair(IslandSym, Addend);
}
}
@ -1899,9 +1900,9 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
std::unique_ptr<MCStreamer> Streamer(TheTarget->createMCObjectStreamer(
*TheTriple, *LocalCtx, std::unique_ptr<MCAsmBackend>(MAB), VecOS,
std::unique_ptr<MCCodeEmitter>(MCEInstance.MCE.release()), *STI,
/* RelaxAll */ false,
/* IncrementalLinkerCompatible */ false,
/* DWARFMustBeAtTheEnd */ false));
/*RelaxAll=*/false,
/*IncrementalLinkerCompatible=*/false,
/*DWARFMustBeAtTheEnd=*/false));
Streamer->InitSections(false);
@ -1915,7 +1916,8 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
Streamer->SwitchSection(Section);
Streamer->EmitLabel(StartLabel);
BF.emitBody(*Streamer, /*EmitColdPart = */false, /*EmitCodeOnly = */true);
emitFunctionBody(*Streamer, BF, /*EmitColdPart=*/false,
/*EmitCodeOnly=*/true);
Streamer->EmitLabel(EndLabel);
if (BF.isSplit()) {
@ -1927,7 +1929,8 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
Streamer->SwitchSection(ColdSection);
Streamer->EmitLabel(ColdStartLabel);
BF.emitBody(*Streamer, /*EmitColdPart = */true, /*EmitCodeOnly = */true);
emitFunctionBody(*Streamer, BF, /*EmitColdPart=*/true,
/*EmitCodeOnly=*/true);
Streamer->EmitLabel(ColdEndLabel);
}

24
bolt/src/BinaryContext.h
View File

@ -23,6 +23,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/Triple.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
#include "llvm/DebugInfo/DWARF/DWARFContext.h"
#include "llvm/MC/MCAsmBackend.h"
@ -257,6 +258,24 @@ public:
return nullptr;
}
unsigned getDWARFEncodingSize(unsigned Encoding) {
switch (Encoding & 0x0f) {
default: llvm_unreachable("unknown encoding");
case dwarf::DW_EH_PE_absptr:
case dwarf::DW_EH_PE_signed:
return AsmInfo->getCodePointerSize();
case dwarf::DW_EH_PE_udata2:
case dwarf::DW_EH_PE_sdata2:
return 2;
case dwarf::DW_EH_PE_udata4:
case dwarf::DW_EH_PE_sdata4:
return 4;
case dwarf::DW_EH_PE_udata8:
case dwarf::DW_EH_PE_sdata8:
return 8;
}
}
/// [MCSymbol] -> [BinaryFunction]
///
/// As we fold identical functions, multiple symbols can point
@ -675,6 +694,11 @@ public:
ELF::SHF_EXECINSTR | ELF::SHF_ALLOC);
}
/// Return data section with a given name.
MCSection *getDataSection(StringRef SectionName) const {
return Ctx->getELFSection(SectionName, ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
}
/// \name Pre-assigned Section Names
/// @{

1044
bolt/src/BinaryEmitter.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

37
bolt/src/BinaryEmitter.h Normal file
View File

@ -0,0 +1,37 @@
//===--- BinaryEmitter.h - collection of functions to emit code and data --===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_TOOLS_LLVM_BOLT_BINARY_EMITTER_H
#define LLVM_TOOLS_LLVM_BOLT_BINARY_EMITTER_H
#include "BinaryContext.h"
#include "llvm/MC/MCStreamer.h"
namespace llvm {
namespace bolt {
/// Emit all code and data in the BinaryContext \p BC.
///
/// \p OrgSecPrefix is used to modify name of emitted original sections
/// contained in \p BC. This is done to distinguish them from sections emitted
/// by LLVM backend.
void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
StringRef OrgSecPrefix = "");
/// Emit \p BF function code. The caller is responsible for emitting function
/// symbol(s) and setting the section to emit the code to.
void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
bool EmitColdPart, bool EmitCodeOnly = false);
} // namespace bolt
} // namespace llvm
#endif

425
bolt/src/BinaryFunction.cpp
View File

@ -62,26 +62,6 @@ extern cl::opt<bool> StrictMode;
extern cl::opt<bool> UpdateDebugSections;
extern cl::opt<unsigned> Verbosity;
cl::opt<bool>
AlignBlocks("align-blocks",
cl::desc("align basic blocks"),
cl::init(false),
cl::ZeroOrMore,
cl::cat(BoltOptCategory));
cl::opt<MacroFusionType>
AlignMacroOpFusion("align-macro-fusion",
cl::desc("fix instruction alignment for macro-fusion (x86 relocation mode)"),
cl::init(MFT_HOT),
cl::values(clEnumValN(MFT_NONE, "none",
"do not insert alignment no-ops for macro-fusion"),
clEnumValN(MFT_HOT, "hot",
"only insert alignment no-ops on hot execution paths (default)"),
clEnumValN(MFT_ALL, "all",
"always align instructions to allow macro-fusion")),
cl::ZeroOrMore,
cl::cat(BoltRelocCategory));
cl::opt<bool>
CheckEncoding("check-encoding",
cl::desc("perform verification of LLVM instruction encoding/decoding. "
@ -139,13 +119,6 @@ PrintDynoStatsOnly("print-dyno-stats-only",
cl::Hidden,
cl::cat(BoltCategory));
static cl::opt<bool>
PrintJumpTables("print-jump-tables",
cl::desc("print jump tables"),
cl::ZeroOrMore,
cl::Hidden,
cl::cat(BoltCategory));
static cl::list<std::string>
PrintOnly("print-only",
cl::CommaSeparated,
@ -1003,9 +976,9 @@ void BinaryFunction::disassemble() {
const uint64_t AbsoluteInstrAddr = getAddress() + Offset;
// Check for data inside code and ignore it
if (DataOffsets.find(Offset) != DataOffsets.end()) {
auto Iter = CodeOffsets.upper_bound(Offset);
if (Iter != CodeOffsets.end()) {
if (Islands.DataOffsets.find(Offset) != Islands.DataOffsets.end()) {
auto Iter = Islands.CodeOffsets.upper_bound(Offset);
if (Iter != Islands.CodeOffsets.end()) {
Size = *Iter - Offset;
continue;
}
@ -1022,8 +995,8 @@ void BinaryFunction::disassemble() {
// can have 0-byte padding at the end.
bool IsZeroPadding = true;
uint64_t EndOfCode = getSize();
auto Iter = DataOffsets.upper_bound(Offset);
if (Iter != DataOffsets.end())
auto Iter = Islands.DataOffsets.upper_bound(Offset);
if (Iter != Islands.DataOffsets.end())
EndOfCode = *Iter;
for (auto I = Offset; I < EndOfCode; ++I) {
if (FunctionData[I] != 0) {
@ -2652,151 +2625,6 @@ uint64_t BinaryFunction::getEditDistance() const {
BasicBlocksLayout);
}
void BinaryFunction::emitBody(MCStreamer &Streamer, bool EmitColdPart,
bool EmitCodeOnly) {
if (!EmitCodeOnly && EmitColdPart && hasConstantIsland())
duplicateConstantIslands();
// Track first emitted instruction with debug info.
bool FirstInstr = true;
for (auto BB : layout()) {
if (EmitColdPart != BB->isCold())
continue;
if ((opts::AlignBlocks || opts::PreserveBlocksAlignment)
&& BB->getAlignment() > 1) {
Streamer.EmitCodeAlignment(BB->getAlignment(),
BB->getAlignmentMaxBytes());
}
Streamer.EmitLabel(BB->getLabel());
// Check if special alignment for macro-fusion is needed.
bool MayNeedMacroFusionAlignment =
(opts::AlignMacroOpFusion == MFT_ALL) ||
(opts::AlignMacroOpFusion == MFT_HOT &&
BB->getKnownExecutionCount());
BinaryBasicBlock::const_iterator MacroFusionPair;
if (MayNeedMacroFusionAlignment) {
MacroFusionPair = BB->getMacroOpFusionPair();
if (MacroFusionPair == BB->end())
MayNeedMacroFusionAlignment = false;
}
SMLoc LastLocSeen;
// Remember if the last instruction emitted was a prefix.
bool LastIsPrefix = false;
for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
auto &Instr = *I;
if (EmitCodeOnly && BC.MII->get(Instr.getOpcode()).isPseudo())
continue;
// Handle pseudo instructions.
if (BC.MIB->isEHLabel(Instr)) {
const auto *Label = BC.MIB->getTargetSymbol(Instr);
assert(Instr.getNumOperands() >= 1 && Label &&
"bad EH_LABEL instruction");
Streamer.EmitLabel(const_cast<MCSymbol *>(Label));
continue;
}
if (BC.MIB->isCFI(Instr)) {
Streamer.EmitCFIInstruction(*getCFIFor(Instr));
continue;
}
// Handle macro-fusion alignment. If we emitted a prefix as
// the last instruction, we should've already emitted the associated
// alignment hint, so don't emit it twice.
if (MayNeedMacroFusionAlignment && !LastIsPrefix && I == MacroFusionPair){
// This assumes the second instruction in the macro-op pair will get
// assigned to its own MCRelaxableFragment. Since all JCC instructions
// are relaxable, we should be safe.
Streamer.EmitNeverAlignCodeAtEnd(/*Alignment to avoid=*/64);
}
if (!EmitCodeOnly && opts::UpdateDebugSections && UnitLineTable.first) {
LastLocSeen = emitLineInfo(Instr.getLoc(), LastLocSeen, FirstInstr);
FirstInstr = false;
}
// Prepare to tag this location with a label if we need to keep track of
// the location of calls/returns for BOLT address translation maps
if (!EmitCodeOnly && requiresAddressTranslation() &&
BC.MIB->hasAnnotation(Instr, "Offset")) {
const auto Offset = BC.MIB->getAnnotationAs<uint32_t>(Instr, "Offset");
MCSymbol *LocSym = BC.Ctx->createTempSymbol(/*CanBeUnnamed=*/true);
Streamer.EmitLabel(LocSym);
BB->getLocSyms().emplace_back(std::make_pair(Offset, LocSym));
}
Streamer.EmitInstruction(Instr, *BC.STI);
LastIsPrefix = BC.MIB->isPrefix(Instr);
}
}
if (!EmitCodeOnly)
emitConstantIslands(Streamer, EmitColdPart);
}
void BinaryFunction::emitBodyRaw(MCStreamer *Streamer) {
// #14998851: Fix gold linker's '--emit-relocs'.
assert(false &&
"cannot emit raw body unless relocation accuracy is guaranteed");
assert(!isInjected() && "cannot emit raw body of injected function");
// Raw contents of the function.
StringRef SectionContents = InputSection->getContents();
// Raw contents of the function.
StringRef FunctionContents =
SectionContents.substr(getAddress() - InputSection->getAddress(),
getSize());
if (opts::Verbosity)
outs() << "BOLT-INFO: emitting function " << *this << " in raw ("
<< getSize() << " bytes).\n";
// We split the function blob into smaller blocks and output relocations
// and/or labels between them.
uint64_t FunctionOffset = 0;
auto LI = Labels.begin();
auto RI = MoveRelocations.begin();
while (LI != Labels.end() ||
RI != MoveRelocations.end()) {
uint64_t NextLabelOffset = (LI == Labels.end() ? getSize() : LI->first);
uint64_t NextRelocationOffset =
(RI == MoveRelocations.end() ? getSize() : RI->first);
auto NextStop = std::min(NextLabelOffset, NextRelocationOffset);
assert(NextStop <= getSize() && "internal overflow error");
if (FunctionOffset < NextStop) {
Streamer->EmitBytes(
FunctionContents.slice(FunctionOffset, NextStop));
FunctionOffset = NextStop;
}
if (LI != Labels.end() && FunctionOffset == LI->first) {
Streamer->EmitLabel(LI->second);
DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << LI->second->getName()
<< " at offset 0x" << Twine::utohexstr(LI->first) << '\n');
++LI;
}
if (RI != MoveRelocations.end() && FunctionOffset == RI->first) {
auto RelocationSize = RI->second.emit(Streamer);
DEBUG(dbgs() << "BOLT-DEBUG: emitted relocation for symbol "
<< RI->second.Symbol->getName() << " at offset 0x"
<< Twine::utohexstr(RI->first)
<< " with size " << RelocationSize << '\n');
FunctionOffset += RelocationSize;
++RI;
}
}
assert(FunctionOffset <= getSize() && "overflow error");
if (FunctionOffset < getSize()) {
Streamer->EmitBytes(FunctionContents.substr(FunctionOffset));
}
}
void BinaryFunction::setTrapOnEntry() {
clearList(Instructions);
clearList(IgnoredBranches);
@ -2811,133 +2639,6 @@ void BinaryFunction::setTrapOnEntry() {
TrapsOnEntry = true;
}
void BinaryFunction::emitConstantIslands(
MCStreamer &Streamer, bool EmitColdPart,
BinaryFunction *OnBehalfOf) {
if (DataOffsets.empty() && IslandDependency.empty())
return;
if (!OnBehalfOf) {
if (!EmitColdPart)
Streamer.EmitLabel(getFunctionConstantIslandLabel());
else
Streamer.EmitLabel(getFunctionColdConstantIslandLabel());
}
assert((!OnBehalfOf || IslandProxies[OnBehalfOf].size() > 0) &&
"spurious OnBehalfOf constant island emission");
assert(!isInjected() &&
"injected functions should not have constant islands");
// Raw contents of the function.
StringRef SectionContents = InputSection->getContents();
// Raw contents of the function.
StringRef FunctionContents =
SectionContents.substr(getAddress() - InputSection->getAddress(),
getMaxSize());
if (opts::Verbosity && !OnBehalfOf)
outs() << "BOLT-INFO: emitting constant island for function " << *this
<< "\n";
// We split the island into smaller blocks and output labels between them.
auto IS = IslandOffsets.begin();
for (auto DataIter = DataOffsets.begin(); DataIter != DataOffsets.end();
++DataIter) {
uint64_t FunctionOffset = *DataIter;
uint64_t EndOffset = 0ULL;
// Determine size of this data chunk
auto NextData = std::next(DataIter);
auto CodeIter = CodeOffsets.lower_bound(*DataIter);
if (CodeIter == CodeOffsets.end() && NextData == DataOffsets.end()) {
EndOffset = getMaxSize();
} else if (CodeIter == CodeOffsets.end()) {
EndOffset = *NextData;
} else if (NextData == DataOffsets.end()) {
EndOffset = *CodeIter;
} else {
EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter;
}
if (FunctionOffset == EndOffset)
continue; // Size is zero, nothing to emit
// Emit labels, relocs and data
auto RI = MoveRelocations.lower_bound(FunctionOffset);
while ((IS != IslandOffsets.end() && IS->first < EndOffset) ||
(RI != MoveRelocations.end() && RI->first < EndOffset)) {
auto NextLabelOffset = IS == IslandOffsets.end() ? EndOffset : IS->first;
auto NextRelOffset = RI == MoveRelocations.end() ? EndOffset : RI->first;
auto NextStop = std::min(NextLabelOffset, NextRelOffset);
assert(NextStop <= EndOffset && "internal overflow error");
if (FunctionOffset < NextStop) {
Streamer.EmitBytes(FunctionContents.slice(FunctionOffset, NextStop));
FunctionOffset = NextStop;
}
if (IS != IslandOffsets.end() && FunctionOffset == IS->first) {
// This is a slightly complex code to decide which label to emit. We
// have 4 cases to handle: regular symbol, cold symbol, regular or cold
// symbol being emitted on behalf of an external function.
if (!OnBehalfOf) {
if (!EmitColdPart) {
DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
<< IS->second->getName() << " at offset 0x"
<< Twine::utohexstr(IS->first) << '\n');
if (IS->second->isUndefined())
Streamer.EmitLabel(IS->second);
else
assert(hasName(IS->second->getName()));
} else if (ColdIslandSymbols.count(IS->second) != 0) {
DEBUG(dbgs() << "BOLT-DEBUG: emitted label "
<< ColdIslandSymbols[IS->second]->getName() << '\n');
if (ColdIslandSymbols[IS->second]->isUndefined())
Streamer.EmitLabel(ColdIslandSymbols[IS->second]);
}
} else {
if (!EmitColdPart) {
if (MCSymbol *Sym = IslandProxies[OnBehalfOf][IS->second]) {
DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName()
<< '\n');
Streamer.EmitLabel(Sym);
}
} else if (MCSymbol *Sym =
ColdIslandProxies[OnBehalfOf][IS->second]) {
DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName()
<< '\n');
Streamer.EmitLabel(Sym);
}
}
++IS;
}
if (RI != MoveRelocations.end() && FunctionOffset == RI->first) {
auto RelocationSize = RI->second.emit(&Streamer);
DEBUG(dbgs() << "BOLT-DEBUG: emitted relocation for symbol "
<< RI->second.Symbol->getName() << " at offset 0x"
<< Twine::utohexstr(RI->first)
<< " with size " << RelocationSize << '\n');
FunctionOffset += RelocationSize;
++RI;
}
}
assert(FunctionOffset <= EndOffset && "overflow error");
if (FunctionOffset < EndOffset) {
Streamer.EmitBytes(FunctionContents.slice(FunctionOffset, EndOffset));
}
}
assert(IS == IslandOffsets.end() && "some symbols were not emitted!");
if (OnBehalfOf)
return;
// Now emit constant islands from other functions that we may have used in
// this function.
for (auto *ExternalFunc : IslandDependency) {
ExternalFunc->emitConstantIslands(Streamer, EmitColdPart, this);
}
}
void BinaryFunction::duplicateConstantIslands() {
for (auto BB : layout()) {
if (!BB->isCold())
@ -2952,22 +2653,24 @@ void BinaryFunction::duplicateConstantIslands() {
}
auto *Symbol = BC.MIB->getTargetSymbol(Inst, OpNum);
// Check if this is an island symbol
if (!IslandSymbols.count(Symbol) && !ProxyIslandSymbols.count(Symbol))
if (!Islands.Symbols.count(Symbol) &&
!Islands.ProxySymbols.count(Symbol))
continue;
// Create cold symbol, if missing
auto ISym = ColdIslandSymbols.find(Symbol);
auto ISym = Islands.ColdSymbols.find(Symbol);
MCSymbol *ColdSymbol;
if (ISym != ColdIslandSymbols.end()) {
if (ISym != Islands.ColdSymbols.end()) {
ColdSymbol = ISym->second;
} else {
ColdSymbol = BC.Ctx->getOrCreateSymbol(Symbol->getName() + ".cold");
ColdIslandSymbols[Symbol] = ColdSymbol;
Islands.ColdSymbols[Symbol] = ColdSymbol;
// Check if this is a proxy island symbol and update owner proxy map
if (ProxyIslandSymbols.count(Symbol)) {
BinaryFunction *Owner = ProxyIslandSymbols[Symbol];
auto IProxiedSym = Owner->IslandProxies[this].find(Symbol);
Owner->ColdIslandProxies[this][IProxiedSym->second] = ColdSymbol;
if (Islands.ProxySymbols.count(Symbol)) {
BinaryFunction *Owner = Islands.ProxySymbols[Symbol];
auto IProxiedSym = Owner->Islands.Proxies[this].find(Symbol);
Owner->Islands.ColdProxies[this][IProxiedSym->second] =
ColdSymbol;
}
}
@ -3891,63 +3594,6 @@ bool BinaryFunction::isSymbolValidInScope(const SymbolRef &Symbol,
return true;
}
SMLoc BinaryFunction::emitLineInfo(SMLoc NewLoc, SMLoc PrevLoc,
bool FirstInstr) const {
auto *FunctionCU = UnitLineTable.first;
const auto *FunctionLineTable = UnitLineTable.second;
assert(FunctionCU && "cannot emit line info for function without CU");
auto RowReference = DebugLineTableRowRef::fromSMLoc(NewLoc);
// Check if no new line info needs to be emitted.
if (RowReference == DebugLineTableRowRef::NULL_ROW ||
NewLoc.getPointer() == PrevLoc.getPointer())
return PrevLoc;
unsigned CurrentFilenum = 0;
const auto *CurrentLineTable = FunctionLineTable;
// If the CU id from the current instruction location does not
// match the CU id from the current function, it means that we
// have come across some inlined code. We must look up the CU
// for the instruction's original function and get the line table
// from that.
const auto FunctionUnitIndex = FunctionCU->getOffset();
const auto CurrentUnitIndex = RowReference.DwCompileUnitIndex;
if (CurrentUnitIndex != FunctionUnitIndex) {
CurrentLineTable = BC.DwCtx->getLineTableForUnit(
BC.DwCtx->getCompileUnitForOffset(CurrentUnitIndex));
// Add filename from the inlined function to the current CU.
CurrentFilenum =
BC.addDebugFilenameToUnit(FunctionUnitIndex, CurrentUnitIndex,
CurrentLineTable->Rows[RowReference.RowIndex - 1].File);
}
const auto &CurrentRow = CurrentLineTable->Rows[RowReference.RowIndex - 1];
if (!CurrentFilenum)
CurrentFilenum = CurrentRow.File;
unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) |
(DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) |
(DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) |
(DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin);
// Always emit is_stmt at the beginning of function fragment.
if (FirstInstr)
Flags |= DWARF2_FLAG_IS_STMT;
BC.Ctx->setCurrentDwarfLoc(
CurrentFilenum,
CurrentRow.Line,
CurrentRow.Column,
Flags,
CurrentRow.Isa,
CurrentRow.Discriminator);
BC.Ctx->setDwarfCompileUnitID(FunctionUnitIndex);
return NewLoc;
}
void BinaryFunction::adjustExecutionCount(uint64_t Count) {
if (getKnownExecutionCount() == 0 || Count == 0)
return;
@ -3974,47 +3620,6 @@ BinaryFunction::~BinaryFunction() {
}
}
void BinaryFunction::emitJumpTables(MCStreamer *Streamer) {
if (JumpTables.empty())
return;
if (opts::PrintJumpTables) {
outs() << "BOLT-INFO: jump tables for function " << *this << ":\n";
}
for (auto &JTI : JumpTables) {
auto &JT = *JTI.second;
if (opts::PrintJumpTables)
JT.print(outs());
if ((opts::JumpTables == JTS_BASIC || !isSimple()) && BC.HasRelocations) {
JT.updateOriginal();
} else {
MCSection *HotSection, *ColdSection;
if (opts::JumpTables == JTS_BASIC) {
std::string Name = ".local." + JT.Labels[0]->getName().str();
std::replace(Name.begin(), Name.end(), '/', '.');
auto &Section = BC.registerOrUpdateSection(Name,
ELF::SHT_PROGBITS,
ELF::SHF_ALLOC);
Section.setAnonymous(true);
JT.setOutputSection(Section);
HotSection = BC.Ctx->getELFSection(Name,
ELF::SHT_PROGBITS,
ELF::SHF_ALLOC);
ColdSection = HotSection;
} else {
if (isSimple()) {
HotSection = BC.MOFI->getReadOnlySection();
ColdSection = BC.MOFI->getReadOnlyColdSection();
} else {
HotSection = hasProfile() ? BC.MOFI->getReadOnlySection()
: BC.MOFI->getReadOnlyColdSection();
ColdSection = HotSection;
}
}
JT.emit(Streamer, HotSection, ColdSection);
}
}
}
void BinaryFunction::calculateLoopInfo() {
// Discover loops.
BinaryDominatorTree DomTree;

185
bolt/src/BinaryFunction.h
View File

@ -135,6 +135,39 @@ public:
PF_MEMEVENT = 4, /// Profile has mem events.
};
/// Struct for tracking exception handling ranges.
struct CallSite {
const MCSymbol *Start;
const MCSymbol *End;
const MCSymbol *LP;
uint64_t Action;
};
using IslandProxiesType =
std::map<BinaryFunction *, std::map<const MCSymbol *, MCSymbol *>>;
struct IslandInfo {
/// Temporary holder of offsets that are data markers (used in AArch)
/// It is possible to have data in code sections. To ease the identification
/// of data in code sections, the ABI requires the symbol table to have
/// symbols named "$d" identifying the start of data inside code and "$x"
/// identifying the end of a chunk of data inside code. DataOffsets contain
/// all offsets of $d symbols and CodeOffsets all offsets of $x symbols.
std::set<uint64_t> DataOffsets;
std::set<uint64_t> CodeOffsets;
/// Offsets in function that are data values in a constant island identified
/// after disassembling
std::map<uint64_t, MCSymbol *> Offsets;
SmallPtrSet<MCSymbol *, 4> Symbols;
std::map<const MCSymbol *, BinaryFunction *> ProxySymbols;
std::map<const MCSymbol *, MCSymbol *> ColdSymbols;
/// Keeps track of other functions we depend on because there is a reference
/// to the constant islands in them.
IslandProxiesType Proxies, ColdProxies;
std::set<BinaryFunction *> Dependency; // The other way around
};
static constexpr uint64_t COUNT_NO_PROFILE =
BinaryBasicBlock::COUNT_NO_PROFILE;
@ -372,14 +405,6 @@ private:
/// Temporary holder of offsets that are potentially entry points.
std::unordered_set<uint64_t> EntryOffsets;
/// Temporary holder of offsets that are data markers (used in AArch)
/// It is possible to have data in code sections. To ease the identification
/// of data in code sections, the ABI requires the symbol table to have
/// symbols named "$d" identifying the start of data inside code and "$x"
/// identifying the end of a chunk of data inside code. DataOffsets contain
/// all offsets of $d symbols and CodeOffsets all offsets of $x symbols.
std::set<uint64_t> DataOffsets;
std::set<uint64_t> CodeOffsets;
/// The address offset where we emitted the constant island, that is, the
/// chunk of data in the function code area (AArch only)
int64_t OutputDataOffset{0};
@ -439,13 +464,7 @@ private:
/// remember-restore CFI instructions when rewriting CFI.
DenseMap<int32_t , SmallVector<int32_t, 4>> FrameRestoreEquivalents;
/// Exception handling ranges.
struct CallSite {
const MCSymbol *Start;
const MCSymbol *End;
const MCSymbol *LP;
uint64_t Action;
};
// For tracking exception handling ranges.
std::vector<CallSite> CallSites;
std::vector<CallSite> ColdCallSites;
@ -475,12 +494,6 @@ private:
/// <OriginalAddress> -> <JumpTable *>
std::map<uint64_t, JumpTable *> JumpTables;
/// Iterate over all jump tables associated with this function.
iterator_range<std::map<uint64_t, JumpTable *>::const_iterator>
jumpTables() const {
return make_range(JumpTables.begin(), JumpTables.end());
}
/// All jump table sites in the function before CFG is built.
std::vector<std::pair<uint64_t, uint64_t>> JTSites;
@ -488,19 +501,11 @@ private:
std::map<uint64_t, Relocation> Relocations;
/// Map of relocations used for moving the function body as it is.
std::map<uint64_t, Relocation> MoveRelocations;
using MoveRelocationsTy = std::map<uint64_t, Relocation>;
MoveRelocationsTy MoveRelocations;
/// Offsets in function that are data values in a constant island identified
/// after disassembling
std::map<uint64_t, MCSymbol *> IslandOffsets;
SmallPtrSet<MCSymbol *, 4> IslandSymbols;
std::map<const MCSymbol *, BinaryFunction *> ProxyIslandSymbols;
std::map<const MCSymbol *, MCSymbol *> ColdIslandSymbols;
/// Keeps track of other functions we depend on because there is a reference
/// to the constant islands in them.
std::map<BinaryFunction *, std::map<const MCSymbol *, MCSymbol *>>
IslandProxies, ColdIslandProxies;
std::set<BinaryFunction *> IslandDependency; // The other way around
/// Information on function constant islands.
IslandInfo Islands;
// Blocks are kept sorted in the layout order. If we need to change the
// layout (if BasicBlocksLayout stores a different order than BasicBlocks),
@ -561,12 +566,12 @@ private:
/// Register an entry point at a given \p Offset into the function.
void markDataAtOffset(uint64_t Offset) {
DataOffsets.emplace(Offset);
Islands.DataOffsets.emplace(Offset);
}
/// Register an entry point at a given \p Offset into the function.
void markCodeAtOffset(uint64_t Offset) {
CodeOffsets.emplace(Offset);
Islands.CodeOffsets.emplace(Offset);
}
/// Register an entry point at a given \p Offset into the function.
@ -637,14 +642,6 @@ private:
DenseMap<const MCInst *, SmallVector<MCInst *, 4>>
computeLocalUDChain(const MCInst *CurInstr);
/// Emit line number information corresponding to \p NewLoc. \p PrevLoc
/// provides a context for de-duplication of line number info.
/// \p FirstInstr indicates if \p NewLoc represents the first instruction
/// in a sequence, such as a function fragment.
///
/// Return new current location which is either \p NewLoc or \p PrevLoc.
SMLoc emitLineInfo(SMLoc NewLoc, SMLoc PrevLoc, bool FirstInstr) const;
BinaryFunction& operator=(const BinaryFunction &) = delete;
BinaryFunction(const BinaryFunction &) = delete;
@ -776,6 +773,12 @@ public:
return iterator_range<const_cfi_iterator>(cie_begin(), cie_end());
}
/// Iterate over all jump tables associated with this function.
iterator_range<std::map<uint64_t, JumpTable *>::const_iterator>
jumpTables() const {
return make_range(JumpTables.begin(), JumpTables.end());
}
/// Returns the raw binary encoding of this function.
ErrorOr<ArrayRef<uint8_t>> getData() const;
@ -1360,6 +1363,42 @@ public:
return PersonalityEncoding;
}
const std::vector<CallSite> &getCallSites() const {
return CallSites;
}
const std::vector<CallSite> &getColdCallSites() const {
return ColdCallSites;
}
const ArrayRef<uint8_t> getLSDAActionTable() const {
return LSDAActionTable;
}
const std::vector<uint64_t> &getLSDATypeTable() const {
return LSDATypeTable;
}
const ArrayRef<uint8_t> getLSDATypeIndexTable() const {
return LSDATypeIndexTable;
}
const MoveRelocationsTy &getMoveRelocations() const {
return MoveRelocations;
}
const LabelsMapType &getLabels() const {
return Labels;
}
IslandInfo &getIslandInfo() {
return Islands;
}
const IslandInfo &getIslandInfo() const {
return Islands;
}
/// Return true if the function has CFI instructions
bool hasCFI() const {
return !FrameInstructions.empty() || !CIEFrameInstructions.empty();
@ -1873,11 +1912,12 @@ public:
// Internal bookkeeping
const auto Offset = Address - getAddress();
assert((!IslandOffsets.count(Offset) || IslandOffsets[Offset] == Symbol) &&
assert((!Islands.Offsets.count(Offset) ||
Islands.Offsets[Offset] == Symbol) &&
"Inconsistent island symbol management");
if (!IslandOffsets.count(Offset)) {
IslandOffsets[Offset] = Symbol;
IslandSymbols.insert(Symbol);
if (!Islands.Offsets.count(Offset)) {
Islands.Offsets[Offset] = Symbol;
Islands.Symbols.insert(Symbol);
}
return Symbol;
}
@ -1893,14 +1933,14 @@ public:
return nullptr;
MCSymbol *Proxy;
if (!IslandProxies[&Referrer].count(Symbol)) {
if (!Islands.Proxies[&Referrer].count(Symbol)) {
Proxy =
BC.Ctx->getOrCreateSymbol(Symbol->getName() +
".proxy.for." + Referrer.getPrintName());
IslandProxies[&Referrer][Symbol] = Proxy;
IslandProxies[&Referrer][Proxy] = Symbol;
Islands.Proxies[&Referrer][Symbol] = Proxy;
Islands.Proxies[&Referrer][Proxy] = Symbol;
}
Proxy = IslandProxies[&Referrer][Symbol];
Proxy = Islands.Proxies[&Referrer][Symbol];
return Proxy;
}
@ -1915,13 +1955,13 @@ public:
if (Offset >= getMaxSize())
return false;
auto DataIter = DataOffsets.upper_bound(Offset);
if (DataIter == DataOffsets.begin())
auto DataIter = Islands.DataOffsets.upper_bound(Offset);
if (DataIter == Islands.DataOffsets.begin())
return false;
DataIter = std::prev(DataIter);
auto CodeIter = CodeOffsets.upper_bound(Offset);
if (CodeIter == CodeOffsets.begin())
auto CodeIter = Islands.CodeOffsets.upper_bound(Offset);
if (CodeIter == Islands.CodeOffsets.begin())
return true;
return *std::prev(CodeIter) <= *DataIter;
@ -1930,20 +1970,21 @@ public:
uint64_t
estimateConstantIslandSize(const BinaryFunction *OnBehalfOf = nullptr) const {
uint64_t Size = 0;
for (auto DataIter = DataOffsets.begin(); DataIter != DataOffsets.end();
for (auto DataIter = Islands.DataOffsets.begin();
DataIter != Islands.DataOffsets.end();
++DataIter) {
auto NextData = std::next(DataIter);
auto CodeIter = CodeOffsets.lower_bound(*DataIter);
if (CodeIter == CodeOffsets.end() &&
NextData == DataOffsets.end()) {
auto CodeIter = Islands.CodeOffsets.lower_bound(*DataIter);
if (CodeIter == Islands.CodeOffsets.end() &&
NextData == Islands.DataOffsets.end()) {
Size += getMaxSize() - *DataIter;
continue;
}
uint64_t NextMarker;
if (CodeIter == CodeOffsets.end())
if (CodeIter == Islands.CodeOffsets.end())
NextMarker = *NextData;
else if (NextData == DataOffsets.end())
else if (NextData == Islands.DataOffsets.end())
NextMarker = *CodeIter;
else
NextMarker = (*CodeIter > *NextData) ? *NextData : *CodeIter;
@ -1952,14 +1993,14 @@ public:
}
if (!OnBehalfOf) {
for (auto *ExternalFunc : IslandDependency)
for (auto *ExternalFunc : Islands.Dependency)
Size += ExternalFunc->estimateConstantIslandSize(this);
}
return Size;
}
bool hasConstantIsland() const {
return !DataOffsets.empty();
return !Islands.DataOffsets.empty();
}
/// Return true iff the symbol could be seen inside this function otherwise
@ -2184,24 +2225,6 @@ public:
/// Update exception handling ranges for the function.
void updateEHRanges();
/// Emit exception handling ranges for the function.
void emitLSDA(MCStreamer *Streamer, bool EmitColdPart);
/// Emit jump tables for the function.
void emitJumpTables(MCStreamer *Streamer);
/// Emit function code. The caller is responsible for emitting function
/// symbol(s) and setting the section to emit the code to.
void emitBody(MCStreamer &Streamer, bool EmitColdPart,
bool EmitCodeOnly = false);
/// Emit function as a blob with relocations and labels for relocations.
void emitBodyRaw(MCStreamer *Streamer);
/// Helper for emitBody to write data inside a function (used for AArch64)
void emitConstantIslands(MCStreamer &Streamer, bool EmitColdPart,
BinaryFunction *OnBehalfOf = nullptr);
/// Traverse cold basic blocks and replace references to constants in islands
/// with a proxy symbol for the duplicated constant island that is going to be
/// emitted in the cold region.

1
bolt/src/CMakeLists.txt
View File

@ -76,6 +76,7 @@ add_llvm_tool(llvm-bolt
BinaryBasicBlock.cpp
BinaryContext.cpp
BinaryData.cpp
BinaryEmitter.cpp
BinaryFunction.cpp
BinaryFunctionProfile.cpp
BinaryPassManager.cpp

58
bolt/src/DWARFRewriter.cpp
View File

@ -369,64 +369,6 @@ void DWARFRewriter::updateDWARFObjectAddressRanges(
}
}
void DWARFRewriter::updateDebugLineInfoForNonSimpleFunctions() {
for (auto &It : BC.getBinaryFunctions()) {
const auto &Function = It.second;
if (Function.isSimple())
continue;
auto ULT = Function.getDWARFUnitLineTable();
auto Unit = ULT.first;
auto LineTable = ULT.second;
if (!LineTable)
continue; // nothing to update for this function
std::vector<uint32_t> Results;
MCSectionELF *FunctionSection =
BC.Ctx->getELFSection(Function.getCodeSectionName(),
ELF::SHT_PROGBITS,
ELF::SHF_EXECINSTR | ELF::SHF_ALLOC);
uint64_t Address = It.first;
if (LineTable->lookupAddressRange(Address, Function.getMaxSize(),
Results)) {
auto &OutputLineTable =
BC.Ctx->getMCDwarfLineTable(Unit->getOffset()).getMCLineSections();
for (auto RowIndex : Results) {
const auto &Row = LineTable->Rows[RowIndex];
BC.Ctx->setCurrentDwarfLoc(
Row.File,
Row.Line,
Row.Column,
(DWARF2_FLAG_IS_STMT * Row.IsStmt) |
(DWARF2_FLAG_BASIC_BLOCK * Row.BasicBlock) |
(DWARF2_FLAG_PROLOGUE_END * Row.PrologueEnd) |
(DWARF2_FLAG_EPILOGUE_BEGIN * Row.EpilogueBegin),
Row.Isa,
Row.Discriminator,
Row.Address);
auto Loc = BC.Ctx->getCurrentDwarfLoc();
BC.Ctx->clearDwarfLocSeen();
OutputLineTable.addLineEntry(MCDwarfLineEntry{nullptr, Loc},
FunctionSection);
}
// Add an empty entry past the end of the function
// for end_sequence mark.
BC.Ctx->setCurrentDwarfLoc(0, 0, 0, 0, 0, 0,
Address + Function.getMaxSize());
auto Loc = BC.Ctx->getCurrentDwarfLoc();
BC.Ctx->clearDwarfLocSeen();
OutputLineTable.addLineEntry(MCDwarfLineEntry{nullptr, Loc},
FunctionSection);
} else {
DEBUG(dbgs() << "BOLT-DEBUG: Function " << Function
<< " has no associated line number information.\n");
}
}
}
void DWARFRewriter::updateLineTableOffsets() {
const auto *LineSection =
BC.Ctx->getObjectFileInfo()->getDwarfLineSection();

4
bolt/src/DWARFRewriter.h
View File

@ -134,10 +134,6 @@ public:
/// Computes output .debug_line line table offsets for each compile unit,
/// and updates stmt_list for a corresponding compile unit.
void updateLineTableOffsets();
/// Updates debug line information for non-simple functions, which are not
/// rewritten.
void updateDebugLineInfoForNonSimpleFunctions();
};
} // namespace bolt

178
bolt/src/Exceptions.cpp
View File

@ -54,28 +54,6 @@ PrintExceptions("print-exceptions",
namespace llvm {
namespace bolt {
namespace {
unsigned getEncodingSize(unsigned Encoding, BinaryContext &BC) {
switch (Encoding & 0x0f) {
default: llvm_unreachable("unknown encoding");
case dwarf::DW_EH_PE_absptr:
case dwarf::DW_EH_PE_signed:
return BC.AsmInfo->getCodePointerSize();
case dwarf::DW_EH_PE_udata2:
case dwarf::DW_EH_PE_sdata2:
return 2;
case dwarf::DW_EH_PE_udata4:
case dwarf::DW_EH_PE_sdata4:
return 4;
case dwarf::DW_EH_PE_udata8:
case dwarf::DW_EH_PE_sdata8:
return 8;
}
}
} // anonymous namespace
// Read and dump the .gcc_exception_table section entry.
//
// .gcc_except_table section contains a set of Language-Specific Data Areas -
@ -153,7 +131,7 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
uintptr_t TTypeEnd = 0;
if (TTypeEncoding != DW_EH_PE_omit) {
TTypeEnd = Data.getULEB128(&Offset);
TTypeEncodingSize = getEncodingSize(TTypeEncoding, BC);
TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding);
}
if (opts::PrintExceptions) {
@ -486,160 +464,6 @@ void BinaryFunction::updateEHRanges() {
}
}
// The code is based on EHStreamer::emitExceptionTable().
void BinaryFunction::emitLSDA(MCStreamer *Streamer, bool EmitColdPart) {
const auto *Sites = EmitColdPart ? &ColdCallSites : &CallSites;
if (Sites->empty()) {
return;
}
// Calculate callsite table size. Size of each callsite entry is:
//
// sizeof(start) + sizeof(length) + sizeof(LP) + sizeof(uleb128(action))
//
// or
//
// sizeof(dwarf::DW_EH_PE_data4) * 3 + sizeof(uleb128(action))
uint64_t CallSiteTableLength = Sites->size() * 4 * 3;
for (const auto &CallSite : *Sites) {
CallSiteTableLength += getULEB128Size(CallSite.Action);
}
Streamer->SwitchSection(BC.MOFI->getLSDASection());
const auto TTypeEncoding = BC.MOFI->getTTypeEncoding();
const auto TTypeEncodingSize = getEncodingSize(TTypeEncoding, BC);
const auto TTypeAlignment = 4;
// Type tables have to be aligned at 4 bytes.
Streamer->EmitValueToAlignment(TTypeAlignment);
// Emit the LSDA label.
auto LSDASymbol = EmitColdPart ? getColdLSDASymbol() : getLSDASymbol();
assert(LSDASymbol && "no LSDA symbol set");
Streamer->EmitLabel(LSDASymbol);
// Corresponding FDE start.
const auto *StartSymbol = EmitColdPart ? getColdSymbol() : getSymbol();
// Emit the LSDA header.
// If LPStart is omitted, then the start of the FDE is used as a base for
// landing pad displacements. Then if a cold fragment starts with
// a landing pad, this means that the first landing pad offset will be 0.
// As a result, an exception handling runtime will ignore this landing pad,
// because zero offset denotes the absence of a landing pad.
// For this reason, we emit LPStart value of 0 and output an absolute value
// of the landing pad in the table.
//
// FIXME: this may break PIEs and DSOs where the base address is not 0.
Streamer->EmitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format
Streamer->EmitIntValue(0, 4);
auto emitLandingPad = [&](const MCSymbol *LPSymbol) {
if (!LPSymbol) {
Streamer->EmitIntValue(0, 4);
return;
}
Streamer->EmitSymbolValue(LPSymbol, 4);
};
Streamer->EmitIntValue(TTypeEncoding, 1); // TType format
// See the comment in EHStreamer::emitExceptionTable() on to use
// uleb128 encoding (which can use variable number of bytes to encode the same
// value) to ensure type info table is properly aligned at 4 bytes without
// iteratively fixing sizes of the tables.
unsigned CallSiteTableLengthSize = getULEB128Size(CallSiteTableLength);
unsigned TTypeBaseOffset =
sizeof(int8_t) + // Call site format
CallSiteTableLengthSize + // Call site table length size
CallSiteTableLength + // Call site table length
LSDAActionTable.size() + // Actions table size
LSDATypeTable.size() * TTypeEncodingSize; // Types table size
unsigned TTypeBaseOffsetSize = getULEB128Size(TTypeBaseOffset);
unsigned TotalSize =
sizeof(int8_t) + // LPStart format
sizeof(int8_t) + // TType format
TTypeBaseOffsetSize + // TType base offset size
TTypeBaseOffset; // TType base offset
unsigned SizeAlign = (4 - TotalSize) & 3;
// Account for any extra padding that will be added to the call site table
// length.
Streamer->EmitPaddedULEB128IntValue(TTypeBaseOffset,
TTypeBaseOffsetSize + SizeAlign);
// Emit the landing pad call site table. We use signed data4 since we can emit
// a landing pad in a different part of the split function that could appear
// earlier in the address space than LPStart.
Streamer->EmitIntValue(dwarf::DW_EH_PE_sdata4, 1);
Streamer->EmitULEB128IntValue(CallSiteTableLength);
for (const auto &CallSite : *Sites) {
const auto *BeginLabel = CallSite.Start;
const auto *EndLabel = CallSite.End;
assert(BeginLabel && "start EH label expected");
assert(EndLabel && "end EH label expected");
// Start of the range is emitted relative to the start of current
// function split part.
Streamer->emitAbsoluteSymbolDiff(BeginLabel, StartSymbol, 4);
Streamer->emitAbsoluteSymbolDiff(EndLabel, BeginLabel, 4);
emitLandingPad(CallSite.LP);
Streamer->EmitULEB128IntValue(CallSite.Action);
}
// Write out action, type, and type index tables at the end.
//
// For action and type index tables there's no need to change the original
// table format unless we are doing function splitting, in which case we can
// split and optimize the tables.
//
// For type table we (re-)encode the table using TTypeEncoding matching
// the current assembler mode.
for (auto const &Byte : LSDAActionTable) {
Streamer->EmitIntValue(Byte, 1);
}
assert(!(TTypeEncoding & dwarf::DW_EH_PE_indirect) &&
"indirect type info encoding is not supported yet");
for (int Index = LSDATypeTable.size() - 1; Index >= 0; --Index) {
// Note: the address could be an indirect one.
const auto TypeAddress = LSDATypeTable[Index];
switch (TTypeEncoding & 0x70) {
default:
llvm_unreachable("unsupported TTypeEncoding");
case 0:
Streamer->EmitIntValue(TypeAddress, TTypeEncodingSize);
break;
case dwarf::DW_EH_PE_pcrel: {
if (TypeAddress) {
const auto *TypeSymbol =
BC.getOrCreateGlobalSymbol(TypeAddress,
"TI",
TTypeEncodingSize,
TTypeAlignment);
auto *DotSymbol = BC.Ctx->createTempSymbol();
Streamer->EmitLabel(DotSymbol);
const auto *SubDotExpr = MCBinaryExpr::createSub(
MCSymbolRefExpr::create(TypeSymbol, *BC.Ctx),
MCSymbolRefExpr::create(DotSymbol, *BC.Ctx),
*BC.Ctx);
Streamer->EmitValue(SubDotExpr, TTypeEncodingSize);
} else {
Streamer->EmitIntValue(0, TTypeEncodingSize);
}
break;
}
}
}
for (auto const &Byte : LSDATypeIndexTable) {
Streamer->EmitIntValue(Byte, 1);
}
}
const uint8_t DWARF_CFI_PRIMARY_OPCODE_MASK = 0xc0;
CFIReaderWriter::CFIReaderWriter(const DWARFDebugFrame &EHFrame) {

60
bolt/src/JumpTable.cpp
View File

@ -110,66 +110,6 @@ void JumpTable::updateOriginal() {
}
}
uint64_t JumpTable::emit(MCStreamer *Streamer,
MCSection *HotSection,
MCSection *ColdSection) {
// Pre-process entries for aggressive splitting.
// Each label represents a separate switch table and gets its own count
// determining its destination.
std::map<MCSymbol *, uint64_t> LabelCounts;
if (opts::JumpTables > JTS_SPLIT && !Counts.empty()) {
MCSymbol *CurrentLabel = Labels[0];
uint64_t CurrentLabelCount = 0;
for (unsigned Index = 0; Index < Entries.size(); ++Index) {
auto LI = Labels.find(Index * EntrySize);
if (LI != Labels.end()) {
LabelCounts[CurrentLabel] = CurrentLabelCount;
CurrentLabel = LI->second;
CurrentLabelCount = 0;
}
CurrentLabelCount += Counts[Index].Count;
}
LabelCounts[CurrentLabel] = CurrentLabelCount;
} else {
Streamer->SwitchSection(Count > 0 ? HotSection : ColdSection);
Streamer->EmitValueToAlignment(EntrySize);
}
MCSymbol *LastLabel = nullptr;
uint64_t Offset = 0;
for (auto *Entry : Entries) {
auto LI = Labels.find(Offset);
if (LI != Labels.end()) {
DEBUG(dbgs() << "BOLT-DEBUG: emitting jump table "
<< LI->second->getName() << " (originally was at address 0x"
<< Twine::utohexstr(getAddress() + Offset)
<< (Offset ? "as part of larger jump table\n" : "\n"));
if (!LabelCounts.empty()) {
DEBUG(dbgs() << "BOLT-DEBUG: jump table count: "
<< LabelCounts[LI->second] << '\n');
if (LabelCounts[LI->second] > 0) {
Streamer->SwitchSection(HotSection);
} else {
Streamer->SwitchSection(ColdSection);
}
Streamer->EmitValueToAlignment(EntrySize);
}
Streamer->EmitLabel(LI->second);
LastLabel = LI->second;
}
if (Type == JTT_NORMAL) {
Streamer->EmitSymbolValue(Entry, OutputEntrySize);
} else { // JTT_PIC
auto JT = MCSymbolRefExpr::create(LastLabel, Streamer->getContext());
auto E = MCSymbolRefExpr::create(Entry, Streamer->getContext());
auto Value = MCBinaryExpr::createSub(E, JT, Streamer->getContext());
Streamer->EmitValue(Value, EntrySize);
}
Offset += EntrySize;
}
return Offset;
}
void JumpTable::print(raw_ostream &OS) const {
uint64_t Offset = 0;
if (Type == JTT_PIC)

5
bolt/src/JumpTable.h
View File

@ -122,11 +122,6 @@ public:
/// Update jump table at its original location.
void updateOriginal();
/// Emit jump table data. Callee supplies sections for the data.
/// Return the number of total bytes emitted.
uint64_t emit(MCStreamer *Streamer, MCSection *HotSection,
MCSection *ColdSection);
/// Print for debugging purposes.
virtual void print(raw_ostream &OS) const override;
};

270
bolt/src/RewriteInstance.cpp
View File

@ -12,6 +12,7 @@
#include "RewriteInstance.h"
#include "BinaryBasicBlock.h"
#include "BinaryContext.h"
#include "BinaryEmitter.h"
#include "BinaryFunction.h"
#include "BinaryPassManager.h"
#include "BoltAddressTranslation.h"
@ -30,10 +31,8 @@
#include "Utils.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/BinaryFormat/Magic.h"
#include "llvm/DebugInfo/DWARF/DWARFContext.h"
#include "llvm/DebugInfo/DWARF/DWARFDebugLine.h"
#include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
#include "llvm/ExecutionEngine/RTDyldMemoryManager.h"
@ -42,7 +41,6 @@
#include "llvm/MC/MCAsmLayout.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCDisassembler/MCDisassembler.h"
#include "llvm/MC/MCDwarf.h"
#include "llvm/MC/MCInstPrinter.h"
#include "llvm/MC/MCInstrAnalysis.h"
#include "llvm/MC/MCInstrInfo.h"
@ -50,14 +48,11 @@
#include "llvm/MC/MCObjectStreamer.h"
#include "llvm/MC/MCObjectWriter.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCSection.h"
#include "llvm/MC/MCSectionELF.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSubtargetInfo.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/Object/Archive.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Object/SymbolicFile.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/DataExtractor.h"
@ -122,7 +117,7 @@ ForceToDataRelocations("force-data-relocations",
cl::ZeroOrMore,
cl::cat(BoltCategory));
static cl::opt<bool>
cl::opt<bool>
PrintCacheMetrics("print-cache-metrics",
cl::desc("calculate and print various metrics for instruction cache"),
cl::init(false),
@ -146,14 +141,6 @@ BoltProfile("b",
cl::desc("<bolt profile>"),
cl::cat(BoltCategory));
static cl::list<std::string>
BreakFunctionNames("break-funcs",
cl::CommaSeparated,
cl::desc("list of functions to core dump on (debugging)"),
cl::value_desc("func1,func2,func3,..."),
cl::Hidden,
cl::cat(BoltCategory));
cl::opt<bool>
DumpDotAll("dump-dot-all",
cl::desc("dump function CFGs to graphviz format after each stage"),
@ -182,14 +169,6 @@ FunctionNamesFile("funcs-file",
cl::Hidden,
cl::cat(BoltCategory));
static cl::list<std::string>
FunctionPadSpec("pad-funcs",
cl::CommaSeparated,
cl::desc("list of functions to pad with amount of bytes"),
cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."),
cl::Hidden,
cl::cat(BoltCategory));
cl::opt<bool>
HotFunctionsAtEnd(
"hot-functions-at-end",
@ -198,7 +177,7 @@ HotFunctionsAtEnd(
cl::ZeroOrMore,
cl::cat(BoltCategory));
static cl::opt<bool>
cl::opt<bool>
HotText("hot-text",
cl::desc("hot text symbols support (relocation mode)"),
cl::ZeroOrMore,
@ -220,7 +199,7 @@ HotData("hot-data",
cl::ZeroOrMore,
cl::cat(BoltCategory));
static cl::opt<bool>
cl::opt<bool>
UpdateEnd("update-end",
cl::desc("update the _end symbol to point to the end of all data sections"),
cl::init(true),
@ -233,14 +212,6 @@ KeepTmp("keep-tmp",
cl::Hidden,
cl::cat(BoltCategory));
static cl::opt<bool>
MarkFuncs("mark-funcs",
cl::desc("mark function boundaries with break instruction to make "
"sure we accidentally don't cross them"),
cl::ReallyHidden,
cl::ZeroOrMore,
cl::cat(BoltCategory));
static cl::opt<unsigned>
MaxFunctions("max-funcs",
cl::desc("maximum number of functions to overwrite"),
@ -420,12 +391,6 @@ WriteBoltInfoSection("bolt-info",
cl::Hidden,
cl::cat(BoltOutputCategory));
static cl::opt<bool>
X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only",
cl::desc("only apply branch boundary alignment in hot code"),
cl::init(true),
cl::cat(BoltOptCategory));
bool isHotTextMover(const BinaryFunction &Function) {
for (auto &SectionName : opts::HotTextMoveSections) {
if (Function.getOriginSectionName() == SectionName)
@ -485,31 +450,6 @@ bool shouldProcess(const BinaryFunction &Function) {
return true;
}
size_t padFunction(const BinaryFunction &Function) {
static std::map<std::string, size_t> FunctionPadding;
if (FunctionPadding.empty() && !FunctionPadSpec.empty()) {
for (auto &Spec : FunctionPadSpec) {
auto N = Spec.find(':');
if (N == std::string::npos)
continue;
auto Name = Spec.substr(0, N);
auto Padding = std::stoull(Spec.substr(N+1));
FunctionPadding[Name] = Padding;
}
}
for (auto &FPI : FunctionPadding) {
auto Name = FPI.first;
auto Padding = FPI.second;
if (Function.hasNameRegex(Name)) {
return Padding;
}
}
return 0;
}
} // namespace opts
constexpr const char *RewriteInstance::SectionsToOverwrite[];
@ -2591,117 +2531,6 @@ void RewriteInstance::runOptimizationPasses() {
BinaryFunctionPassManager::runAllPasses(*BC);
}
// Helper function to emit the contents of a function via a MCStreamer object.
bool RewriteInstance::emitFunction(MCStreamer &Streamer,
BinaryFunction &Function,
bool EmitColdPart) {
if (Function.size() == 0)
return false;
if (Function.getState() == BinaryFunction::State::Empty)
return false;
auto *Section =
BC->getCodeSection(EmitColdPart ? Function.getColdCodeSectionName()
: Function.getCodeSectionName());
Streamer.SwitchSection(Section);
Section->setHasInstructions(true);
BC->Ctx->addGenDwarfSection(Section);
if (BC->HasRelocations) {
Streamer.EmitCodeAlignment(BinaryFunction::MinAlign);
auto MaxAlignBytes = EmitColdPart
? Function.getMaxColdAlignmentBytes()
: Function.getMaxAlignmentBytes();
if (MaxAlignBytes > 0)
Streamer.EmitCodeAlignment(Function.getAlignment(), MaxAlignBytes);
} else {
Streamer.EmitCodeAlignment(Function.getAlignment());
}
MCContext &Context = Streamer.getContext();
const MCAsmInfo *MAI = Context.getAsmInfo();
// Emit all symbols associated with the main function entry.
if (!EmitColdPart) {
for (auto *Symbol : Function.getSymbols()) {
Streamer.EmitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction);
Streamer.EmitLabel(Symbol);
}
} else {
auto *Symbol = Function.getColdSymbol();
Streamer.EmitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction);
Streamer.EmitLabel(Symbol);
}
// Emit CFI start
if (Function.hasCFI()) {
Streamer.EmitCFIStartProc(/*IsSimple=*/false);
if (Function.getPersonalityFunction() != nullptr) {
Streamer.EmitCFIPersonality(Function.getPersonalityFunction(),
Function.getPersonalityEncoding());
}
auto *LSDASymbol = EmitColdPart ? Function.getColdLSDASymbol()
: Function.getLSDASymbol();
if (LSDASymbol) {
Streamer.EmitCFILsda(LSDASymbol, BC->MOFI->getLSDAEncoding());
} else {
Streamer.EmitCFILsda(0, dwarf::DW_EH_PE_omit);
}
// Emit CFI instructions relative to the CIE
for (const auto &CFIInstr : Function.cie()) {
// Only write CIE CFI insns that LLVM will not already emit
const std::vector<MCCFIInstruction> &FrameInstrs =
MAI->getInitialFrameState();
if (std::find(FrameInstrs.begin(), FrameInstrs.end(), CFIInstr) ==
FrameInstrs.end())
Streamer.EmitCFIInstruction(CFIInstr);
}
}
assert((Function.empty() || !(*Function.begin()).isCold()) &&
"first basic block should never be cold");
// Emit UD2 at the beginning if requested by user.
if (!opts::BreakFunctionNames.empty()) {
for (auto &Name : opts::BreakFunctionNames) {
if (Function.hasNameRegex(Name)) {
Streamer.EmitIntValue(0x0B0F, 2); // UD2: 0F 0B
break;
}
}
}
// Emit code.
Function.emitBody(Streamer, EmitColdPart, /*EmitCodeOnly=*/false);
// Emit padding if requested.
if (auto Padding = opts::padFunction(Function)) {
DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with "
<< Padding << " bytes\n");
Streamer.emitFill(Padding, MAI->getTextAlignFillValue());
}
if (opts::MarkFuncs) {
Streamer.EmitIntValue(MAI->getTrapFillValue(), 1);
}
// Emit CFI end
if (Function.hasCFI())
Streamer.EmitCFIEndProc();
Streamer.EmitLabel(EmitColdPart ? Function.getFunctionColdEndLabel()
: Function.getFunctionEndLabel());
// Exception handling info for the function.
Function.emitLSDA(&Streamer, EmitColdPart);
if (!EmitColdPart && opts::JumpTables > JTS_NONE)
Function.emitJumpTables(&Streamer);
return true;
}
namespace {
template <typename T>
@ -2742,30 +2571,18 @@ void RewriteInstance::emitAndLink() {
/* IncrementalLinkerCompatible */ false,
/* DWARFMustBeAtTheEnd */ false));
Streamer->InitSections(false);
BC->getTextSection()->setAlignment(BC->PageAlign);
emitFunctions(Streamer.get());
if (opts::Instrument) {
Instrumenter->emit(*BC, *Streamer.get());
}
if (!BC->HasRelocations && opts::UpdateDebugSections)
DebugInfoRewriter->updateDebugLineInfoForNonSimpleFunctions();
// Make .eh_frame relocatable.
if (EHFrameSection) {
relocateEHFrameSection();
}
emitDataSections(Streamer.get());
// Update _end if needed.
if (opts::UpdateEnd) {
Streamer->EmitLabel(BC->Ctx->getOrCreateSymbol("_end"));
// Emit contents outside of BinaryContext.
if (opts::Instrument) {
Instrumenter->emit(*BC, *Streamer.get());
}
emitBinaryContext(*Streamer, *BC, OrgSecPrefix);
Streamer->Finish();
//////////////////////////////////////////////////////////////////////////////
@ -2981,59 +2798,6 @@ void RewriteInstance::updateSDTMarkers() {
}
}
void RewriteInstance::emitFunctions(MCStreamer *Streamer) {
auto emit = [&](const std::vector<BinaryFunction *> &Functions) {
const auto HasProfile = BC->NumProfiledFuncs > 0;
const uint32_t OriginalBranchBoundaryAlign = X86AlignBranchBoundary;
for (auto *Function : Functions) {
if (!BC->HasRelocations &&
(!Function->isSimple() || !opts::shouldProcess(*Function)))
continue;
DEBUG(dbgs() << "BOLT: generating code for function \""
<< *Function << "\" : "
<< Function->getFunctionNumber() << '\n');
bool Emitted{false};
// Turn off Intel JCC Erratum mitigation for cold code if requested
if (HasProfile && opts::X86AlignBranchBoundaryHotOnly &&
!Function->hasValidProfile())
X86AlignBranchBoundary = 0;
Emitted |= emitFunction(*Streamer, *Function, /*EmitColdPart=*/false);
if (Function->isSplit()) {
if (opts::X86AlignBranchBoundaryHotOnly)
X86AlignBranchBoundary = 0;
Emitted |= emitFunction(*Streamer, *Function, /*EmitColdPart=*/true);
}
X86AlignBranchBoundary = OriginalBranchBoundaryAlign;
if (Emitted)
Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics);
}
};
// Mark the start of hot text.
if (opts::HotText) {
Streamer->SwitchSection(BC->getTextSection());
Streamer->EmitLabel(BC->getHotTextStartSymbol());
}
// Emit functions in sorted order.
std::vector<BinaryFunction *> SortedFunctions = BC->getSortedFunctions();
emit(SortedFunctions);
// Emit functions added by BOLT.
emit(BC->getInjectedBinaryFunctions());
// Mark the end of hot text.
if (opts::HotText) {
Streamer->SwitchSection(BC->getTextSection());
Streamer->EmitLabel(BC->getHotTextEndSymbol());
}
}
void RewriteInstance::mapFileSections(orc::VModuleKey Key) {
mapCodeSections(Key);
mapDataSections(Key);
@ -3358,19 +3122,6 @@ void RewriteInstance::updateOutputValues(const MCAsmLayout &Layout) {
}
}
void RewriteInstance::emitDataSections(MCStreamer *Streamer) {
for (const auto &Section : BC->sections()) {
if (!Section.hasRelocations() || !Section.hasSectionRef())
continue;
StringRef SectionName = Section.getName();
std::string EmitName = Section.isReordered()
? std::string(Section.getOutputName())
: OrgSecPrefix + std::string(SectionName);
Section.emitAsData(*Streamer, EmitName);
}
}
void RewriteInstance::patchELFPHDRTable() {
auto ELF64LEFile = dyn_cast<ELF64LEObjectFile>(InputFile);
if (!ELF64LEFile) {
@ -3720,7 +3471,8 @@ std::vector<ELFShdrTy> RewriteInstance::getOutputSections(
if (!Section.isFinalized())
continue;
if (Section.getName().startswith(OrgSecPrefix) || Section.isAnonymous()) {
if (Section.getName().startswith(OrgSecPrefix) ||
Section.isAnonymous()) {
if (opts::Verbosity)
outs() << "BOLT-INFO: not writing section header for section "
<< Section.getName() << '\n';

18
bolt/src/RewriteInstance.h
View File

@ -135,12 +135,6 @@ private:
/// Link additional runtime code to support instrumentation.
void linkRuntime();
/// Emit function code.
void emitFunctions(MCStreamer *Streamer);
/// Emit data sections that have code references in them.
void emitDataSections(MCStreamer *Streamer);
/// Update debug and other auxiliary information in the file.
void updateMetadata();
@ -176,10 +170,6 @@ private:
"findSymbol failed");
}
/// Emit a single function.
bool emitFunction(MCStreamer &Streamer, BinaryFunction &Function,
bool EmitColdPart);
/// Detect addresses and offsets available in the binary for allocating
/// new sections.
void discoverStorage();
@ -269,9 +259,9 @@ private:
/// Add a notes section containing the BOLT revision and command line options.
void addBoltInfoSection();
/// Add a notes section containing the serialized BOLT Address Translation maps
/// that can be used to enable sampling of the output binary for the purposes
/// of generating BOLT profile data for the input binary.
/// Add a notes section containing the serialized BOLT Address Translation
/// maps that can be used to enable sampling of the output binary for the
/// purposes of generating BOLT profile data for the input binary.
void addBATSection();
/// Loop over now emitted functions to write translation maps
@ -388,8 +378,6 @@ private:
/// Extra linking
uint64_t InstrumentationRuntimeFiniAddress{0};
uint64_t InstrumentationRuntimeStartAddress{0};
const BinaryFunction *StartFunction{nullptr};
const BinaryFunction *FiniFunction{nullptr};
/// Track next available address for new allocatable sections.
uint64_t NextAvailableAddress{0};