diff --git a/bolt/src/BinaryContext.cpp b/bolt/src/BinaryContext.cpp index 36ece4582eed..c5e5ab8352f6 100644 --- a/bolt/src/BinaryContext.cpp +++ b/bolt/src/BinaryContext.cpp @@ -10,6 +10,7 @@ //===----------------------------------------------------------------------===// #include "BinaryContext.h" +#include "BinaryEmitter.h" #include "BinaryFunction.h" #include "DataReader.h" #include "ParallelUtilities.h" @@ -402,8 +403,8 @@ BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF, /// a reference to its constant island. When emitting this function, /// we will also emit IslandIter->second's constants. This only /// happens in custom AArch64 assembly code. - BF.IslandDependency.insert(IslandIter->second); - BF.ProxyIslandSymbols[IslandSym] = IslandIter->second; + BF.Islands.Dependency.insert(IslandIter->second); + BF.Islands.ProxySymbols[IslandSym] = IslandIter->second; return std::make_pair(IslandSym, Addend); } } @@ -1899,9 +1900,9 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) { std::unique_ptr Streamer(TheTarget->createMCObjectStreamer( *TheTriple, *LocalCtx, std::unique_ptr(MAB), VecOS, std::unique_ptr(MCEInstance.MCE.release()), *STI, - /* RelaxAll */ false, - /* IncrementalLinkerCompatible */ false, - /* DWARFMustBeAtTheEnd */ false)); + /*RelaxAll=*/false, + /*IncrementalLinkerCompatible=*/false, + /*DWARFMustBeAtTheEnd=*/false)); Streamer->InitSections(false); @@ -1915,7 +1916,8 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) { Streamer->SwitchSection(Section); Streamer->EmitLabel(StartLabel); - BF.emitBody(*Streamer, /*EmitColdPart = */false, /*EmitCodeOnly = */true); + emitFunctionBody(*Streamer, BF, /*EmitColdPart=*/false, + /*EmitCodeOnly=*/true); Streamer->EmitLabel(EndLabel); if (BF.isSplit()) { @@ -1927,7 +1929,8 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) { Streamer->SwitchSection(ColdSection); Streamer->EmitLabel(ColdStartLabel); - BF.emitBody(*Streamer, /*EmitColdPart = */true, /*EmitCodeOnly = */true); + emitFunctionBody(*Streamer, BF, /*EmitColdPart=*/true, + /*EmitCodeOnly=*/true); Streamer->EmitLabel(ColdEndLabel); } diff --git a/bolt/src/BinaryContext.h b/bolt/src/BinaryContext.h index a80055bbaea1..0cd1a775b526 100644 --- a/bolt/src/BinaryContext.h +++ b/bolt/src/BinaryContext.h @@ -23,6 +23,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/Triple.h" +#include "llvm/BinaryFormat/Dwarf.h" #include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/MC/MCAsmBackend.h" @@ -257,6 +258,24 @@ public: return nullptr; } + unsigned getDWARFEncodingSize(unsigned Encoding) { + switch (Encoding & 0x0f) { + default: llvm_unreachable("unknown encoding"); + case dwarf::DW_EH_PE_absptr: + case dwarf::DW_EH_PE_signed: + return AsmInfo->getCodePointerSize(); + case dwarf::DW_EH_PE_udata2: + case dwarf::DW_EH_PE_sdata2: + return 2; + case dwarf::DW_EH_PE_udata4: + case dwarf::DW_EH_PE_sdata4: + return 4; + case dwarf::DW_EH_PE_udata8: + case dwarf::DW_EH_PE_sdata8: + return 8; + } + } + /// [MCSymbol] -> [BinaryFunction] /// /// As we fold identical functions, multiple symbols can point @@ -675,6 +694,11 @@ public: ELF::SHF_EXECINSTR | ELF::SHF_ALLOC); } + /// Return data section with a given name. + MCSection *getDataSection(StringRef SectionName) const { + return Ctx->getELFSection(SectionName, ELF::SHT_PROGBITS, ELF::SHF_ALLOC); + } + /// \name Pre-assigned Section Names /// @{ diff --git a/bolt/src/BinaryEmitter.cpp b/bolt/src/BinaryEmitter.cpp new file mode 100644 index 000000000000..06f9b02aaf9f --- /dev/null +++ b/bolt/src/BinaryEmitter.cpp @@ -0,0 +1,1044 @@ +//===--- BinaryEmitter.cpp - collection of functions to emit code and data ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +//===----------------------------------------------------------------------===// + +#include "BinaryContext.h" +#include "BinaryEmitter.h" +#include "BinaryFunction.h" +#include "llvm/MC/MCSection.h" +#include "llvm/Support/LEB128.h" +#include "llvm/Support/SMLoc.h" + +#undef DEBUG_TYPE +#define DEBUG_TYPE "bolt" + +using namespace llvm; +using namespace bolt; + +extern cl::opt X86AlignBranchBoundary; + +namespace opts { + +extern bool shouldProcess(const BinaryFunction &); + +extern cl::OptionCategory BoltCategory; +extern cl::OptionCategory BoltOptCategory; +extern cl::OptionCategory BoltRelocCategory; + +extern cl::opt HotText; +extern cl::opt Instrument; +extern cl::opt JumpTables; +extern cl::opt PreserveBlocksAlignment; +extern cl::opt PrintCacheMetrics; +extern cl::opt UpdateDebugSections; +extern cl::opt UpdateEnd; +extern cl::opt Verbosity; + +cl::opt +AlignBlocks("align-blocks", + cl::desc("align basic blocks"), + cl::init(false), + cl::ZeroOrMore, + cl::cat(BoltOptCategory)); + +cl::opt +AlignMacroOpFusion("align-macro-fusion", + cl::desc("fix instruction alignment for macro-fusion (x86 relocation mode)"), + cl::init(MFT_HOT), + cl::values(clEnumValN(MFT_NONE, "none", + "do not insert alignment no-ops for macro-fusion"), + clEnumValN(MFT_HOT, "hot", + "only insert alignment no-ops on hot execution paths (default)"), + clEnumValN(MFT_ALL, "all", + "always align instructions to allow macro-fusion")), + cl::ZeroOrMore, + cl::cat(BoltRelocCategory)); + +static cl::list +BreakFunctionNames("break-funcs", + cl::CommaSeparated, + cl::desc("list of functions to core dump on (debugging)"), + cl::value_desc("func1,func2,func3,..."), + cl::Hidden, + cl::cat(BoltCategory)); + +static cl::list +FunctionPadSpec("pad-funcs", + cl::CommaSeparated, + cl::desc("list of functions to pad with amount of bytes"), + cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."), + cl::Hidden, + cl::cat(BoltCategory)); + +static cl::opt +MarkFuncs("mark-funcs", + cl::desc("mark function boundaries with break instruction to make " + "sure we accidentally don't cross them"), + cl::ReallyHidden, + cl::ZeroOrMore, + cl::cat(BoltCategory)); + +static cl::opt +PrintJumpTables("print-jump-tables", + cl::desc("print jump tables"), + cl::ZeroOrMore, + cl::Hidden, + cl::cat(BoltCategory)); + +static cl::opt +X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only", + cl::desc("only apply branch boundary alignment in hot code"), + cl::init(true), + cl::cat(BoltOptCategory)); + +size_t padFunction(const BinaryFunction &Function) { + static std::map FunctionPadding; + + if (FunctionPadding.empty() && !FunctionPadSpec.empty()) { + for (auto &Spec : FunctionPadSpec) { + auto N = Spec.find(':'); + if (N == std::string::npos) + continue; + auto Name = Spec.substr(0, N); + auto Padding = std::stoull(Spec.substr(N+1)); + FunctionPadding[Name] = Padding; + } + } + + for (auto &FPI : FunctionPadding) { + auto Name = FPI.first; + auto Padding = FPI.second; + if (Function.hasNameRegex(Name)) { + return Padding; + } + } + + return 0; +} + +} // namespace opts + +namespace { + +class BinaryEmitter { +private: + BinaryEmitter(const BinaryEmitter &) = delete; + BinaryEmitter &operator=(const BinaryEmitter &) = delete; + + MCStreamer &Streamer; + BinaryContext &BC; + +public: + BinaryEmitter(MCStreamer &Streamer, BinaryContext &BC) + : Streamer(Streamer), + BC(BC) {} + + /// Emit all code and data. + void emitAll(StringRef OrgSecPrefix); + + /// Emit function code. The caller is responsible for emitting function + /// symbol(s) and setting the section to emit the code to. + void emitFunctionBody(BinaryFunction &BF, bool EmitColdPart, + bool EmitCodeOnly = false); + +private: + /// Emit function code. + void emitFunctions(); + + /// Emit a single function. + bool emitFunction(BinaryFunction &BF, bool EmitColdPart); + + /// Helper for emitFunctionBody to write data inside a function + /// (used for AArch64) + void emitConstantIslands(BinaryFunction &BF, bool EmitColdPart, + BinaryFunction *OnBehalfOf = nullptr); + + /// Emit jump tables for the function. + void emitJumpTables(const BinaryFunction &BF); + + /// Emit jump table data. Callee supplies sections for the data. + void emitJumpTable(const JumpTable &JT, MCSection *HotSection, + MCSection *ColdSection); + + /// Emit exception handling ranges for the function. + void emitLSDA(BinaryFunction &BF, bool EmitColdPart); + + /// Emit line number information corresponding to \p NewLoc. \p PrevLoc + /// provides a context for de-duplication of line number info. + /// \p FirstInstr indicates if \p NewLoc represents the first instruction + /// in a sequence, such as a function fragment. + /// + /// Return new current location which is either \p NewLoc or \p PrevLoc. + SMLoc emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, SMLoc PrevLoc, + bool FirstInstr); + + /// Emit debug line information for functions that were not emitted. + void emitDebugLineInfoForNonSimpleFunctions(); + + /// Emit function as a blob with relocations and labels for relocations. + void emitFunctionBodyRaw(BinaryFunction &BF) LLVM_ATTRIBUTE_UNUSED; + + /// Emit data sections that have code references in them. + void emitDataSections(StringRef OrgSecPrefix); +}; + +} // anonymous namespace + +void BinaryEmitter::emitAll(StringRef OrgSecPrefix) { + Streamer.InitSections(false); + + BC.getTextSection()->setAlignment(BC.PageAlign); + + emitFunctions(); + + if (!BC.HasRelocations && opts::UpdateDebugSections) + emitDebugLineInfoForNonSimpleFunctions(); + + emitDataSections(OrgSecPrefix); + + // Update _end if needed. + if (opts::UpdateEnd) { + Streamer.EmitLabel(BC.Ctx->getOrCreateSymbol("_end")); + } +} + +void BinaryEmitter::emitFunctions() { + auto emit = [&](const std::vector &Functions) { + const auto HasProfile = BC.NumProfiledFuncs > 0; + const uint32_t OriginalBranchBoundaryAlign = X86AlignBranchBoundary; + for (auto *Function : Functions) { + if (!BC.HasRelocations && + (!Function->isSimple() || !opts::shouldProcess(*Function))) + continue; + + DEBUG(dbgs() << "BOLT: generating code for function \"" + << *Function << "\" : " + << Function->getFunctionNumber() << '\n'); + + // Was any part of the function emitted. + bool Emitted{false}; + + // Turn off Intel JCC Erratum mitigation for cold code if requested + if (HasProfile && opts::X86AlignBranchBoundaryHotOnly && + !Function->hasValidProfile()) + X86AlignBranchBoundary = 0; + + Emitted |= emitFunction(*Function, /*EmitColdPart=*/false); + + if (Function->isSplit()) { + if (opts::X86AlignBranchBoundaryHotOnly) + X86AlignBranchBoundary = 0; + Emitted |= emitFunction(*Function, /*EmitColdPart=*/true); + } + X86AlignBranchBoundary = OriginalBranchBoundaryAlign; + + if (Emitted) + Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics); + } + }; + + // Mark the start of hot text. + if (opts::HotText) { + Streamer.SwitchSection(BC.getTextSection()); + Streamer.EmitLabel(BC.getHotTextStartSymbol()); + } + + // Emit functions in sorted order. + std::vector SortedFunctions = BC.getSortedFunctions(); + emit(SortedFunctions); + + // Emit functions added by BOLT. + emit(BC.getInjectedBinaryFunctions()); + + // Mark the end of hot text. + if (opts::HotText) { + Streamer.SwitchSection(BC.getTextSection()); + Streamer.EmitLabel(BC.getHotTextEndSymbol()); + } +} + +bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) { + if (Function.size() == 0) + return false; + + if (Function.getState() == BinaryFunction::State::Empty) + return false; + + auto *Section = + BC.getCodeSection(EmitColdPart ? Function.getColdCodeSectionName() + : Function.getCodeSectionName()); + Streamer.SwitchSection(Section); + Section->setHasInstructions(true); + BC.Ctx->addGenDwarfSection(Section); + + if (BC.HasRelocations) { + Streamer.EmitCodeAlignment(BinaryFunction::MinAlign); + auto MaxAlignBytes = EmitColdPart + ? Function.getMaxColdAlignmentBytes() + : Function.getMaxAlignmentBytes(); + if (MaxAlignBytes > 0) + Streamer.EmitCodeAlignment(Function.getAlignment(), MaxAlignBytes); + } else { + Streamer.EmitCodeAlignment(Function.getAlignment()); + } + + MCContext &Context = Streamer.getContext(); + const MCAsmInfo *MAI = Context.getAsmInfo(); + + // Emit all symbols associated with the main function entry. + if (!EmitColdPart) { + for (auto *Symbol : Function.getSymbols()) { + Streamer.EmitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction); + Streamer.EmitLabel(Symbol); + } + } else { + auto *Symbol = Function.getColdSymbol(); + Streamer.EmitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction); + Streamer.EmitLabel(Symbol); + } + + // Emit CFI start + if (Function.hasCFI()) { + Streamer.EmitCFIStartProc(/*IsSimple=*/false); + if (Function.getPersonalityFunction() != nullptr) { + Streamer.EmitCFIPersonality(Function.getPersonalityFunction(), + Function.getPersonalityEncoding()); + } + auto *LSDASymbol = EmitColdPart ? Function.getColdLSDASymbol() + : Function.getLSDASymbol(); + if (LSDASymbol) { + Streamer.EmitCFILsda(LSDASymbol, BC.MOFI->getLSDAEncoding()); + } else { + Streamer.EmitCFILsda(0, dwarf::DW_EH_PE_omit); + } + // Emit CFI instructions relative to the CIE + for (const auto &CFIInstr : Function.cie()) { + // Only write CIE CFI insns that LLVM will not already emit + const std::vector &FrameInstrs = + MAI->getInitialFrameState(); + if (std::find(FrameInstrs.begin(), FrameInstrs.end(), CFIInstr) == + FrameInstrs.end()) + Streamer.EmitCFIInstruction(CFIInstr); + } + } + + assert((Function.empty() || !(*Function.begin()).isCold()) && + "first basic block should never be cold"); + + // Emit UD2 at the beginning if requested by user. + if (!opts::BreakFunctionNames.empty()) { + for (auto &Name : opts::BreakFunctionNames) { + if (Function.hasNameRegex(Name)) { + Streamer.EmitIntValue(0x0B0F, 2); // UD2: 0F 0B + break; + } + } + } + + // Emit code. + emitFunctionBody(Function, EmitColdPart, /*EmitCodeOnly=*/false); + + // Emit padding if requested. + if (auto Padding = opts::padFunction(Function)) { + DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with " + << Padding << " bytes\n"); + Streamer.emitFill(Padding, MAI->getTextAlignFillValue()); + } + + if (opts::MarkFuncs) { + Streamer.EmitIntValue(MAI->getTrapFillValue(), 1); + } + + // Emit CFI end + if (Function.hasCFI()) + Streamer.EmitCFIEndProc(); + + Streamer.EmitLabel(EmitColdPart ? Function.getFunctionColdEndLabel() + : Function.getFunctionEndLabel()); + + // Exception handling info for the function. + emitLSDA(Function, EmitColdPart); + + if (!EmitColdPart && opts::JumpTables > JTS_NONE) + emitJumpTables(Function); + + return true; +} + +void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, bool EmitColdPart, + bool EmitCodeOnly) { + if (!EmitCodeOnly && EmitColdPart && BF.hasConstantIsland()) + BF.duplicateConstantIslands(); + + // Track the first emitted instruction with debug info. + bool FirstInstr = true; + for (auto BB : BF.layout()) { + if (EmitColdPart != BB->isCold()) + continue; + + if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) + && BB->getAlignment() > 1) { + Streamer.EmitCodeAlignment(BB->getAlignment(), + BB->getAlignmentMaxBytes()); + } + Streamer.EmitLabel(BB->getLabel()); + + // Check if special alignment for macro-fusion is needed. + bool MayNeedMacroFusionAlignment = + (opts::AlignMacroOpFusion == MFT_ALL) || + (opts::AlignMacroOpFusion == MFT_HOT && + BB->getKnownExecutionCount()); + BinaryBasicBlock::const_iterator MacroFusionPair; + if (MayNeedMacroFusionAlignment) { + MacroFusionPair = BB->getMacroOpFusionPair(); + if (MacroFusionPair == BB->end()) + MayNeedMacroFusionAlignment = false; + } + + SMLoc LastLocSeen; + // Remember if the last instruction emitted was a prefix. + bool LastIsPrefix = false; + for (auto I = BB->begin(), E = BB->end(); I != E; ++I) { + auto &Instr = *I; + + if (EmitCodeOnly && BC.MII->get(Instr.getOpcode()).isPseudo()) + continue; + + // Handle pseudo instructions. + if (BC.MIB->isEHLabel(Instr)) { + const auto *Label = BC.MIB->getTargetSymbol(Instr); + assert(Instr.getNumOperands() >= 1 && Label && + "bad EH_LABEL instruction"); + Streamer.EmitLabel(const_cast(Label)); + continue; + } + if (BC.MIB->isCFI(Instr)) { + Streamer.EmitCFIInstruction(*BF.getCFIFor(Instr)); + continue; + } + + // Handle macro-fusion alignment. If we emitted a prefix as + // the last instruction, we should've already emitted the associated + // alignment hint, so don't emit it twice. + if (MayNeedMacroFusionAlignment && !LastIsPrefix && I == MacroFusionPair){ + // This assumes the second instruction in the macro-op pair will get + // assigned to its own MCRelaxableFragment. Since all JCC instructions + // are relaxable, we should be safe. + Streamer.EmitNeverAlignCodeAtEnd(/*Alignment to avoid=*/64); + } + + if (!EmitCodeOnly && opts::UpdateDebugSections && + BF.getDWARFUnitLineTable().first) { + LastLocSeen = emitLineInfo(BF, Instr.getLoc(), LastLocSeen, FirstInstr); + FirstInstr = false; + } + + // Prepare to tag this location with a label if we need to keep track of + // the location of calls/returns for BOLT address translation maps + if (!EmitCodeOnly && BF.requiresAddressTranslation() && + BC.MIB->hasAnnotation(Instr, "Offset")) { + const auto Offset = BC.MIB->getAnnotationAs(Instr, "Offset"); + MCSymbol *LocSym = BC.Ctx->createTempSymbol(/*CanBeUnnamed=*/true); + Streamer.EmitLabel(LocSym); + BB->getLocSyms().emplace_back(std::make_pair(Offset, LocSym)); + } + + Streamer.EmitInstruction(Instr, *BC.STI); + LastIsPrefix = BC.MIB->isPrefix(Instr); + } + } + + if (!EmitCodeOnly) + emitConstantIslands(BF, EmitColdPart); +} + +void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart, + BinaryFunction *OnBehalfOf) { + BinaryFunction::IslandInfo &Islands = BF.getIslandInfo(); + if (Islands.DataOffsets.empty() && Islands.Dependency.empty()) + return; + + if (!OnBehalfOf) { + if (!EmitColdPart) + Streamer.EmitLabel(BF.getFunctionConstantIslandLabel()); + else + Streamer.EmitLabel(BF.getFunctionColdConstantIslandLabel()); + } + + assert((!OnBehalfOf || Islands.Proxies[OnBehalfOf].size() > 0) && + "spurious OnBehalfOf constant island emission"); + + assert(!BF.isInjected() && + "injected functions should not have constant islands"); + // Raw contents of the function. + StringRef SectionContents = BF.getSection().getContents(); + + // Raw contents of the function. + StringRef FunctionContents = + SectionContents.substr( + BF.getAddress() - BF.getSection().getAddress(), + BF.getMaxSize()); + + if (opts::Verbosity && !OnBehalfOf) + outs() << "BOLT-INFO: emitting constant island for function " << BF << "\n"; + + // We split the island into smaller blocks and output labels between them. + auto IS = Islands.Offsets.begin(); + for (auto DataIter = Islands.DataOffsets.begin(); + DataIter != Islands.DataOffsets.end(); + ++DataIter) { + uint64_t FunctionOffset = *DataIter; + uint64_t EndOffset = 0ULL; + + // Determine size of this data chunk + auto NextData = std::next(DataIter); + auto CodeIter = Islands.CodeOffsets.lower_bound(*DataIter); + if (CodeIter == Islands.CodeOffsets.end() && + NextData == Islands.DataOffsets.end()) { + EndOffset = BF.getMaxSize(); + } else if (CodeIter == Islands.CodeOffsets.end()) { + EndOffset = *NextData; + } else if (NextData == Islands.DataOffsets.end()) { + EndOffset = *CodeIter; + } else { + EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter; + } + + if (FunctionOffset == EndOffset) + continue; // Size is zero, nothing to emit + + // Emit labels, relocs and data + auto RI = BF.getMoveRelocations().lower_bound(FunctionOffset); + while ((IS != Islands.Offsets.end() && IS->first < EndOffset) || + (RI != BF.getMoveRelocations().end() && RI->first < EndOffset)) { + auto NextLabelOffset = + IS == Islands.Offsets.end() ? EndOffset : IS->first; + auto NextRelOffset = + RI == BF.getMoveRelocations().end() ? EndOffset : RI->first; + auto NextStop = std::min(NextLabelOffset, NextRelOffset); + assert(NextStop <= EndOffset && "internal overflow error"); + if (FunctionOffset < NextStop) { + Streamer.EmitBytes(FunctionContents.slice(FunctionOffset, NextStop)); + FunctionOffset = NextStop; + } + if (IS != Islands.Offsets.end() && FunctionOffset == IS->first) { + // This is a slightly complex code to decide which label to emit. We + // have 4 cases to handle: regular symbol, cold symbol, regular or cold + // symbol being emitted on behalf of an external function. + if (!OnBehalfOf) { + if (!EmitColdPart) { + DEBUG(dbgs() << "BOLT-DEBUG: emitted label " + << IS->second->getName() << " at offset 0x" + << Twine::utohexstr(IS->first) << '\n'); + if (IS->second->isUndefined()) + Streamer.EmitLabel(IS->second); + else + assert(BF.hasName(IS->second->getName())); + } else if (Islands.ColdSymbols.count(IS->second) != 0) { + DEBUG(dbgs() << "BOLT-DEBUG: emitted label " + << Islands.ColdSymbols[IS->second]->getName() + << '\n'); + if (Islands.ColdSymbols[IS->second]->isUndefined()) + Streamer.EmitLabel(Islands.ColdSymbols[IS->second]); + } + } else { + if (!EmitColdPart) { + if (MCSymbol *Sym = Islands.Proxies[OnBehalfOf][IS->second]) { + DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName() + << '\n'); + Streamer.EmitLabel(Sym); + } + } else if (MCSymbol *Sym = + Islands.ColdProxies[OnBehalfOf][IS->second]) { + DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName() + << '\n'); + Streamer.EmitLabel(Sym); + } + } + ++IS; + } + if (RI != BF.getMoveRelocations().end() && FunctionOffset == RI->first) { + auto RelocationSize = RI->second.emit(&Streamer); + DEBUG(dbgs() << "BOLT-DEBUG: emitted relocation for symbol " + << RI->second.Symbol->getName() << " at offset 0x" + << Twine::utohexstr(RI->first) + << " with size " << RelocationSize << '\n'); + FunctionOffset += RelocationSize; + ++RI; + } + } + assert(FunctionOffset <= EndOffset && "overflow error"); + if (FunctionOffset < EndOffset) { + Streamer.EmitBytes(FunctionContents.slice(FunctionOffset, EndOffset)); + } + } + assert(IS == Islands.Offsets.end() && "some symbols were not emitted!"); + + if (OnBehalfOf) + return; + // Now emit constant islands from other functions that we may have used in + // this function. + for (auto *ExternalFunc : Islands.Dependency) { + emitConstantIslands(*ExternalFunc, EmitColdPart, &BF); + } +} + +SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, + SMLoc PrevLoc, bool FirstInstr) { + auto *FunctionCU = BF.getDWARFUnitLineTable().first; + const auto *FunctionLineTable = BF.getDWARFUnitLineTable().second; + assert(FunctionCU && "cannot emit line info for function without CU"); + + auto RowReference = DebugLineTableRowRef::fromSMLoc(NewLoc); + + // Check if no new line info needs to be emitted. + if (RowReference == DebugLineTableRowRef::NULL_ROW || + NewLoc.getPointer() == PrevLoc.getPointer()) + return PrevLoc; + + unsigned CurrentFilenum = 0; + const auto *CurrentLineTable = FunctionLineTable; + + // If the CU id from the current instruction location does not + // match the CU id from the current function, it means that we + // have come across some inlined code. We must look up the CU + // for the instruction's original function and get the line table + // from that. + const auto FunctionUnitIndex = FunctionCU->getOffset(); + const auto CurrentUnitIndex = RowReference.DwCompileUnitIndex; + if (CurrentUnitIndex != FunctionUnitIndex) { + CurrentLineTable = BC.DwCtx->getLineTableForUnit( + BC.DwCtx->getCompileUnitForOffset(CurrentUnitIndex)); + // Add filename from the inlined function to the current CU. + CurrentFilenum = + BC.addDebugFilenameToUnit(FunctionUnitIndex, CurrentUnitIndex, + CurrentLineTable->Rows[RowReference.RowIndex - 1].File); + } + + const auto &CurrentRow = CurrentLineTable->Rows[RowReference.RowIndex - 1]; + if (!CurrentFilenum) + CurrentFilenum = CurrentRow.File; + + unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) | + (DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) | + (DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) | + (DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin); + + // Always emit is_stmt at the beginning of function fragment. + if (FirstInstr) + Flags |= DWARF2_FLAG_IS_STMT; + + BC.Ctx->setCurrentDwarfLoc( + CurrentFilenum, + CurrentRow.Line, + CurrentRow.Column, + Flags, + CurrentRow.Isa, + CurrentRow.Discriminator); + BC.Ctx->setDwarfCompileUnitID(FunctionUnitIndex); + + return NewLoc; +} + +void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) { + if (!BF.hasJumpTables()) + return; + + if (opts::PrintJumpTables) { + outs() << "BOLT-INFO: jump tables for function " << BF << ":\n"; + } + + for (auto &JTI : BF.jumpTables()) { + auto &JT = *JTI.second; + if (opts::PrintJumpTables) + JT.print(outs()); + if ((opts::JumpTables == JTS_BASIC || !BF.isSimple()) && + BC.HasRelocations) { + JT.updateOriginal(); + } else { + MCSection *HotSection, *ColdSection; + if (opts::JumpTables == JTS_BASIC) { + std::string Name = ".local." + JT.Labels[0]->getName().str(); + std::replace(Name.begin(), Name.end(), '/', '.'); + auto &Section = BC.registerOrUpdateSection(Name, + ELF::SHT_PROGBITS, + ELF::SHF_ALLOC); + Section.setAnonymous(true); + JT.setOutputSection(Section); + HotSection = BC.getDataSection(Name); + ColdSection = HotSection; + } else { + if (BF.isSimple()) { + HotSection = BC.MOFI->getReadOnlySection(); + ColdSection = BC.MOFI->getReadOnlyColdSection(); + } else { + HotSection = BF.hasProfile() ? BC.MOFI->getReadOnlySection() + : BC.MOFI->getReadOnlyColdSection(); + ColdSection = HotSection; + } + } + emitJumpTable(JT, HotSection, ColdSection); + } + } +} + +void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection, + MCSection *ColdSection) { + // Pre-process entries for aggressive splitting. + // Each label represents a separate switch table and gets its own count + // determining its destination. + std::map LabelCounts; + if (opts::JumpTables > JTS_SPLIT && !JT.Counts.empty()) { + MCSymbol *CurrentLabel = JT.Labels.at(0); + uint64_t CurrentLabelCount = 0; + for (unsigned Index = 0; Index < JT.Entries.size(); ++Index) { + auto LI = JT.Labels.find(Index * JT.EntrySize); + if (LI != JT.Labels.end()) { + LabelCounts[CurrentLabel] = CurrentLabelCount; + CurrentLabel = LI->second; + CurrentLabelCount = 0; + } + CurrentLabelCount += JT.Counts[Index].Count; + } + LabelCounts[CurrentLabel] = CurrentLabelCount; + } else { + Streamer.SwitchSection(JT.Count > 0 ? HotSection : ColdSection); + Streamer.EmitValueToAlignment(JT.EntrySize); + } + MCSymbol *LastLabel = nullptr; + uint64_t Offset = 0; + for (auto *Entry : JT.Entries) { + auto LI = JT.Labels.find(Offset); + if (LI != JT.Labels.end()) { + DEBUG(dbgs() << "BOLT-DEBUG: emitting jump table " + << LI->second->getName() << " (originally was at address 0x" + << Twine::utohexstr(JT.getAddress() + Offset) + << (Offset ? "as part of larger jump table\n" : "\n")); + if (!LabelCounts.empty()) { + DEBUG(dbgs() << "BOLT-DEBUG: jump table count: " + << LabelCounts[LI->second] << '\n'); + if (LabelCounts[LI->second] > 0) { + Streamer.SwitchSection(HotSection); + } else { + Streamer.SwitchSection(ColdSection); + } + Streamer.EmitValueToAlignment(JT.EntrySize); + } + Streamer.EmitLabel(LI->second); + LastLabel = LI->second; + } + if (JT.Type == JumpTable::JTT_NORMAL) { + Streamer.EmitSymbolValue(Entry, JT.OutputEntrySize); + } else { // JTT_PIC + auto JTExpr = MCSymbolRefExpr::create(LastLabel, Streamer.getContext()); + auto E = MCSymbolRefExpr::create(Entry, Streamer.getContext()); + auto Value = MCBinaryExpr::createSub(E, JTExpr, Streamer.getContext()); + Streamer.EmitValue(Value, JT.EntrySize); + } + Offset += JT.EntrySize; + } +} + +// The code is based on EHStreamer::emitExceptionTable(). +void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) { + const auto *Sites = + EmitColdPart ? &BF.getColdCallSites() : &BF.getCallSites(); + if (Sites->empty()) { + return; + } + + + // Calculate callsite table size. Size of each callsite entry is: + // + // sizeof(start) + sizeof(length) + sizeof(LP) + sizeof(uleb128(action)) + // + // or + // + // sizeof(dwarf::DW_EH_PE_data4) * 3 + sizeof(uleb128(action)) + uint64_t CallSiteTableLength = Sites->size() * 4 * 3; + for (const auto &CallSite : *Sites) { + CallSiteTableLength += getULEB128Size(CallSite.Action); + } + + Streamer.SwitchSection(BC.MOFI->getLSDASection()); + + const auto TTypeEncoding = BC.MOFI->getTTypeEncoding(); + const auto TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding); + const auto TTypeAlignment = 4; + + // Type tables have to be aligned at 4 bytes. + Streamer.EmitValueToAlignment(TTypeAlignment); + + // Emit the LSDA label. + auto *LSDASymbol = EmitColdPart ? BF.getColdLSDASymbol() : BF.getLSDASymbol(); + assert(LSDASymbol && "no LSDA symbol set"); + Streamer.EmitLabel(LSDASymbol); + + // Corresponding FDE start. + const auto *StartSymbol = EmitColdPart ? BF.getColdSymbol() : BF.getSymbol(); + + // Emit the LSDA header. + + // If LPStart is omitted, then the start of the FDE is used as a base for + // landing pad displacements. Then if a cold fragment starts with + // a landing pad, this means that the first landing pad offset will be 0. + // As a result, an exception handling runtime will ignore this landing pad, + // because zero offset denotes the absence of a landing pad. + // For this reason, we emit LPStart value of 0 and output an absolute value + // of the landing pad in the table. + // + // FIXME: this may break PIEs and DSOs where the base address is not 0. + Streamer.EmitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format + Streamer.EmitIntValue(0, 4); + auto emitLandingPad = [&](const MCSymbol *LPSymbol) { + if (!LPSymbol) { + Streamer.EmitIntValue(0, 4); + return; + } + Streamer.EmitSymbolValue(LPSymbol, 4); + }; + + Streamer.EmitIntValue(TTypeEncoding, 1); // TType format + + // See the comment in EHStreamer::emitExceptionTable() on to use + // uleb128 encoding (which can use variable number of bytes to encode the same + // value) to ensure type info table is properly aligned at 4 bytes without + // iteratively fixing sizes of the tables. + unsigned CallSiteTableLengthSize = getULEB128Size(CallSiteTableLength); + unsigned TTypeBaseOffset = + sizeof(int8_t) + // Call site format + CallSiteTableLengthSize + // Call site table length size + CallSiteTableLength + // Call site table length + BF.getLSDAActionTable().size() + // Actions table size + BF.getLSDATypeTable().size() * TTypeEncodingSize; // Types table size + unsigned TTypeBaseOffsetSize = getULEB128Size(TTypeBaseOffset); + unsigned TotalSize = + sizeof(int8_t) + // LPStart format + sizeof(int8_t) + // TType format + TTypeBaseOffsetSize + // TType base offset size + TTypeBaseOffset; // TType base offset + unsigned SizeAlign = (4 - TotalSize) & 3; + + // Account for any extra padding that will be added to the call site table + // length. + Streamer.EmitPaddedULEB128IntValue(TTypeBaseOffset, + TTypeBaseOffsetSize + SizeAlign); + + // Emit the landing pad call site table. We use signed data4 since we can emit + // a landing pad in a different part of the split function that could appear + // earlier in the address space than LPStart. + Streamer.EmitIntValue(dwarf::DW_EH_PE_sdata4, 1); + Streamer.EmitULEB128IntValue(CallSiteTableLength); + + for (const auto &CallSite : *Sites) { + const auto *BeginLabel = CallSite.Start; + const auto *EndLabel = CallSite.End; + + assert(BeginLabel && "start EH label expected"); + assert(EndLabel && "end EH label expected"); + + // Start of the range is emitted relative to the start of current + // function split part. + Streamer.emitAbsoluteSymbolDiff(BeginLabel, StartSymbol, 4); + Streamer.emitAbsoluteSymbolDiff(EndLabel, BeginLabel, 4); + emitLandingPad(CallSite.LP); + Streamer.EmitULEB128IntValue(CallSite.Action); + } + + // Write out action, type, and type index tables at the end. + // + // For action and type index tables there's no need to change the original + // table format unless we are doing function splitting, in which case we can + // split and optimize the tables. + // + // For type table we (re-)encode the table using TTypeEncoding matching + // the current assembler mode. + for (auto const &Byte : BF.getLSDAActionTable()) { + Streamer.EmitIntValue(Byte, 1); + } + assert(!(TTypeEncoding & dwarf::DW_EH_PE_indirect) && + "indirect type info encoding is not supported yet"); + for (int Index = BF.getLSDATypeTable().size() - 1; Index >= 0; --Index) { + // Note: the address could be an indirect one. + const auto TypeAddress = BF.getLSDATypeTable()[Index]; + switch (TTypeEncoding & 0x70) { + default: + llvm_unreachable("unsupported TTypeEncoding"); + case 0: + Streamer.EmitIntValue(TypeAddress, TTypeEncodingSize); + break; + case dwarf::DW_EH_PE_pcrel: { + if (TypeAddress) { + const auto *TypeSymbol = + BC.getOrCreateGlobalSymbol(TypeAddress, + "TI", + TTypeEncodingSize, + TTypeAlignment); + auto *DotSymbol = BC.Ctx->createTempSymbol(); + Streamer.EmitLabel(DotSymbol); + const auto *SubDotExpr = MCBinaryExpr::createSub( + MCSymbolRefExpr::create(TypeSymbol, *BC.Ctx), + MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), + *BC.Ctx); + Streamer.EmitValue(SubDotExpr, TTypeEncodingSize); + } else { + Streamer.EmitIntValue(0, TTypeEncodingSize); + } + break; + } + } + } + for (auto const &Byte : BF.getLSDATypeIndexTable()) { + Streamer.EmitIntValue(Byte, 1); + } +} + +void BinaryEmitter::emitDebugLineInfoForNonSimpleFunctions() { + for (auto &It : BC.getBinaryFunctions()) { + const auto &Function = It.second; + + if (Function.isSimple()) + continue; + + auto ULT = Function.getDWARFUnitLineTable(); + auto Unit = ULT.first; + auto LineTable = ULT.second; + + if (!LineTable) + continue; // nothing to update for this function + + std::vector Results; + MCSection *FunctionSection = + BC.getCodeSection(Function.getCodeSectionName()); + + uint64_t Address = It.first; + if (LineTable->lookupAddressRange(Address, Function.getMaxSize(), + Results)) { + auto &OutputLineTable = + BC.Ctx->getMCDwarfLineTable(Unit->getOffset()).getMCLineSections(); + for (auto RowIndex : Results) { + const auto &Row = LineTable->Rows[RowIndex]; + BC.Ctx->setCurrentDwarfLoc( + Row.File, + Row.Line, + Row.Column, + (DWARF2_FLAG_IS_STMT * Row.IsStmt) | + (DWARF2_FLAG_BASIC_BLOCK * Row.BasicBlock) | + (DWARF2_FLAG_PROLOGUE_END * Row.PrologueEnd) | + (DWARF2_FLAG_EPILOGUE_BEGIN * Row.EpilogueBegin), + Row.Isa, + Row.Discriminator, + Row.Address); + auto Loc = BC.Ctx->getCurrentDwarfLoc(); + BC.Ctx->clearDwarfLocSeen(); + OutputLineTable.addLineEntry(MCDwarfLineEntry{nullptr, Loc}, + FunctionSection); + } + // Add an empty entry past the end of the function + // for end_sequence mark. + BC.Ctx->setCurrentDwarfLoc(0, 0, 0, 0, 0, 0, + Address + Function.getMaxSize()); + auto Loc = BC.Ctx->getCurrentDwarfLoc(); + BC.Ctx->clearDwarfLocSeen(); + OutputLineTable.addLineEntry(MCDwarfLineEntry{nullptr, Loc}, + FunctionSection); + } else { + DEBUG(dbgs() << "BOLT-DEBUG: function " << Function + << " has no associated line number information\n"); + } + } +} + +void BinaryEmitter::emitFunctionBodyRaw(BinaryFunction &BF) { + // #14998851: Fix gold linker's '--emit-relocs'. + llvm_unreachable( + "cannot emit raw body unless relocation accuracy is guaranteed"); + + assert(!BF.isInjected() && "cannot emit raw body of injected function"); + + // Raw contents of the function. + StringRef SectionContents = BF.getSection().getContents(); + + // Raw contents of the function. + StringRef FunctionContents = SectionContents.substr( + BF.getAddress() - BF.getSection().getAddress(), BF.getSize()); + + if (opts::Verbosity) + outs() << "BOLT-INFO: emitting function " << BF << " in raw (" + << BF.getSize() << " bytes)\n"; + + // We split the function blob into smaller blocks and output relocations + // and/or labels between them. + uint64_t FunctionOffset = 0; + auto LI = BF.getLabels().begin(); + auto RI = BF.getMoveRelocations().begin(); + while (LI != BF.getLabels().end() || + RI != BF.getMoveRelocations().end()) { + uint64_t NextLabelOffset = + (LI == BF.getLabels().end() ? BF.getSize() : LI->first); + uint64_t NextRelocationOffset = + (RI == BF.getMoveRelocations().end() ? BF.getSize() : RI->first); + auto NextStop = std::min(NextLabelOffset, NextRelocationOffset); + assert(NextStop <= BF.getSize() && "internal overflow error"); + if (FunctionOffset < NextStop) { + Streamer.EmitBytes(FunctionContents.slice(FunctionOffset, NextStop)); + FunctionOffset = NextStop; + } + if (LI != BF.getLabels().end() && FunctionOffset == LI->first) { + Streamer.EmitLabel(LI->second); + DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << LI->second->getName() + << " at offset 0x" << Twine::utohexstr(LI->first) << '\n'); + ++LI; + } + if (RI != BF.getMoveRelocations().end() && FunctionOffset == RI->first) { + auto RelocationSize = RI->second.emit(&Streamer); + DEBUG(dbgs() << "BOLT-DEBUG: emitted relocation for symbol " + << RI->second.Symbol->getName() << " at offset 0x" + << Twine::utohexstr(RI->first) + << " with size " << RelocationSize << '\n'); + FunctionOffset += RelocationSize; + ++RI; + } + } + assert(FunctionOffset <= BF.getSize() && "overflow error"); + if (FunctionOffset < BF.getSize()) { + Streamer.EmitBytes(FunctionContents.substr(FunctionOffset)); + } +} + +void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) { + for (const auto &Section : BC.sections()) { + if (!Section.hasRelocations() || !Section.hasSectionRef()) + continue; + + StringRef SectionName = Section.getName(); + std::string EmitName = Section.isReordered() + ? std::string(Section.getOutputName()) + : OrgSecPrefix.str() + std::string(SectionName); + Section.emitAsData(Streamer, EmitName); + } +} + +namespace llvm { +namespace bolt { + +void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC, + StringRef OrgSecPrefix) { + BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix); +} + +void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF, + bool EmitColdPart, bool EmitCodeOnly) { + BinaryEmitter(Streamer, BF.getBinaryContext()). + emitFunctionBody(BF, EmitColdPart, EmitCodeOnly); +} + +} // namespace bolt +} // namespace llvm diff --git a/bolt/src/BinaryEmitter.h b/bolt/src/BinaryEmitter.h new file mode 100644 index 000000000000..8c73c552cd3f --- /dev/null +++ b/bolt/src/BinaryEmitter.h @@ -0,0 +1,37 @@ +//===--- BinaryEmitter.h - collection of functions to emit code and data --===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVM_BOLT_BINARY_EMITTER_H +#define LLVM_TOOLS_LLVM_BOLT_BINARY_EMITTER_H + +#include "BinaryContext.h" +#include "llvm/MC/MCStreamer.h" + +namespace llvm { +namespace bolt { + +/// Emit all code and data in the BinaryContext \p BC. +/// +/// \p OrgSecPrefix is used to modify name of emitted original sections +/// contained in \p BC. This is done to distinguish them from sections emitted +/// by LLVM backend. +void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC, + StringRef OrgSecPrefix = ""); + +/// Emit \p BF function code. The caller is responsible for emitting function +/// symbol(s) and setting the section to emit the code to. +void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF, + bool EmitColdPart, bool EmitCodeOnly = false); + +} // namespace bolt +} // namespace llvm + +#endif diff --git a/bolt/src/BinaryFunction.cpp b/bolt/src/BinaryFunction.cpp index c43e16fea80e..ab33f3be221e 100644 --- a/bolt/src/BinaryFunction.cpp +++ b/bolt/src/BinaryFunction.cpp @@ -62,26 +62,6 @@ extern cl::opt StrictMode; extern cl::opt UpdateDebugSections; extern cl::opt Verbosity; -cl::opt -AlignBlocks("align-blocks", - cl::desc("align basic blocks"), - cl::init(false), - cl::ZeroOrMore, - cl::cat(BoltOptCategory)); - -cl::opt -AlignMacroOpFusion("align-macro-fusion", - cl::desc("fix instruction alignment for macro-fusion (x86 relocation mode)"), - cl::init(MFT_HOT), - cl::values(clEnumValN(MFT_NONE, "none", - "do not insert alignment no-ops for macro-fusion"), - clEnumValN(MFT_HOT, "hot", - "only insert alignment no-ops on hot execution paths (default)"), - clEnumValN(MFT_ALL, "all", - "always align instructions to allow macro-fusion")), - cl::ZeroOrMore, - cl::cat(BoltRelocCategory)); - cl::opt CheckEncoding("check-encoding", cl::desc("perform verification of LLVM instruction encoding/decoding. " @@ -139,13 +119,6 @@ PrintDynoStatsOnly("print-dyno-stats-only", cl::Hidden, cl::cat(BoltCategory)); -static cl::opt -PrintJumpTables("print-jump-tables", - cl::desc("print jump tables"), - cl::ZeroOrMore, - cl::Hidden, - cl::cat(BoltCategory)); - static cl::list PrintOnly("print-only", cl::CommaSeparated, @@ -1003,9 +976,9 @@ void BinaryFunction::disassemble() { const uint64_t AbsoluteInstrAddr = getAddress() + Offset; // Check for data inside code and ignore it - if (DataOffsets.find(Offset) != DataOffsets.end()) { - auto Iter = CodeOffsets.upper_bound(Offset); - if (Iter != CodeOffsets.end()) { + if (Islands.DataOffsets.find(Offset) != Islands.DataOffsets.end()) { + auto Iter = Islands.CodeOffsets.upper_bound(Offset); + if (Iter != Islands.CodeOffsets.end()) { Size = *Iter - Offset; continue; } @@ -1022,8 +995,8 @@ void BinaryFunction::disassemble() { // can have 0-byte padding at the end. bool IsZeroPadding = true; uint64_t EndOfCode = getSize(); - auto Iter = DataOffsets.upper_bound(Offset); - if (Iter != DataOffsets.end()) + auto Iter = Islands.DataOffsets.upper_bound(Offset); + if (Iter != Islands.DataOffsets.end()) EndOfCode = *Iter; for (auto I = Offset; I < EndOfCode; ++I) { if (FunctionData[I] != 0) { @@ -2652,151 +2625,6 @@ uint64_t BinaryFunction::getEditDistance() const { BasicBlocksLayout); } -void BinaryFunction::emitBody(MCStreamer &Streamer, bool EmitColdPart, - bool EmitCodeOnly) { - if (!EmitCodeOnly && EmitColdPart && hasConstantIsland()) - duplicateConstantIslands(); - - // Track first emitted instruction with debug info. - bool FirstInstr = true; - for (auto BB : layout()) { - if (EmitColdPart != BB->isCold()) - continue; - - if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) - && BB->getAlignment() > 1) { - Streamer.EmitCodeAlignment(BB->getAlignment(), - BB->getAlignmentMaxBytes()); - } - Streamer.EmitLabel(BB->getLabel()); - - // Check if special alignment for macro-fusion is needed. - bool MayNeedMacroFusionAlignment = - (opts::AlignMacroOpFusion == MFT_ALL) || - (opts::AlignMacroOpFusion == MFT_HOT && - BB->getKnownExecutionCount()); - BinaryBasicBlock::const_iterator MacroFusionPair; - if (MayNeedMacroFusionAlignment) { - MacroFusionPair = BB->getMacroOpFusionPair(); - if (MacroFusionPair == BB->end()) - MayNeedMacroFusionAlignment = false; - } - - SMLoc LastLocSeen; - // Remember if the last instruction emitted was a prefix. - bool LastIsPrefix = false; - for (auto I = BB->begin(), E = BB->end(); I != E; ++I) { - auto &Instr = *I; - - if (EmitCodeOnly && BC.MII->get(Instr.getOpcode()).isPseudo()) - continue; - - // Handle pseudo instructions. - if (BC.MIB->isEHLabel(Instr)) { - const auto *Label = BC.MIB->getTargetSymbol(Instr); - assert(Instr.getNumOperands() >= 1 && Label && - "bad EH_LABEL instruction"); - Streamer.EmitLabel(const_cast(Label)); - continue; - } - if (BC.MIB->isCFI(Instr)) { - Streamer.EmitCFIInstruction(*getCFIFor(Instr)); - continue; - } - - // Handle macro-fusion alignment. If we emitted a prefix as - // the last instruction, we should've already emitted the associated - // alignment hint, so don't emit it twice. - if (MayNeedMacroFusionAlignment && !LastIsPrefix && I == MacroFusionPair){ - // This assumes the second instruction in the macro-op pair will get - // assigned to its own MCRelaxableFragment. Since all JCC instructions - // are relaxable, we should be safe. - Streamer.EmitNeverAlignCodeAtEnd(/*Alignment to avoid=*/64); - } - - if (!EmitCodeOnly && opts::UpdateDebugSections && UnitLineTable.first) { - LastLocSeen = emitLineInfo(Instr.getLoc(), LastLocSeen, FirstInstr); - FirstInstr = false; - } - - // Prepare to tag this location with a label if we need to keep track of - // the location of calls/returns for BOLT address translation maps - if (!EmitCodeOnly && requiresAddressTranslation() && - BC.MIB->hasAnnotation(Instr, "Offset")) { - const auto Offset = BC.MIB->getAnnotationAs(Instr, "Offset"); - MCSymbol *LocSym = BC.Ctx->createTempSymbol(/*CanBeUnnamed=*/true); - Streamer.EmitLabel(LocSym); - BB->getLocSyms().emplace_back(std::make_pair(Offset, LocSym)); - } - - Streamer.EmitInstruction(Instr, *BC.STI); - LastIsPrefix = BC.MIB->isPrefix(Instr); - } - } - - if (!EmitCodeOnly) - emitConstantIslands(Streamer, EmitColdPart); -} - -void BinaryFunction::emitBodyRaw(MCStreamer *Streamer) { - - // #14998851: Fix gold linker's '--emit-relocs'. - assert(false && - "cannot emit raw body unless relocation accuracy is guaranteed"); - - assert(!isInjected() && "cannot emit raw body of injected function"); - - // Raw contents of the function. - StringRef SectionContents = InputSection->getContents(); - - // Raw contents of the function. - StringRef FunctionContents = - SectionContents.substr(getAddress() - InputSection->getAddress(), - getSize()); - - if (opts::Verbosity) - outs() << "BOLT-INFO: emitting function " << *this << " in raw (" - << getSize() << " bytes).\n"; - - // We split the function blob into smaller blocks and output relocations - // and/or labels between them. - uint64_t FunctionOffset = 0; - auto LI = Labels.begin(); - auto RI = MoveRelocations.begin(); - while (LI != Labels.end() || - RI != MoveRelocations.end()) { - uint64_t NextLabelOffset = (LI == Labels.end() ? getSize() : LI->first); - uint64_t NextRelocationOffset = - (RI == MoveRelocations.end() ? getSize() : RI->first); - auto NextStop = std::min(NextLabelOffset, NextRelocationOffset); - assert(NextStop <= getSize() && "internal overflow error"); - if (FunctionOffset < NextStop) { - Streamer->EmitBytes( - FunctionContents.slice(FunctionOffset, NextStop)); - FunctionOffset = NextStop; - } - if (LI != Labels.end() && FunctionOffset == LI->first) { - Streamer->EmitLabel(LI->second); - DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << LI->second->getName() - << " at offset 0x" << Twine::utohexstr(LI->first) << '\n'); - ++LI; - } - if (RI != MoveRelocations.end() && FunctionOffset == RI->first) { - auto RelocationSize = RI->second.emit(Streamer); - DEBUG(dbgs() << "BOLT-DEBUG: emitted relocation for symbol " - << RI->second.Symbol->getName() << " at offset 0x" - << Twine::utohexstr(RI->first) - << " with size " << RelocationSize << '\n'); - FunctionOffset += RelocationSize; - ++RI; - } - } - assert(FunctionOffset <= getSize() && "overflow error"); - if (FunctionOffset < getSize()) { - Streamer->EmitBytes(FunctionContents.substr(FunctionOffset)); - } -} - void BinaryFunction::setTrapOnEntry() { clearList(Instructions); clearList(IgnoredBranches); @@ -2811,133 +2639,6 @@ void BinaryFunction::setTrapOnEntry() { TrapsOnEntry = true; } -void BinaryFunction::emitConstantIslands( - MCStreamer &Streamer, bool EmitColdPart, - BinaryFunction *OnBehalfOf) { - if (DataOffsets.empty() && IslandDependency.empty()) - return; - - if (!OnBehalfOf) { - if (!EmitColdPart) - Streamer.EmitLabel(getFunctionConstantIslandLabel()); - else - Streamer.EmitLabel(getFunctionColdConstantIslandLabel()); - } - - assert((!OnBehalfOf || IslandProxies[OnBehalfOf].size() > 0) && - "spurious OnBehalfOf constant island emission"); - - assert(!isInjected() && - "injected functions should not have constant islands"); - // Raw contents of the function. - StringRef SectionContents = InputSection->getContents(); - - // Raw contents of the function. - StringRef FunctionContents = - SectionContents.substr(getAddress() - InputSection->getAddress(), - getMaxSize()); - - if (opts::Verbosity && !OnBehalfOf) - outs() << "BOLT-INFO: emitting constant island for function " << *this - << "\n"; - - // We split the island into smaller blocks and output labels between them. - auto IS = IslandOffsets.begin(); - for (auto DataIter = DataOffsets.begin(); DataIter != DataOffsets.end(); - ++DataIter) { - uint64_t FunctionOffset = *DataIter; - uint64_t EndOffset = 0ULL; - - // Determine size of this data chunk - auto NextData = std::next(DataIter); - auto CodeIter = CodeOffsets.lower_bound(*DataIter); - if (CodeIter == CodeOffsets.end() && NextData == DataOffsets.end()) { - EndOffset = getMaxSize(); - } else if (CodeIter == CodeOffsets.end()) { - EndOffset = *NextData; - } else if (NextData == DataOffsets.end()) { - EndOffset = *CodeIter; - } else { - EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter; - } - - if (FunctionOffset == EndOffset) - continue; // Size is zero, nothing to emit - - // Emit labels, relocs and data - auto RI = MoveRelocations.lower_bound(FunctionOffset); - while ((IS != IslandOffsets.end() && IS->first < EndOffset) || - (RI != MoveRelocations.end() && RI->first < EndOffset)) { - auto NextLabelOffset = IS == IslandOffsets.end() ? EndOffset : IS->first; - auto NextRelOffset = RI == MoveRelocations.end() ? EndOffset : RI->first; - auto NextStop = std::min(NextLabelOffset, NextRelOffset); - assert(NextStop <= EndOffset && "internal overflow error"); - if (FunctionOffset < NextStop) { - Streamer.EmitBytes(FunctionContents.slice(FunctionOffset, NextStop)); - FunctionOffset = NextStop; - } - if (IS != IslandOffsets.end() && FunctionOffset == IS->first) { - // This is a slightly complex code to decide which label to emit. We - // have 4 cases to handle: regular symbol, cold symbol, regular or cold - // symbol being emitted on behalf of an external function. - if (!OnBehalfOf) { - if (!EmitColdPart) { - DEBUG(dbgs() << "BOLT-DEBUG: emitted label " - << IS->second->getName() << " at offset 0x" - << Twine::utohexstr(IS->first) << '\n'); - if (IS->second->isUndefined()) - Streamer.EmitLabel(IS->second); - else - assert(hasName(IS->second->getName())); - } else if (ColdIslandSymbols.count(IS->second) != 0) { - DEBUG(dbgs() << "BOLT-DEBUG: emitted label " - << ColdIslandSymbols[IS->second]->getName() << '\n'); - if (ColdIslandSymbols[IS->second]->isUndefined()) - Streamer.EmitLabel(ColdIslandSymbols[IS->second]); - } - } else { - if (!EmitColdPart) { - if (MCSymbol *Sym = IslandProxies[OnBehalfOf][IS->second]) { - DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName() - << '\n'); - Streamer.EmitLabel(Sym); - } - } else if (MCSymbol *Sym = - ColdIslandProxies[OnBehalfOf][IS->second]) { - DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName() - << '\n'); - Streamer.EmitLabel(Sym); - } - } - ++IS; - } - if (RI != MoveRelocations.end() && FunctionOffset == RI->first) { - auto RelocationSize = RI->second.emit(&Streamer); - DEBUG(dbgs() << "BOLT-DEBUG: emitted relocation for symbol " - << RI->second.Symbol->getName() << " at offset 0x" - << Twine::utohexstr(RI->first) - << " with size " << RelocationSize << '\n'); - FunctionOffset += RelocationSize; - ++RI; - } - } - assert(FunctionOffset <= EndOffset && "overflow error"); - if (FunctionOffset < EndOffset) { - Streamer.EmitBytes(FunctionContents.slice(FunctionOffset, EndOffset)); - } - } - assert(IS == IslandOffsets.end() && "some symbols were not emitted!"); - - if (OnBehalfOf) - return; - // Now emit constant islands from other functions that we may have used in - // this function. - for (auto *ExternalFunc : IslandDependency) { - ExternalFunc->emitConstantIslands(Streamer, EmitColdPart, this); - } - -} - void BinaryFunction::duplicateConstantIslands() { for (auto BB : layout()) { if (!BB->isCold()) @@ -2952,22 +2653,24 @@ void BinaryFunction::duplicateConstantIslands() { } auto *Symbol = BC.MIB->getTargetSymbol(Inst, OpNum); // Check if this is an island symbol - if (!IslandSymbols.count(Symbol) && !ProxyIslandSymbols.count(Symbol)) + if (!Islands.Symbols.count(Symbol) && + !Islands.ProxySymbols.count(Symbol)) continue; // Create cold symbol, if missing - auto ISym = ColdIslandSymbols.find(Symbol); + auto ISym = Islands.ColdSymbols.find(Symbol); MCSymbol *ColdSymbol; - if (ISym != ColdIslandSymbols.end()) { + if (ISym != Islands.ColdSymbols.end()) { ColdSymbol = ISym->second; } else { ColdSymbol = BC.Ctx->getOrCreateSymbol(Symbol->getName() + ".cold"); - ColdIslandSymbols[Symbol] = ColdSymbol; + Islands.ColdSymbols[Symbol] = ColdSymbol; // Check if this is a proxy island symbol and update owner proxy map - if (ProxyIslandSymbols.count(Symbol)) { - BinaryFunction *Owner = ProxyIslandSymbols[Symbol]; - auto IProxiedSym = Owner->IslandProxies[this].find(Symbol); - Owner->ColdIslandProxies[this][IProxiedSym->second] = ColdSymbol; + if (Islands.ProxySymbols.count(Symbol)) { + BinaryFunction *Owner = Islands.ProxySymbols[Symbol]; + auto IProxiedSym = Owner->Islands.Proxies[this].find(Symbol); + Owner->Islands.ColdProxies[this][IProxiedSym->second] = + ColdSymbol; } } @@ -3891,63 +3594,6 @@ bool BinaryFunction::isSymbolValidInScope(const SymbolRef &Symbol, return true; } -SMLoc BinaryFunction::emitLineInfo(SMLoc NewLoc, SMLoc PrevLoc, - bool FirstInstr) const { - auto *FunctionCU = UnitLineTable.first; - const auto *FunctionLineTable = UnitLineTable.second; - assert(FunctionCU && "cannot emit line info for function without CU"); - - auto RowReference = DebugLineTableRowRef::fromSMLoc(NewLoc); - - // Check if no new line info needs to be emitted. - if (RowReference == DebugLineTableRowRef::NULL_ROW || - NewLoc.getPointer() == PrevLoc.getPointer()) - return PrevLoc; - - unsigned CurrentFilenum = 0; - const auto *CurrentLineTable = FunctionLineTable; - - // If the CU id from the current instruction location does not - // match the CU id from the current function, it means that we - // have come across some inlined code. We must look up the CU - // for the instruction's original function and get the line table - // from that. - const auto FunctionUnitIndex = FunctionCU->getOffset(); - const auto CurrentUnitIndex = RowReference.DwCompileUnitIndex; - if (CurrentUnitIndex != FunctionUnitIndex) { - CurrentLineTable = BC.DwCtx->getLineTableForUnit( - BC.DwCtx->getCompileUnitForOffset(CurrentUnitIndex)); - // Add filename from the inlined function to the current CU. - CurrentFilenum = - BC.addDebugFilenameToUnit(FunctionUnitIndex, CurrentUnitIndex, - CurrentLineTable->Rows[RowReference.RowIndex - 1].File); - } - - const auto &CurrentRow = CurrentLineTable->Rows[RowReference.RowIndex - 1]; - if (!CurrentFilenum) - CurrentFilenum = CurrentRow.File; - - unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) | - (DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) | - (DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) | - (DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin); - - // Always emit is_stmt at the beginning of function fragment. - if (FirstInstr) - Flags |= DWARF2_FLAG_IS_STMT; - - BC.Ctx->setCurrentDwarfLoc( - CurrentFilenum, - CurrentRow.Line, - CurrentRow.Column, - Flags, - CurrentRow.Isa, - CurrentRow.Discriminator); - BC.Ctx->setDwarfCompileUnitID(FunctionUnitIndex); - - return NewLoc; -} - void BinaryFunction::adjustExecutionCount(uint64_t Count) { if (getKnownExecutionCount() == 0 || Count == 0) return; @@ -3974,47 +3620,6 @@ BinaryFunction::~BinaryFunction() { } } -void BinaryFunction::emitJumpTables(MCStreamer *Streamer) { - if (JumpTables.empty()) - return; - if (opts::PrintJumpTables) { - outs() << "BOLT-INFO: jump tables for function " << *this << ":\n"; - } - for (auto &JTI : JumpTables) { - auto &JT = *JTI.second; - if (opts::PrintJumpTables) - JT.print(outs()); - if ((opts::JumpTables == JTS_BASIC || !isSimple()) && BC.HasRelocations) { - JT.updateOriginal(); - } else { - MCSection *HotSection, *ColdSection; - if (opts::JumpTables == JTS_BASIC) { - std::string Name = ".local." + JT.Labels[0]->getName().str(); - std::replace(Name.begin(), Name.end(), '/', '.'); - auto &Section = BC.registerOrUpdateSection(Name, - ELF::SHT_PROGBITS, - ELF::SHF_ALLOC); - Section.setAnonymous(true); - JT.setOutputSection(Section); - HotSection = BC.Ctx->getELFSection(Name, - ELF::SHT_PROGBITS, - ELF::SHF_ALLOC); - ColdSection = HotSection; - } else { - if (isSimple()) { - HotSection = BC.MOFI->getReadOnlySection(); - ColdSection = BC.MOFI->getReadOnlyColdSection(); - } else { - HotSection = hasProfile() ? BC.MOFI->getReadOnlySection() - : BC.MOFI->getReadOnlyColdSection(); - ColdSection = HotSection; - } - } - JT.emit(Streamer, HotSection, ColdSection); - } - } -} - void BinaryFunction::calculateLoopInfo() { // Discover loops. BinaryDominatorTree DomTree; diff --git a/bolt/src/BinaryFunction.h b/bolt/src/BinaryFunction.h index 730eb6454f4b..18246946ee3b 100644 --- a/bolt/src/BinaryFunction.h +++ b/bolt/src/BinaryFunction.h @@ -135,6 +135,39 @@ public: PF_MEMEVENT = 4, /// Profile has mem events. }; + /// Struct for tracking exception handling ranges. + struct CallSite { + const MCSymbol *Start; + const MCSymbol *End; + const MCSymbol *LP; + uint64_t Action; + }; + + using IslandProxiesType = + std::map>; + + struct IslandInfo { + /// Temporary holder of offsets that are data markers (used in AArch) + /// It is possible to have data in code sections. To ease the identification + /// of data in code sections, the ABI requires the symbol table to have + /// symbols named "$d" identifying the start of data inside code and "$x" + /// identifying the end of a chunk of data inside code. DataOffsets contain + /// all offsets of $d symbols and CodeOffsets all offsets of $x symbols. + std::set DataOffsets; + std::set CodeOffsets; + + /// Offsets in function that are data values in a constant island identified + /// after disassembling + std::map Offsets; + SmallPtrSet Symbols; + std::map ProxySymbols; + std::map ColdSymbols; + /// Keeps track of other functions we depend on because there is a reference + /// to the constant islands in them. + IslandProxiesType Proxies, ColdProxies; + std::set Dependency; // The other way around + }; + static constexpr uint64_t COUNT_NO_PROFILE = BinaryBasicBlock::COUNT_NO_PROFILE; @@ -372,14 +405,6 @@ private: /// Temporary holder of offsets that are potentially entry points. std::unordered_set EntryOffsets; - /// Temporary holder of offsets that are data markers (used in AArch) - /// It is possible to have data in code sections. To ease the identification - /// of data in code sections, the ABI requires the symbol table to have - /// symbols named "$d" identifying the start of data inside code and "$x" - /// identifying the end of a chunk of data inside code. DataOffsets contain - /// all offsets of $d symbols and CodeOffsets all offsets of $x symbols. - std::set DataOffsets; - std::set CodeOffsets; /// The address offset where we emitted the constant island, that is, the /// chunk of data in the function code area (AArch only) int64_t OutputDataOffset{0}; @@ -439,13 +464,7 @@ private: /// remember-restore CFI instructions when rewriting CFI. DenseMap> FrameRestoreEquivalents; - /// Exception handling ranges. - struct CallSite { - const MCSymbol *Start; - const MCSymbol *End; - const MCSymbol *LP; - uint64_t Action; - }; + // For tracking exception handling ranges. std::vector CallSites; std::vector ColdCallSites; @@ -475,12 +494,6 @@ private: /// -> std::map JumpTables; - /// Iterate over all jump tables associated with this function. - iterator_range::const_iterator> - jumpTables() const { - return make_range(JumpTables.begin(), JumpTables.end()); - } - /// All jump table sites in the function before CFG is built. std::vector> JTSites; @@ -488,19 +501,11 @@ private: std::map Relocations; /// Map of relocations used for moving the function body as it is. - std::map MoveRelocations; + using MoveRelocationsTy = std::map; + MoveRelocationsTy MoveRelocations; - /// Offsets in function that are data values in a constant island identified - /// after disassembling - std::map IslandOffsets; - SmallPtrSet IslandSymbols; - std::map ProxyIslandSymbols; - std::map ColdIslandSymbols; - /// Keeps track of other functions we depend on because there is a reference - /// to the constant islands in them. - std::map> - IslandProxies, ColdIslandProxies; - std::set IslandDependency; // The other way around + /// Information on function constant islands. + IslandInfo Islands; // Blocks are kept sorted in the layout order. If we need to change the // layout (if BasicBlocksLayout stores a different order than BasicBlocks), @@ -561,12 +566,12 @@ private: /// Register an entry point at a given \p Offset into the function. void markDataAtOffset(uint64_t Offset) { - DataOffsets.emplace(Offset); + Islands.DataOffsets.emplace(Offset); } /// Register an entry point at a given \p Offset into the function. void markCodeAtOffset(uint64_t Offset) { - CodeOffsets.emplace(Offset); + Islands.CodeOffsets.emplace(Offset); } /// Register an entry point at a given \p Offset into the function. @@ -637,14 +642,6 @@ private: DenseMap> computeLocalUDChain(const MCInst *CurInstr); - /// Emit line number information corresponding to \p NewLoc. \p PrevLoc - /// provides a context for de-duplication of line number info. - /// \p FirstInstr indicates if \p NewLoc represents the first instruction - /// in a sequence, such as a function fragment. - /// - /// Return new current location which is either \p NewLoc or \p PrevLoc. - SMLoc emitLineInfo(SMLoc NewLoc, SMLoc PrevLoc, bool FirstInstr) const; - BinaryFunction& operator=(const BinaryFunction &) = delete; BinaryFunction(const BinaryFunction &) = delete; @@ -776,6 +773,12 @@ public: return iterator_range(cie_begin(), cie_end()); } + /// Iterate over all jump tables associated with this function. + iterator_range::const_iterator> + jumpTables() const { + return make_range(JumpTables.begin(), JumpTables.end()); + } + /// Returns the raw binary encoding of this function. ErrorOr> getData() const; @@ -1360,6 +1363,42 @@ public: return PersonalityEncoding; } + const std::vector &getCallSites() const { + return CallSites; + } + + const std::vector &getColdCallSites() const { + return ColdCallSites; + } + + const ArrayRef getLSDAActionTable() const { + return LSDAActionTable; + } + + const std::vector &getLSDATypeTable() const { + return LSDATypeTable; + } + + const ArrayRef getLSDATypeIndexTable() const { + return LSDATypeIndexTable; + } + + const MoveRelocationsTy &getMoveRelocations() const { + return MoveRelocations; + } + + const LabelsMapType &getLabels() const { + return Labels; + } + + IslandInfo &getIslandInfo() { + return Islands; + } + + const IslandInfo &getIslandInfo() const { + return Islands; + } + /// Return true if the function has CFI instructions bool hasCFI() const { return !FrameInstructions.empty() || !CIEFrameInstructions.empty(); @@ -1873,11 +1912,12 @@ public: // Internal bookkeeping const auto Offset = Address - getAddress(); - assert((!IslandOffsets.count(Offset) || IslandOffsets[Offset] == Symbol) && + assert((!Islands.Offsets.count(Offset) || + Islands.Offsets[Offset] == Symbol) && "Inconsistent island symbol management"); - if (!IslandOffsets.count(Offset)) { - IslandOffsets[Offset] = Symbol; - IslandSymbols.insert(Symbol); + if (!Islands.Offsets.count(Offset)) { + Islands.Offsets[Offset] = Symbol; + Islands.Symbols.insert(Symbol); } return Symbol; } @@ -1893,14 +1933,14 @@ public: return nullptr; MCSymbol *Proxy; - if (!IslandProxies[&Referrer].count(Symbol)) { + if (!Islands.Proxies[&Referrer].count(Symbol)) { Proxy = BC.Ctx->getOrCreateSymbol(Symbol->getName() + ".proxy.for." + Referrer.getPrintName()); - IslandProxies[&Referrer][Symbol] = Proxy; - IslandProxies[&Referrer][Proxy] = Symbol; + Islands.Proxies[&Referrer][Symbol] = Proxy; + Islands.Proxies[&Referrer][Proxy] = Symbol; } - Proxy = IslandProxies[&Referrer][Symbol]; + Proxy = Islands.Proxies[&Referrer][Symbol]; return Proxy; } @@ -1915,13 +1955,13 @@ public: if (Offset >= getMaxSize()) return false; - auto DataIter = DataOffsets.upper_bound(Offset); - if (DataIter == DataOffsets.begin()) + auto DataIter = Islands.DataOffsets.upper_bound(Offset); + if (DataIter == Islands.DataOffsets.begin()) return false; DataIter = std::prev(DataIter); - auto CodeIter = CodeOffsets.upper_bound(Offset); - if (CodeIter == CodeOffsets.begin()) + auto CodeIter = Islands.CodeOffsets.upper_bound(Offset); + if (CodeIter == Islands.CodeOffsets.begin()) return true; return *std::prev(CodeIter) <= *DataIter; @@ -1930,20 +1970,21 @@ public: uint64_t estimateConstantIslandSize(const BinaryFunction *OnBehalfOf = nullptr) const { uint64_t Size = 0; - for (auto DataIter = DataOffsets.begin(); DataIter != DataOffsets.end(); + for (auto DataIter = Islands.DataOffsets.begin(); + DataIter != Islands.DataOffsets.end(); ++DataIter) { auto NextData = std::next(DataIter); - auto CodeIter = CodeOffsets.lower_bound(*DataIter); - if (CodeIter == CodeOffsets.end() && - NextData == DataOffsets.end()) { + auto CodeIter = Islands.CodeOffsets.lower_bound(*DataIter); + if (CodeIter == Islands.CodeOffsets.end() && + NextData == Islands.DataOffsets.end()) { Size += getMaxSize() - *DataIter; continue; } uint64_t NextMarker; - if (CodeIter == CodeOffsets.end()) + if (CodeIter == Islands.CodeOffsets.end()) NextMarker = *NextData; - else if (NextData == DataOffsets.end()) + else if (NextData == Islands.DataOffsets.end()) NextMarker = *CodeIter; else NextMarker = (*CodeIter > *NextData) ? *NextData : *CodeIter; @@ -1952,14 +1993,14 @@ public: } if (!OnBehalfOf) { - for (auto *ExternalFunc : IslandDependency) + for (auto *ExternalFunc : Islands.Dependency) Size += ExternalFunc->estimateConstantIslandSize(this); } return Size; } bool hasConstantIsland() const { - return !DataOffsets.empty(); + return !Islands.DataOffsets.empty(); } /// Return true iff the symbol could be seen inside this function otherwise @@ -2184,24 +2225,6 @@ public: /// Update exception handling ranges for the function. void updateEHRanges(); - /// Emit exception handling ranges for the function. - void emitLSDA(MCStreamer *Streamer, bool EmitColdPart); - - /// Emit jump tables for the function. - void emitJumpTables(MCStreamer *Streamer); - - /// Emit function code. The caller is responsible for emitting function - /// symbol(s) and setting the section to emit the code to. - void emitBody(MCStreamer &Streamer, bool EmitColdPart, - bool EmitCodeOnly = false); - - /// Emit function as a blob with relocations and labels for relocations. - void emitBodyRaw(MCStreamer *Streamer); - - /// Helper for emitBody to write data inside a function (used for AArch64) - void emitConstantIslands(MCStreamer &Streamer, bool EmitColdPart, - BinaryFunction *OnBehalfOf = nullptr); - /// Traverse cold basic blocks and replace references to constants in islands /// with a proxy symbol for the duplicated constant island that is going to be /// emitted in the cold region. diff --git a/bolt/src/CMakeLists.txt b/bolt/src/CMakeLists.txt index 8088e9fae2c9..299e6616ae69 100644 --- a/bolt/src/CMakeLists.txt +++ b/bolt/src/CMakeLists.txt @@ -76,6 +76,7 @@ add_llvm_tool(llvm-bolt BinaryBasicBlock.cpp BinaryContext.cpp BinaryData.cpp + BinaryEmitter.cpp BinaryFunction.cpp BinaryFunctionProfile.cpp BinaryPassManager.cpp diff --git a/bolt/src/DWARFRewriter.cpp b/bolt/src/DWARFRewriter.cpp index e64425731871..9a8903124f23 100644 --- a/bolt/src/DWARFRewriter.cpp +++ b/bolt/src/DWARFRewriter.cpp @@ -369,64 +369,6 @@ void DWARFRewriter::updateDWARFObjectAddressRanges( } } -void DWARFRewriter::updateDebugLineInfoForNonSimpleFunctions() { - for (auto &It : BC.getBinaryFunctions()) { - const auto &Function = It.second; - - if (Function.isSimple()) - continue; - - auto ULT = Function.getDWARFUnitLineTable(); - auto Unit = ULT.first; - auto LineTable = ULT.second; - - if (!LineTable) - continue; // nothing to update for this function - - std::vector Results; - MCSectionELF *FunctionSection = - BC.Ctx->getELFSection(Function.getCodeSectionName(), - ELF::SHT_PROGBITS, - ELF::SHF_EXECINSTR | ELF::SHF_ALLOC); - - uint64_t Address = It.first; - if (LineTable->lookupAddressRange(Address, Function.getMaxSize(), - Results)) { - auto &OutputLineTable = - BC.Ctx->getMCDwarfLineTable(Unit->getOffset()).getMCLineSections(); - for (auto RowIndex : Results) { - const auto &Row = LineTable->Rows[RowIndex]; - BC.Ctx->setCurrentDwarfLoc( - Row.File, - Row.Line, - Row.Column, - (DWARF2_FLAG_IS_STMT * Row.IsStmt) | - (DWARF2_FLAG_BASIC_BLOCK * Row.BasicBlock) | - (DWARF2_FLAG_PROLOGUE_END * Row.PrologueEnd) | - (DWARF2_FLAG_EPILOGUE_BEGIN * Row.EpilogueBegin), - Row.Isa, - Row.Discriminator, - Row.Address); - auto Loc = BC.Ctx->getCurrentDwarfLoc(); - BC.Ctx->clearDwarfLocSeen(); - OutputLineTable.addLineEntry(MCDwarfLineEntry{nullptr, Loc}, - FunctionSection); - } - // Add an empty entry past the end of the function - // for end_sequence mark. - BC.Ctx->setCurrentDwarfLoc(0, 0, 0, 0, 0, 0, - Address + Function.getMaxSize()); - auto Loc = BC.Ctx->getCurrentDwarfLoc(); - BC.Ctx->clearDwarfLocSeen(); - OutputLineTable.addLineEntry(MCDwarfLineEntry{nullptr, Loc}, - FunctionSection); - } else { - DEBUG(dbgs() << "BOLT-DEBUG: Function " << Function - << " has no associated line number information.\n"); - } - } -} - void DWARFRewriter::updateLineTableOffsets() { const auto *LineSection = BC.Ctx->getObjectFileInfo()->getDwarfLineSection(); diff --git a/bolt/src/DWARFRewriter.h b/bolt/src/DWARFRewriter.h index bae0f62f5c74..f1f0adfa1938 100644 --- a/bolt/src/DWARFRewriter.h +++ b/bolt/src/DWARFRewriter.h @@ -134,10 +134,6 @@ public: /// Computes output .debug_line line table offsets for each compile unit, /// and updates stmt_list for a corresponding compile unit. void updateLineTableOffsets(); - - /// Updates debug line information for non-simple functions, which are not - /// rewritten. - void updateDebugLineInfoForNonSimpleFunctions(); }; } // namespace bolt diff --git a/bolt/src/Exceptions.cpp b/bolt/src/Exceptions.cpp index 094232b604af..5839f84e3ad5 100644 --- a/bolt/src/Exceptions.cpp +++ b/bolt/src/Exceptions.cpp @@ -54,28 +54,6 @@ PrintExceptions("print-exceptions", namespace llvm { namespace bolt { -namespace { - -unsigned getEncodingSize(unsigned Encoding, BinaryContext &BC) { - switch (Encoding & 0x0f) { - default: llvm_unreachable("unknown encoding"); - case dwarf::DW_EH_PE_absptr: - case dwarf::DW_EH_PE_signed: - return BC.AsmInfo->getCodePointerSize(); - case dwarf::DW_EH_PE_udata2: - case dwarf::DW_EH_PE_sdata2: - return 2; - case dwarf::DW_EH_PE_udata4: - case dwarf::DW_EH_PE_sdata4: - return 4; - case dwarf::DW_EH_PE_udata8: - case dwarf::DW_EH_PE_sdata8: - return 8; - } -} - -} // anonymous namespace - // Read and dump the .gcc_exception_table section entry. // // .gcc_except_table section contains a set of Language-Specific Data Areas - @@ -153,7 +131,7 @@ void BinaryFunction::parseLSDA(ArrayRef LSDASectionData, uintptr_t TTypeEnd = 0; if (TTypeEncoding != DW_EH_PE_omit) { TTypeEnd = Data.getULEB128(&Offset); - TTypeEncodingSize = getEncodingSize(TTypeEncoding, BC); + TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding); } if (opts::PrintExceptions) { @@ -486,160 +464,6 @@ void BinaryFunction::updateEHRanges() { } } -// The code is based on EHStreamer::emitExceptionTable(). -void BinaryFunction::emitLSDA(MCStreamer *Streamer, bool EmitColdPart) { - const auto *Sites = EmitColdPart ? &ColdCallSites : &CallSites; - if (Sites->empty()) { - return; - } - - - // Calculate callsite table size. Size of each callsite entry is: - // - // sizeof(start) + sizeof(length) + sizeof(LP) + sizeof(uleb128(action)) - // - // or - // - // sizeof(dwarf::DW_EH_PE_data4) * 3 + sizeof(uleb128(action)) - uint64_t CallSiteTableLength = Sites->size() * 4 * 3; - for (const auto &CallSite : *Sites) { - CallSiteTableLength += getULEB128Size(CallSite.Action); - } - - Streamer->SwitchSection(BC.MOFI->getLSDASection()); - - const auto TTypeEncoding = BC.MOFI->getTTypeEncoding(); - const auto TTypeEncodingSize = getEncodingSize(TTypeEncoding, BC); - const auto TTypeAlignment = 4; - - // Type tables have to be aligned at 4 bytes. - Streamer->EmitValueToAlignment(TTypeAlignment); - - // Emit the LSDA label. - auto LSDASymbol = EmitColdPart ? getColdLSDASymbol() : getLSDASymbol(); - assert(LSDASymbol && "no LSDA symbol set"); - Streamer->EmitLabel(LSDASymbol); - - // Corresponding FDE start. - const auto *StartSymbol = EmitColdPart ? getColdSymbol() : getSymbol(); - - // Emit the LSDA header. - - // If LPStart is omitted, then the start of the FDE is used as a base for - // landing pad displacements. Then if a cold fragment starts with - // a landing pad, this means that the first landing pad offset will be 0. - // As a result, an exception handling runtime will ignore this landing pad, - // because zero offset denotes the absence of a landing pad. - // For this reason, we emit LPStart value of 0 and output an absolute value - // of the landing pad in the table. - // - // FIXME: this may break PIEs and DSOs where the base address is not 0. - Streamer->EmitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format - Streamer->EmitIntValue(0, 4); - auto emitLandingPad = [&](const MCSymbol *LPSymbol) { - if (!LPSymbol) { - Streamer->EmitIntValue(0, 4); - return; - } - Streamer->EmitSymbolValue(LPSymbol, 4); - }; - - Streamer->EmitIntValue(TTypeEncoding, 1); // TType format - - // See the comment in EHStreamer::emitExceptionTable() on to use - // uleb128 encoding (which can use variable number of bytes to encode the same - // value) to ensure type info table is properly aligned at 4 bytes without - // iteratively fixing sizes of the tables. - unsigned CallSiteTableLengthSize = getULEB128Size(CallSiteTableLength); - unsigned TTypeBaseOffset = - sizeof(int8_t) + // Call site format - CallSiteTableLengthSize + // Call site table length size - CallSiteTableLength + // Call site table length - LSDAActionTable.size() + // Actions table size - LSDATypeTable.size() * TTypeEncodingSize; // Types table size - unsigned TTypeBaseOffsetSize = getULEB128Size(TTypeBaseOffset); - unsigned TotalSize = - sizeof(int8_t) + // LPStart format - sizeof(int8_t) + // TType format - TTypeBaseOffsetSize + // TType base offset size - TTypeBaseOffset; // TType base offset - unsigned SizeAlign = (4 - TotalSize) & 3; - - // Account for any extra padding that will be added to the call site table - // length. - Streamer->EmitPaddedULEB128IntValue(TTypeBaseOffset, - TTypeBaseOffsetSize + SizeAlign); - - // Emit the landing pad call site table. We use signed data4 since we can emit - // a landing pad in a different part of the split function that could appear - // earlier in the address space than LPStart. - Streamer->EmitIntValue(dwarf::DW_EH_PE_sdata4, 1); - Streamer->EmitULEB128IntValue(CallSiteTableLength); - - for (const auto &CallSite : *Sites) { - - const auto *BeginLabel = CallSite.Start; - const auto *EndLabel = CallSite.End; - - assert(BeginLabel && "start EH label expected"); - assert(EndLabel && "end EH label expected"); - - // Start of the range is emitted relative to the start of current - // function split part. - Streamer->emitAbsoluteSymbolDiff(BeginLabel, StartSymbol, 4); - Streamer->emitAbsoluteSymbolDiff(EndLabel, BeginLabel, 4); - emitLandingPad(CallSite.LP); - Streamer->EmitULEB128IntValue(CallSite.Action); - } - - // Write out action, type, and type index tables at the end. - // - // For action and type index tables there's no need to change the original - // table format unless we are doing function splitting, in which case we can - // split and optimize the tables. - // - // For type table we (re-)encode the table using TTypeEncoding matching - // the current assembler mode. - for (auto const &Byte : LSDAActionTable) { - Streamer->EmitIntValue(Byte, 1); - } - assert(!(TTypeEncoding & dwarf::DW_EH_PE_indirect) && - "indirect type info encoding is not supported yet"); - for (int Index = LSDATypeTable.size() - 1; Index >= 0; --Index) { - // Note: the address could be an indirect one. - const auto TypeAddress = LSDATypeTable[Index]; - switch (TTypeEncoding & 0x70) { - default: - llvm_unreachable("unsupported TTypeEncoding"); - case 0: - Streamer->EmitIntValue(TypeAddress, TTypeEncodingSize); - break; - case dwarf::DW_EH_PE_pcrel: { - if (TypeAddress) { - const auto *TypeSymbol = - BC.getOrCreateGlobalSymbol(TypeAddress, - "TI", - TTypeEncodingSize, - TTypeAlignment); - auto *DotSymbol = BC.Ctx->createTempSymbol(); - Streamer->EmitLabel(DotSymbol); - const auto *SubDotExpr = MCBinaryExpr::createSub( - MCSymbolRefExpr::create(TypeSymbol, *BC.Ctx), - MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), - *BC.Ctx); - Streamer->EmitValue(SubDotExpr, TTypeEncodingSize); - } else { - Streamer->EmitIntValue(0, TTypeEncodingSize); - } - break; - } - } - } - for (auto const &Byte : LSDATypeIndexTable) { - Streamer->EmitIntValue(Byte, 1); - } -} - const uint8_t DWARF_CFI_PRIMARY_OPCODE_MASK = 0xc0; CFIReaderWriter::CFIReaderWriter(const DWARFDebugFrame &EHFrame) { diff --git a/bolt/src/JumpTable.cpp b/bolt/src/JumpTable.cpp index f6686015ba6b..d1708466201e 100644 --- a/bolt/src/JumpTable.cpp +++ b/bolt/src/JumpTable.cpp @@ -110,66 +110,6 @@ void JumpTable::updateOriginal() { } } -uint64_t JumpTable::emit(MCStreamer *Streamer, - MCSection *HotSection, - MCSection *ColdSection) { - // Pre-process entries for aggressive splitting. - // Each label represents a separate switch table and gets its own count - // determining its destination. - std::map LabelCounts; - if (opts::JumpTables > JTS_SPLIT && !Counts.empty()) { - MCSymbol *CurrentLabel = Labels[0]; - uint64_t CurrentLabelCount = 0; - for (unsigned Index = 0; Index < Entries.size(); ++Index) { - auto LI = Labels.find(Index * EntrySize); - if (LI != Labels.end()) { - LabelCounts[CurrentLabel] = CurrentLabelCount; - CurrentLabel = LI->second; - CurrentLabelCount = 0; - } - CurrentLabelCount += Counts[Index].Count; - } - LabelCounts[CurrentLabel] = CurrentLabelCount; - } else { - Streamer->SwitchSection(Count > 0 ? HotSection : ColdSection); - Streamer->EmitValueToAlignment(EntrySize); - } - MCSymbol *LastLabel = nullptr; - uint64_t Offset = 0; - for (auto *Entry : Entries) { - auto LI = Labels.find(Offset); - if (LI != Labels.end()) { - DEBUG(dbgs() << "BOLT-DEBUG: emitting jump table " - << LI->second->getName() << " (originally was at address 0x" - << Twine::utohexstr(getAddress() + Offset) - << (Offset ? "as part of larger jump table\n" : "\n")); - if (!LabelCounts.empty()) { - DEBUG(dbgs() << "BOLT-DEBUG: jump table count: " - << LabelCounts[LI->second] << '\n'); - if (LabelCounts[LI->second] > 0) { - Streamer->SwitchSection(HotSection); - } else { - Streamer->SwitchSection(ColdSection); - } - Streamer->EmitValueToAlignment(EntrySize); - } - Streamer->EmitLabel(LI->second); - LastLabel = LI->second; - } - if (Type == JTT_NORMAL) { - Streamer->EmitSymbolValue(Entry, OutputEntrySize); - } else { // JTT_PIC - auto JT = MCSymbolRefExpr::create(LastLabel, Streamer->getContext()); - auto E = MCSymbolRefExpr::create(Entry, Streamer->getContext()); - auto Value = MCBinaryExpr::createSub(E, JT, Streamer->getContext()); - Streamer->EmitValue(Value, EntrySize); - } - Offset += EntrySize; - } - - return Offset; -} - void JumpTable::print(raw_ostream &OS) const { uint64_t Offset = 0; if (Type == JTT_PIC) diff --git a/bolt/src/JumpTable.h b/bolt/src/JumpTable.h index 0c2ed535b04f..fdc55f32d32d 100644 --- a/bolt/src/JumpTable.h +++ b/bolt/src/JumpTable.h @@ -122,11 +122,6 @@ public: /// Update jump table at its original location. void updateOriginal(); - /// Emit jump table data. Callee supplies sections for the data. - /// Return the number of total bytes emitted. - uint64_t emit(MCStreamer *Streamer, MCSection *HotSection, - MCSection *ColdSection); - /// Print for debugging purposes. virtual void print(raw_ostream &OS) const override; }; diff --git a/bolt/src/RewriteInstance.cpp b/bolt/src/RewriteInstance.cpp index a231e59b34f0..c83734f9ba7e 100644 --- a/bolt/src/RewriteInstance.cpp +++ b/bolt/src/RewriteInstance.cpp @@ -12,6 +12,7 @@ #include "RewriteInstance.h" #include "BinaryBasicBlock.h" #include "BinaryContext.h" +#include "BinaryEmitter.h" #include "BinaryFunction.h" #include "BinaryPassManager.h" #include "BoltAddressTranslation.h" @@ -30,10 +31,8 @@ #include "Utils.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/BinaryFormat/Dwarf.h" #include "llvm/BinaryFormat/Magic.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugLine.h" #include "llvm/ExecutionEngine/Orc/LambdaResolver.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" @@ -42,7 +41,6 @@ #include "llvm/MC/MCAsmLayout.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDisassembler/MCDisassembler.h" -#include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCInstPrinter.h" #include "llvm/MC/MCInstrAnalysis.h" #include "llvm/MC/MCInstrInfo.h" @@ -50,14 +48,11 @@ #include "llvm/MC/MCObjectStreamer.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCRegisterInfo.h" -#include "llvm/MC/MCSection.h" -#include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Object/Archive.h" #include "llvm/Object/ObjectFile.h" -#include "llvm/Object/SymbolicFile.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/DataExtractor.h" @@ -122,7 +117,7 @@ ForceToDataRelocations("force-data-relocations", cl::ZeroOrMore, cl::cat(BoltCategory)); -static cl::opt +cl::opt PrintCacheMetrics("print-cache-metrics", cl::desc("calculate and print various metrics for instruction cache"), cl::init(false), @@ -146,14 +141,6 @@ BoltProfile("b", cl::desc(""), cl::cat(BoltCategory)); -static cl::list -BreakFunctionNames("break-funcs", - cl::CommaSeparated, - cl::desc("list of functions to core dump on (debugging)"), - cl::value_desc("func1,func2,func3,..."), - cl::Hidden, - cl::cat(BoltCategory)); - cl::opt DumpDotAll("dump-dot-all", cl::desc("dump function CFGs to graphviz format after each stage"), @@ -182,14 +169,6 @@ FunctionNamesFile("funcs-file", cl::Hidden, cl::cat(BoltCategory)); -static cl::list -FunctionPadSpec("pad-funcs", - cl::CommaSeparated, - cl::desc("list of functions to pad with amount of bytes"), - cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."), - cl::Hidden, - cl::cat(BoltCategory)); - cl::opt HotFunctionsAtEnd( "hot-functions-at-end", @@ -198,7 +177,7 @@ HotFunctionsAtEnd( cl::ZeroOrMore, cl::cat(BoltCategory)); -static cl::opt +cl::opt HotText("hot-text", cl::desc("hot text symbols support (relocation mode)"), cl::ZeroOrMore, @@ -220,7 +199,7 @@ HotData("hot-data", cl::ZeroOrMore, cl::cat(BoltCategory)); -static cl::opt +cl::opt UpdateEnd("update-end", cl::desc("update the _end symbol to point to the end of all data sections"), cl::init(true), @@ -233,14 +212,6 @@ KeepTmp("keep-tmp", cl::Hidden, cl::cat(BoltCategory)); -static cl::opt -MarkFuncs("mark-funcs", - cl::desc("mark function boundaries with break instruction to make " - "sure we accidentally don't cross them"), - cl::ReallyHidden, - cl::ZeroOrMore, - cl::cat(BoltCategory)); - static cl::opt MaxFunctions("max-funcs", cl::desc("maximum number of functions to overwrite"), @@ -420,12 +391,6 @@ WriteBoltInfoSection("bolt-info", cl::Hidden, cl::cat(BoltOutputCategory)); -static cl::opt -X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only", - cl::desc("only apply branch boundary alignment in hot code"), - cl::init(true), - cl::cat(BoltOptCategory)); - bool isHotTextMover(const BinaryFunction &Function) { for (auto &SectionName : opts::HotTextMoveSections) { if (Function.getOriginSectionName() == SectionName) @@ -485,31 +450,6 @@ bool shouldProcess(const BinaryFunction &Function) { return true; } -size_t padFunction(const BinaryFunction &Function) { - static std::map FunctionPadding; - - if (FunctionPadding.empty() && !FunctionPadSpec.empty()) { - for (auto &Spec : FunctionPadSpec) { - auto N = Spec.find(':'); - if (N == std::string::npos) - continue; - auto Name = Spec.substr(0, N); - auto Padding = std::stoull(Spec.substr(N+1)); - FunctionPadding[Name] = Padding; - } - } - - for (auto &FPI : FunctionPadding) { - auto Name = FPI.first; - auto Padding = FPI.second; - if (Function.hasNameRegex(Name)) { - return Padding; - } - } - - return 0; -} - } // namespace opts constexpr const char *RewriteInstance::SectionsToOverwrite[]; @@ -2591,117 +2531,6 @@ void RewriteInstance::runOptimizationPasses() { BinaryFunctionPassManager::runAllPasses(*BC); } -// Helper function to emit the contents of a function via a MCStreamer object. -bool RewriteInstance::emitFunction(MCStreamer &Streamer, - BinaryFunction &Function, - bool EmitColdPart) { - if (Function.size() == 0) - return false; - - if (Function.getState() == BinaryFunction::State::Empty) - return false; - - auto *Section = - BC->getCodeSection(EmitColdPart ? Function.getColdCodeSectionName() - : Function.getCodeSectionName()); - Streamer.SwitchSection(Section); - Section->setHasInstructions(true); - BC->Ctx->addGenDwarfSection(Section); - - if (BC->HasRelocations) { - Streamer.EmitCodeAlignment(BinaryFunction::MinAlign); - auto MaxAlignBytes = EmitColdPart - ? Function.getMaxColdAlignmentBytes() - : Function.getMaxAlignmentBytes(); - if (MaxAlignBytes > 0) - Streamer.EmitCodeAlignment(Function.getAlignment(), MaxAlignBytes); - } else { - Streamer.EmitCodeAlignment(Function.getAlignment()); - } - - MCContext &Context = Streamer.getContext(); - const MCAsmInfo *MAI = Context.getAsmInfo(); - - // Emit all symbols associated with the main function entry. - if (!EmitColdPart) { - for (auto *Symbol : Function.getSymbols()) { - Streamer.EmitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction); - Streamer.EmitLabel(Symbol); - } - } else { - auto *Symbol = Function.getColdSymbol(); - Streamer.EmitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction); - Streamer.EmitLabel(Symbol); - } - - // Emit CFI start - if (Function.hasCFI()) { - Streamer.EmitCFIStartProc(/*IsSimple=*/false); - if (Function.getPersonalityFunction() != nullptr) { - Streamer.EmitCFIPersonality(Function.getPersonalityFunction(), - Function.getPersonalityEncoding()); - } - auto *LSDASymbol = EmitColdPart ? Function.getColdLSDASymbol() - : Function.getLSDASymbol(); - if (LSDASymbol) { - Streamer.EmitCFILsda(LSDASymbol, BC->MOFI->getLSDAEncoding()); - } else { - Streamer.EmitCFILsda(0, dwarf::DW_EH_PE_omit); - } - // Emit CFI instructions relative to the CIE - for (const auto &CFIInstr : Function.cie()) { - // Only write CIE CFI insns that LLVM will not already emit - const std::vector &FrameInstrs = - MAI->getInitialFrameState(); - if (std::find(FrameInstrs.begin(), FrameInstrs.end(), CFIInstr) == - FrameInstrs.end()) - Streamer.EmitCFIInstruction(CFIInstr); - } - } - - assert((Function.empty() || !(*Function.begin()).isCold()) && - "first basic block should never be cold"); - - // Emit UD2 at the beginning if requested by user. - if (!opts::BreakFunctionNames.empty()) { - for (auto &Name : opts::BreakFunctionNames) { - if (Function.hasNameRegex(Name)) { - Streamer.EmitIntValue(0x0B0F, 2); // UD2: 0F 0B - break; - } - } - } - - // Emit code. - Function.emitBody(Streamer, EmitColdPart, /*EmitCodeOnly=*/false); - - // Emit padding if requested. - if (auto Padding = opts::padFunction(Function)) { - DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with " - << Padding << " bytes\n"); - Streamer.emitFill(Padding, MAI->getTextAlignFillValue()); - } - - if (opts::MarkFuncs) { - Streamer.EmitIntValue(MAI->getTrapFillValue(), 1); - } - - // Emit CFI end - if (Function.hasCFI()) - Streamer.EmitCFIEndProc(); - - Streamer.EmitLabel(EmitColdPart ? Function.getFunctionColdEndLabel() - : Function.getFunctionEndLabel()); - - // Exception handling info for the function. - Function.emitLSDA(&Streamer, EmitColdPart); - - if (!EmitColdPart && opts::JumpTables > JTS_NONE) - Function.emitJumpTables(&Streamer); - - return true; -} - namespace { template @@ -2742,30 +2571,18 @@ void RewriteInstance::emitAndLink() { /* IncrementalLinkerCompatible */ false, /* DWARFMustBeAtTheEnd */ false)); - Streamer->InitSections(false); - - BC->getTextSection()->setAlignment(BC->PageAlign); - - emitFunctions(Streamer.get()); - if (opts::Instrument) { - Instrumenter->emit(*BC, *Streamer.get()); - } - - if (!BC->HasRelocations && opts::UpdateDebugSections) - DebugInfoRewriter->updateDebugLineInfoForNonSimpleFunctions(); - // Make .eh_frame relocatable. if (EHFrameSection) { relocateEHFrameSection(); } - emitDataSections(Streamer.get()); - - // Update _end if needed. - if (opts::UpdateEnd) { - Streamer->EmitLabel(BC->Ctx->getOrCreateSymbol("_end")); + // Emit contents outside of BinaryContext. + if (opts::Instrument) { + Instrumenter->emit(*BC, *Streamer.get()); } + emitBinaryContext(*Streamer, *BC, OrgSecPrefix); + Streamer->Finish(); ////////////////////////////////////////////////////////////////////////////// @@ -2981,59 +2798,6 @@ void RewriteInstance::updateSDTMarkers() { } } -void RewriteInstance::emitFunctions(MCStreamer *Streamer) { - auto emit = [&](const std::vector &Functions) { - const auto HasProfile = BC->NumProfiledFuncs > 0; - const uint32_t OriginalBranchBoundaryAlign = X86AlignBranchBoundary; - for (auto *Function : Functions) { - if (!BC->HasRelocations && - (!Function->isSimple() || !opts::shouldProcess(*Function))) - continue; - - DEBUG(dbgs() << "BOLT: generating code for function \"" - << *Function << "\" : " - << Function->getFunctionNumber() << '\n'); - - bool Emitted{false}; - // Turn off Intel JCC Erratum mitigation for cold code if requested - if (HasProfile && opts::X86AlignBranchBoundaryHotOnly && - !Function->hasValidProfile()) - X86AlignBranchBoundary = 0; - - Emitted |= emitFunction(*Streamer, *Function, /*EmitColdPart=*/false); - - if (Function->isSplit()) { - if (opts::X86AlignBranchBoundaryHotOnly) - X86AlignBranchBoundary = 0; - Emitted |= emitFunction(*Streamer, *Function, /*EmitColdPart=*/true); - } - X86AlignBranchBoundary = OriginalBranchBoundaryAlign; - - if (Emitted) - Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics); - } - }; - - // Mark the start of hot text. - if (opts::HotText) { - Streamer->SwitchSection(BC->getTextSection()); - Streamer->EmitLabel(BC->getHotTextStartSymbol()); - } - - // Emit functions in sorted order. - std::vector SortedFunctions = BC->getSortedFunctions(); - emit(SortedFunctions); - - // Emit functions added by BOLT. - emit(BC->getInjectedBinaryFunctions()); - - // Mark the end of hot text. - if (opts::HotText) { - Streamer->SwitchSection(BC->getTextSection()); - Streamer->EmitLabel(BC->getHotTextEndSymbol()); - } -} - void RewriteInstance::mapFileSections(orc::VModuleKey Key) { mapCodeSections(Key); mapDataSections(Key); @@ -3358,19 +3122,6 @@ void RewriteInstance::updateOutputValues(const MCAsmLayout &Layout) { } } -void RewriteInstance::emitDataSections(MCStreamer *Streamer) { - for (const auto &Section : BC->sections()) { - if (!Section.hasRelocations() || !Section.hasSectionRef()) - continue; - - StringRef SectionName = Section.getName(); - std::string EmitName = Section.isReordered() - ? std::string(Section.getOutputName()) - : OrgSecPrefix + std::string(SectionName); - Section.emitAsData(*Streamer, EmitName); - } -} - void RewriteInstance::patchELFPHDRTable() { auto ELF64LEFile = dyn_cast(InputFile); if (!ELF64LEFile) { @@ -3720,7 +3471,8 @@ std::vector RewriteInstance::getOutputSections( if (!Section.isFinalized()) continue; - if (Section.getName().startswith(OrgSecPrefix) || Section.isAnonymous()) { + if (Section.getName().startswith(OrgSecPrefix) || + Section.isAnonymous()) { if (opts::Verbosity) outs() << "BOLT-INFO: not writing section header for section " << Section.getName() << '\n'; diff --git a/bolt/src/RewriteInstance.h b/bolt/src/RewriteInstance.h index 4b69dfb53d2c..65b4045b0488 100644 --- a/bolt/src/RewriteInstance.h +++ b/bolt/src/RewriteInstance.h @@ -135,12 +135,6 @@ private: /// Link additional runtime code to support instrumentation. void linkRuntime(); - /// Emit function code. - void emitFunctions(MCStreamer *Streamer); - - /// Emit data sections that have code references in them. - void emitDataSections(MCStreamer *Streamer); - /// Update debug and other auxiliary information in the file. void updateMetadata(); @@ -176,10 +170,6 @@ private: "findSymbol failed"); } - /// Emit a single function. - bool emitFunction(MCStreamer &Streamer, BinaryFunction &Function, - bool EmitColdPart); - /// Detect addresses and offsets available in the binary for allocating /// new sections. void discoverStorage(); @@ -269,9 +259,9 @@ private: /// Add a notes section containing the BOLT revision and command line options. void addBoltInfoSection(); - /// Add a notes section containing the serialized BOLT Address Translation maps - /// that can be used to enable sampling of the output binary for the purposes - /// of generating BOLT profile data for the input binary. + /// Add a notes section containing the serialized BOLT Address Translation + /// maps that can be used to enable sampling of the output binary for the + /// purposes of generating BOLT profile data for the input binary. void addBATSection(); /// Loop over now emitted functions to write translation maps @@ -388,8 +378,6 @@ private: /// Extra linking uint64_t InstrumentationRuntimeFiniAddress{0}; uint64_t InstrumentationRuntimeStartAddress{0}; - const BinaryFunction *StartFunction{nullptr}; - const BinaryFunction *FiniFunction{nullptr}; /// Track next available address for new allocatable sections. uint64_t NextAvailableAddress{0};