From 0e4d86bf19fba5b62d6435603b79a71ee9c33ff2 Mon Sep 17 00:00:00 2001 From: Bill Nell Date: Tue, 14 Nov 2017 20:05:11 -0800 Subject: [PATCH] [BOLT] Refactor global symbol handling code. Summary: This is preparation work for static data reordering. I've created a new class called BinaryData which represents a symbol contained in a section. It records almost all the information relevant for dealing with data, e.g. names, address, size, alignment, profiling data, etc. BinaryContext still stores and manages BinaryData objects similar to how it managed symbols and global addresses before. The interfaces are not changed too drastically from before either. There is a bit of overlap between BinaryData and BinaryFunction. I would have liked to do some more refactoring to make a BinaryFunctionFragment that subclassed from BinaryData and then have BinaryFunction be composed or associated with BinaryFunctionFragments. I've also attempted to use (symbol + offset) for when addresses are pointing into the middle of symbols with known sizes. This changes the simplify rodata loads optimization slightly since the expression on an instruction can now also be a (symbol + offset) rather than just a symbol. One of the overall goals for this refactoring is to make sure every relocation is associated with a BinaryData object. This requires adding "hole" BinaryData's wherever there are gaps in a section's address space. Most of the holes seem to be data that has no associated symbol info. In this case we can't do any better than lumping all the adjacent hole symbols into one big symbol (there may be more than one actual data object that contributes to a hole). At least the combined holes should be moveable. Jump tables have similar issues. They appear to mostly be sub-objects for top level local symbols. The main problem is that we can't recognize jump tables at the time we scan the symbol table, we have to wait til disassembly. When a jump table is discovered we add it as a sub-object to the existing local symbol. If there are one or more existing BinaryData's that appear in the address range of a newly created jump table, those are added as sub-objects as well. (cherry picked from FBD6362544) --- bolt/BinaryBasicBlock.cpp | 15 +- bolt/BinaryContext.cpp | 392 +++++++++++++++++++++--- bolt/BinaryContext.h | 316 ++++++++++++++++--- bolt/BinaryData.cpp | 132 ++++++++ bolt/BinaryData.h | 207 +++++++++++++ bolt/BinaryFunction.cpp | 374 +++++++++------------- bolt/BinaryFunction.h | 136 ++------ bolt/BinaryFunctionProfile.cpp | 12 +- bolt/BinarySection.cpp | 317 +------------------ bolt/BinarySection.h | 102 ++---- bolt/CMakeLists.txt | 3 + bolt/DWARFRewriter.cpp | 2 +- bolt/DataAggregator.cpp | 10 +- bolt/Exceptions.cpp | 14 +- bolt/JumpTable.cpp | 191 ++++++++++++ bolt/JumpTable.h | 123 ++++++++ bolt/Passes/BinaryFunctionCallGraph.cpp | 10 +- bolt/Passes/BinaryPasses.cpp | 16 +- bolt/Passes/IndirectCallPromotion.cpp | 49 +-- bolt/Passes/IndirectCallPromotion.h | 2 +- bolt/Passes/JTFootprintReduction.cpp | 8 +- bolt/Passes/JTFootprintReduction.h | 6 +- bolt/Passes/LongJmp.cpp | 6 +- bolt/Passes/ReorderFunctions.cpp | 19 +- bolt/ProfileWriter.cpp | 7 +- bolt/Relocation.cpp | 326 ++++++++++++++++++++ bolt/Relocation.h | 90 ++++++ bolt/RewriteInstance.cpp | 386 ++++++++++++++++------- bolt/RewriteInstance.h | 12 +- 29 files changed, 2275 insertions(+), 1008 deletions(-) create mode 100644 bolt/BinaryData.cpp create mode 100644 bolt/BinaryData.h create mode 100644 bolt/JumpTable.cpp create mode 100644 bolt/JumpTable.h create mode 100644 bolt/Relocation.cpp create mode 100644 bolt/Relocation.h diff --git a/bolt/BinaryBasicBlock.cpp b/bolt/BinaryBasicBlock.cpp index 8bb3919b18e1..f64a1b4ee338 100644 --- a/bolt/BinaryBasicBlock.cpp +++ b/bolt/BinaryBasicBlock.cpp @@ -78,13 +78,22 @@ bool BinaryBasicBlock::validateSuccessorInvariants() { // Work on the assumption that jump table blocks don't // have a conditional successor. Valid = false; + errs() << "BOLT-WARNING: Jump table successor " + << Succ->getName() + << " not contained in the jump table.\n"; } } // If there are any leftover entries in the jump table, they // must be one of the function end labels. - for (auto *Sym : UniqueSyms) { - Valid &= (Sym == Function->getFunctionEndLabel() || - Sym == Function->getFunctionColdEndLabel()); + if (Valid) { + for (auto *Sym : UniqueSyms) { + Valid &= (Sym == Function->getFunctionEndLabel() || + Sym == Function->getFunctionColdEndLabel()); + if (!Valid) { + errs() << "BOLT-WARNING: Jump table contains illegal entry: " + << Sym->getName() << "\n"; + } + } } } else { const MCSymbol *TBB = nullptr; diff --git a/bolt/BinaryContext.cpp b/bolt/BinaryContext.cpp index e6000c01dce9..09ebe6b7d58c 100644 --- a/bolt/BinaryContext.cpp +++ b/bolt/BinaryContext.cpp @@ -11,6 +11,7 @@ #include "BinaryContext.h" #include "BinaryFunction.h" +#include "DataReader.h" #include "llvm/ADT/Twine.h" #include "llvm/DebugInfo/DWARF/DWARFFormValue.h" #include "llvm/DebugInfo/DWARF/DWARFUnit.h" @@ -19,6 +20,7 @@ #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Support/CommandLine.h" +#include using namespace llvm; using namespace bolt; @@ -57,6 +59,7 @@ BinaryContext::~BinaryContext() { for (auto *Section : Sections) { delete Section; } + clearBinaryData(); } std::unique_ptr @@ -69,47 +72,224 @@ BinaryContext::createObjectWriter(raw_pwrite_stream &OS) { return MAB->createObjectWriter(OS); } -MCSymbol *BinaryContext::getOrCreateGlobalSymbol(uint64_t Address, - Twine Prefix) { - MCSymbol *Symbol{nullptr}; - std::string Name; - auto NI = GlobalAddresses.find(Address); - if (NI != GlobalAddresses.end()) { - // Even though there could be multiple names registered at the address, - // we only use the first one. - Name = NI->second; - } else { - Name = (Prefix + "0x" + Twine::utohexstr(Address)).str(); - assert(GlobalSymbols.find(Name) == GlobalSymbols.end() && - "created name is not unique"); - GlobalAddresses.emplace(std::make_pair(Address, Name)); +bool BinaryContext::validateObjectNesting() const { + auto Itr = BinaryDataMap.begin(); + auto End = BinaryDataMap.end(); + bool Valid = true; + while (Itr != End) { + auto Next = std::next(Itr); + while (Next != End && + Itr->second->getSection() == Next->second->getSection() && + Itr->second->containsRange(Next->second->getAddress(), + Next->second->getSize())) { + if (Next->second->Parent != Itr->second) { + errs() << "BOLT-WARNING: object nesting incorrect for:\n" + << "BOLT-WARNING: " << *Itr->second << "\n" + << "BOLT-WARNING: " << *Next->second << "\n"; + Valid = false; + } + ++Next; + } + Itr = Next; + } + return Valid; +} + +bool BinaryContext::validateHoles() const { + bool Valid = true; + for (auto &Section : sections()) { + for (const auto &Rel : Section.relocations()) { + auto RelAddr = Rel.Offset + Section.getAddress(); + auto *BD = getBinaryDataContainingAddress(RelAddr); + if (!BD) { + errs() << "BOLT-WARNING: no BinaryData found for relocation at address" + << " 0x" << Twine::utohexstr(RelAddr) << " in " + << Section.getName() << "\n"; + Valid = false; + } else if (!BD->getAtomicRoot()) { + errs() << "BOLT-WARNING: no atomic BinaryData found for relocation at " + << "address 0x" << Twine::utohexstr(RelAddr) << " in " + << Section.getName() << "\n"; + Valid = false; + } + } + } + return Valid; +} + +void BinaryContext::updateObjectNesting(BinaryDataMapType::iterator GAI) { + const auto Address = GAI->second->getAddress(); + const auto Size = GAI->second->getSize(); + + auto fixParents = + [&](BinaryDataMapType::iterator Itr, BinaryData *NewParent) { + auto *OldParent = Itr->second->Parent; + Itr->second->Parent = NewParent; + ++Itr; + while (Itr != BinaryDataMap.end() && OldParent && + Itr->second->Parent == OldParent) { + Itr->second->Parent = NewParent; + ++Itr; + } + }; + + // Check if the previous symbol contains the newly added symbol. + if (GAI != BinaryDataMap.begin()) { + auto *Prev = std::prev(GAI)->second; + while (Prev) { + if (Prev->getSection() == GAI->second->getSection() && + Prev->containsRange(Address, Size)) { + fixParents(GAI, Prev); + } else { + fixParents(GAI, nullptr); + } + Prev = Prev->Parent; + } } - Symbol = Ctx->lookupSymbol(Name); - if (Symbol) - return Symbol; + // Check if the newly added symbol contains any subsequent symbols. + if (Size != 0) { + auto *BD = GAI->second->Parent ? GAI->second->Parent : GAI->second; + auto Itr = std::next(GAI); + while (Itr != BinaryDataMap.end() && + BD->containsRange(Itr->second->getAddress(), + Itr->second->getSize())) { + Itr->second->Parent = BD; + ++Itr; + } + } +} - Symbol = Ctx->getOrCreateSymbol(Name); - GlobalSymbols[Name] = Address; +MCSymbol *BinaryContext::getOrCreateGlobalSymbol(uint64_t Address, + uint64_t Size, + uint16_t Alignment, + Twine Prefix) { + auto Itr = BinaryDataMap.find(Address); + if (Itr != BinaryDataMap.end()) { + assert(Itr->second->getSize() == Size || !Size); + return Itr->second->getSymbol(); + } + std::string Name = (Prefix + "0x" + Twine::utohexstr(Address)).str(); + assert(!GlobalSymbols.count(Name) && "created name is not unique"); + return registerNameAtAddress(Name, Address, Size, Alignment); +} + +MCSymbol *BinaryContext::registerNameAtAddress(StringRef Name, + uint64_t Address, + uint64_t Size, + uint16_t Alignment) { + auto SectionOrErr = getSectionForAddress(Address); + auto &Section = SectionOrErr ? SectionOrErr.get() : absoluteSection(); + auto GAI = BinaryDataMap.find(Address); + BinaryData *BD; + if (GAI == BinaryDataMap.end()) { + BD = new BinaryData(Name, + Address, + Size, + Alignment ? Alignment : 1, + Section); + } else { + BD = GAI->second; + } + return registerNameAtAddress(Name, Address, BD); +} + +MCSymbol *BinaryContext::registerNameAtAddress(StringRef Name, + uint64_t Address, + BinaryData *BD) { + auto GAI = BinaryDataMap.find(Address); + if (GAI != BinaryDataMap.end()) { + if (BD != GAI->second) { + // Note: this could be a source of bugs if client code holds + // on to BinaryData*'s in data structures for any length of time. + auto *OldBD = GAI->second; + BD->merge(GAI->second); + delete OldBD; + GAI->second = BD; + for (auto &Name : BD->names()) { + GlobalSymbols[Name] = BD; + } + updateObjectNesting(GAI); + } else if (!GAI->second->hasName(Name)) { + GAI->second->Names.push_back(Name); + GlobalSymbols[Name] = GAI->second; + } + BD = nullptr; + } else { + GAI = BinaryDataMap.emplace(Address, BD).first; + GlobalSymbols[Name] = BD; + updateObjectNesting(GAI); + } + + // Register the name with MCContext. + auto *Symbol = Ctx->getOrCreateSymbol(Name); + if (BD) { + BD->Symbols.push_back(Symbol); + assert(BD->Symbols.size() == BD->Names.size()); + } return Symbol; } -MCSymbol *BinaryContext::getGlobalSymbolAtAddress(uint64_t Address) const { - auto NI = GlobalAddresses.find(Address); - if (NI == GlobalAddresses.end()) - return nullptr; +const BinaryData * +BinaryContext::getBinaryDataContainingAddressImpl(uint64_t Address, + bool IncludeEnd, + bool BestFit) const { + auto NI = BinaryDataMap.lower_bound(Address); + auto End = BinaryDataMap.end(); + if ((NI != End && Address == NI->first) || + (NI-- != BinaryDataMap.begin())) { + if (NI->second->containsAddress(Address) || + (IncludeEnd && NI->second->getEndAddress() == Address)) { + while (BestFit && + std::next(NI) != End && + (std::next(NI)->second->containsAddress(Address) || + (IncludeEnd && std::next(NI)->second->getEndAddress() == Address))) { + ++NI; + } + return NI->second; + } - auto *Symbol = Ctx->lookupSymbol(NI->second); - assert(Symbol && "symbol cannot be NULL at this point"); - - return Symbol; + // If this is a sub-symbol, see if a parent data contains the address. + auto *BD = NI->second->getParent(); + while (BD) { + if (BD->containsAddress(Address) || + (IncludeEnd && NI->second->getEndAddress() == Address)) + return BD; + BD = BD->getParent(); + } + } + return nullptr; } -MCSymbol *BinaryContext::getGlobalSymbolByName(const std::string &Name) const { - auto Itr = GlobalSymbols.find(Name); - return Itr == GlobalSymbols.end() - ? nullptr : getGlobalSymbolAtAddress(Itr->second); +bool BinaryContext::setBinaryDataSize(uint64_t Address, uint64_t Size) { + auto NI = BinaryDataMap.find(Address); + assert(NI != BinaryDataMap.end()); + if (NI == BinaryDataMap.end()) + return false; + assert(!NI->second->Size || NI->second->Size == Size); + NI->second->Size = Size; + updateObjectNesting(NI); + return true; +} + +void BinaryContext::postProcessSymbolTable() { + fixBinaryDataHoles(); + bool Valid = true; + for (auto &Entry : BinaryDataMap) { + auto *BD = Entry.second; + if ((BD->getName().startswith("SYMBOLat") || + BD->getName().startswith("DATAat")) && + !BD->getParent() && + !BD->getSize() && + !BD->isAbsolute() && + BD->getSection()) { + outs() << "BOLT-WARNING: zero sized top level symbol: " << *BD << "\n"; + Valid = false; + } + } + assert(Valid); + assignMemData(); } void BinaryContext::foldFunction(BinaryFunction &ChildBF, @@ -126,7 +306,7 @@ void BinaryContext::foldFunction(BinaryFunction &ChildBF, assert(Symbol && "symbol cannot be NULL at this point"); SymbolToFunctionMap[Symbol] = &ParentBF; - // NB: there's no need to update GlobalAddresses and GlobalSymbols. + // NB: there's no need to update BinaryDataMap and GlobalSymbols. } // Merge execution counts of ChildBF into those of ParentBF. @@ -148,10 +328,138 @@ void BinaryContext::foldFunction(BinaryFunction &ChildBF, } } +void BinaryContext::fixBinaryDataHoles() { + assert(validateObjectNesting() && "object nesting inconsitency detected"); + + for (auto &Section : allocatableSections()) { + std::vector> Holes; + + auto isNotHole = [&Section](const binary_data_iterator &Itr) { + auto *BD = Itr->second; + bool isHole = (!BD->getParent() && + !BD->getSize() && + BD->isObject() && + (BD->getName().startswith("SYMBOLat0x") || + BD->getName().startswith("DATAat0x") || + BD->getName().startswith("ANONYMOUS"))); + return !isHole && BD->getSection() == Section && !BD->getParent(); + }; + + auto BDStart = BinaryDataMap.begin(); + auto BDEnd = BinaryDataMap.end(); + auto Itr = FilteredBinaryDataIterator(isNotHole, BDStart, BDEnd); + auto End = FilteredBinaryDataIterator(isNotHole, BDEnd, BDEnd); + + uint64_t EndAddress = Section.getAddress(); + + while (Itr != End) { + auto Gap = Itr->second->getAddress() - EndAddress; + if (Gap > 0) { + assert(EndAddress < Itr->second->getAddress()); + Holes.push_back(std::make_pair(EndAddress, Gap)); + } + EndAddress = Itr->second->getEndAddress(); + ++Itr; + } + + if (EndAddress < Section.getEndAddress()) { + Holes.push_back(std::make_pair(EndAddress, + Section.getEndAddress() - EndAddress)); + } + + // If there is already a symbol at the start of the hole, grow that symbol + // to cover the rest. Otherwise, create a new symbol to cover the hole. + for (auto &Hole : Holes) { + auto *BD = getBinaryDataAtAddress(Hole.first); + if (BD) { + // BD->getSection() can be != Section if there are sections that + // overlap. In this case it is probably safe to just skip the holes + // since the overlapping section will not(?) have any symbols in it. + if (BD->getSection() == Section) + setBinaryDataSize(Hole.first, Hole.second); + } else { + getOrCreateGlobalSymbol(Hole.first, Hole.second, 1, "HOLEat"); + } + } + } + + assert(validateObjectNesting() && "object nesting inconsitency detected"); + assert(validateHoles() && "top level hole detected in object map"); +} + void BinaryContext::printGlobalSymbols(raw_ostream& OS) const { - for (auto &Entry : GlobalSymbols) { - OS << "(" << Entry.first << " -> 0x" - << Twine::utohexstr(Entry.second) << ")\n"; + const BinarySection* CurrentSection = nullptr; + bool FirstSection = true; + + for (auto &Entry : BinaryDataMap) { + const auto *BD = Entry.second; + const auto &Section = BD->getSection(); + if (FirstSection || Section != *CurrentSection) { + uint64_t Address, Size; + StringRef Name = Section.getName(); + if (Section) { + Address = Section.getAddress(); + Size = Section.getSize(); + } else { + Address = BD->getAddress(); + Size = BD->getSize(); + } + OS << "BOLT-INFO: Section " << Name << ", " + << "0x" + Twine::utohexstr(Address) << ":" + << "0x" + Twine::utohexstr(Address + Size) << "/" + << Size << "\n"; + CurrentSection = &Section; + FirstSection = false; + } + + OS << "BOLT-INFO: "; + auto *P = BD->getParent(); + while (P) { + OS << " "; + P = P->getParent(); + } + OS << *BD << "\n"; + } +} + +void BinaryContext::assignMemData() { + auto getAddress = [&](const MemInfo &MI) { + if (!MI.Addr.IsSymbol) + return MI.Addr.Offset; + + if (auto *BD = getBinaryDataByName(MI.Addr.Name)) + return BD->getAddress() + MI.Addr.Offset; + + return 0ul; + }; + + // Map of sections (or heap/stack) to count/size. + std::map Counts; + + uint64_t TotalCount = 0; + for (auto &Entry : DR.getAllFuncsMemData()) { + for (auto &MI : Entry.second.Data) { + const auto Addr = getAddress(MI); + auto *BD = getBinaryDataContainingAddress(Addr); + if (BD) { + BD->getAtomicRoot()->addMemData(MI); + Counts[BD->getSectionName()] += MI.Count; + } else { + Counts["Heap/stack"] += MI.Count; + } + TotalCount += MI.Count; + } + } + + if (!Counts.empty()) { + outs() << "BOLT-INFO: Memory stats breakdown:\n"; + for (auto &Entry : Counts) { + const auto Section = Entry.first; + const auto Count = Entry.second; + outs() << "BOLT-INFO: " << Section << " = " << Count + << format(" (%.1f%%)\n", 100.0*Count/TotalCount); + } + outs() << "BOLT-INFO: Total memory events: " << TotalCount << "\n"; } } @@ -484,6 +792,14 @@ BinaryContext::getSectionForAddress(uint64_t Address) const { return std::make_error_code(std::errc::bad_address); } +ErrorOr +BinaryContext::getSectionNameForAddress(uint64_t Address) const { + if (auto Section = getSectionForAddress(Address)) { + return Section->getName(); + } + return std::make_error_code(std::errc::bad_address); +} + BinarySection &BinaryContext::registerSection(BinarySection *Section) { assert(!Section->getName().empty() && "can't register sections without a name"); @@ -562,6 +878,12 @@ void BinaryContext::printSections(raw_ostream &OS) const { } } +BinarySection &BinaryContext::absoluteSection() { + if (auto Section = getUniqueSectionByName("")) + return *Section; + return registerOrUpdateSection("", ELF::SHT_NULL, 0u); +} + ErrorOr BinaryContext::extractPointerAtAddress(uint64_t Address) const { auto Section = getSectionForAddress(Address); diff --git a/bolt/BinaryContext.h b/bolt/BinaryContext.h index bea5ec4a4f21..5cb67ad4fe16 100644 --- a/bolt/BinaryContext.h +++ b/bolt/BinaryContext.h @@ -14,6 +14,7 @@ #ifndef LLVM_TOOLS_LLVM_BOLT_BINARY_CONTEXT_H #define LLVM_TOOLS_LLVM_BOLT_BINARY_CONTEXT_H +#include "BinaryData.h" #include "BinarySection.h" #include "DebugData.h" #include "llvm/ADT/iterator.h" @@ -55,6 +56,50 @@ namespace bolt { class BinaryFunction; class DataReader; +/// Filter iterator. +template > +class FilterIterator + : public std::iterator::value_type> { + using Iterator = FilterIterator; + using T = typename std::iterator_traits::reference; + using PointerT = typename std::iterator_traits::pointer; + + PredType Pred; + ItrType Itr, End; + + void prev() { + while (!Pred(--Itr)) + ; + } + void next() { + ++Itr; + nextMatching(); + } + void nextMatching() { + while (Itr != End && !Pred(Itr)) + ++Itr; + } +public: + Iterator &operator++() { next(); return *this; } + Iterator &operator--() { prev(); return *this; } + Iterator operator++(int) { auto Tmp(Itr); next(); return Tmp; } + Iterator operator--(int) { auto Tmp(Itr); prev(); return Tmp; } + bool operator==(const Iterator& Other) const { + return Itr == Other.Itr; + } + bool operator!=(const Iterator& Other) const { + return !operator==(Other); + } + T operator*() { return *Itr; } + PointerT operator->() { return &operator*(); } + FilterIterator(PredType Pred, ItrType Itr, ItrType End) + : Pred(Pred), Itr(Itr), End(End) { + nextMatching(); + } +}; + class BinaryContext { BinaryContext() = delete; @@ -70,6 +115,9 @@ class BinaryContext { using SectionIterator = pointee_iterator; using SectionConstIterator = pointee_iterator; + using FilteredSectionIterator = FilterIterator; + using FilteredSectionConstIterator = FilterIterator; + /// Map virtual address to a section. It is possible to have more than one /// section mapped to the same address, e.g. non-allocatable sections. using AddressToSectionMapType = std::multimap; @@ -84,13 +132,24 @@ class BinaryContext { BinarySection ®isterSection(BinarySection *Section); public: - /// [name] -> [address] map used for global symbol resolution. - typedef std::map SymbolMapType; + /// [name] -> [BinaryData*] map used for global symbol resolution. + using SymbolMapType = std::map; SymbolMapType GlobalSymbols; - /// [address] -> [name1], [name2], ... - /// Global addresses never change. - std::multimap GlobalAddresses; + /// [address] -> [BinaryData], ... + /// Addresses never change. + /// Note: it is important that clients do not hold on to instances of + /// BinaryData* while the map is still being modified during BinaryFunction + /// disassembly. This is because of the possibility that a regular + /// BinaryData is later discovered to be a JumpTable. + using BinaryDataMapType = std::map; + using binary_data_iterator = BinaryDataMapType::iterator; + using binary_data_const_iterator = BinaryDataMapType::const_iterator; + BinaryDataMapType BinaryDataMap; + + using FilteredBinaryDataConstIterator = + FilterIterator; + using FilteredBinaryDataIterator = FilterIterator; /// [MCSymbol] -> [BinaryFunction] /// @@ -99,6 +158,38 @@ public: std::unordered_map SymbolToFunctionMap; + /// Look up the symbol entry that contains the given \p Address (based on + /// the start address and size for each symbol). Returns a pointer to + /// the BinaryData for that symbol. If no data is found, nullptr is returned. + const BinaryData *getBinaryDataContainingAddressImpl(uint64_t Address, + bool IncludeEnd, + bool BestFit) const; + + /// Update the Parent fields in BinaryDatas after adding a new entry into + /// \p BinaryDataMap. + void updateObjectNesting(BinaryDataMapType::iterator GAI); + + /// Validate that if object address ranges overlap that the object with + /// the larger range is a parent of the object with the smaller range. + bool validateObjectNesting() const; + + /// Validate that there are no top level "holes" in each section + /// and that all relocations with a section are mapped to a valid + /// top level BinaryData. + bool validateHoles() const; + + /// Get a bogus "absolute" section that will be associated with all + /// absolute BinaryDatas. + BinarySection &absoluteSection(); + + /// Process "holes" in between known BinaryData objects. For now, + /// symbols are padded with the space before the next BinaryData object. + void fixBinaryDataHoles(); + + /// Populate \p GlobalMemData. This should be done after all symbol discovery + /// is complete, e.g. after building CFGs for all functions. + void assignMemData(); +public: /// Map address to a constant island owner (constant data in code section) std::map AddressToConstantIslandMap; @@ -204,28 +295,122 @@ public: std::unique_ptr createObjectWriter(raw_pwrite_stream &OS); - /// Return a global symbol registered at a given \p Address. If no symbol - /// exists, create one with unique name using \p Prefix. - /// If there are multiple symbols registered at the \p Address, then - /// return the first one. - MCSymbol *getOrCreateGlobalSymbol(uint64_t Address, Twine Prefix); - - /// Return MCSymbol registered at a given \p Address or nullptr if no - /// global symbol was registered at the location. - MCSymbol *getGlobalSymbolAtAddress(uint64_t Address) const; - - /// Find the address of the global symbol with the given \p Name. - /// return an error if no such symbol exists. - ErrorOr getAddressForGlobalSymbol(StringRef Name) const { - auto Itr = GlobalSymbols.find(Name); - if (Itr != GlobalSymbols.end()) - return Itr->second; - return std::make_error_code(std::errc::bad_address); + /// Iterate over all BinaryData. + iterator_range getBinaryData() const { + return make_range(BinaryDataMap.begin(), BinaryDataMap.end()); } - /// Return MCSymbol for the given \p Name or nullptr if no + /// Iterate over all BinaryData. + iterator_range getBinaryData() { + return make_range(BinaryDataMap.begin(), BinaryDataMap.end()); + } + + /// Iterate over all BinaryData associated with the given \p Section. + iterator_range + getBinaryDataForSection(StringRef SectionName) const { + auto Begin = BinaryDataMap.begin(); + auto End = BinaryDataMap.end(); + auto pred = + [&SectionName](const binary_data_const_iterator &Itr) -> bool { + return Itr->second->getSection().getName() == SectionName; + }; + return make_range(FilteredBinaryDataConstIterator(pred, Begin, End), + FilteredBinaryDataConstIterator(pred, End, End)); + } + + /// Iterate over all BinaryData associated with the given \p Section. + iterator_range + getBinaryDataForSection(StringRef SectionName) { + auto Begin = BinaryDataMap.begin(); + auto End = BinaryDataMap.end(); + auto pred = [&SectionName](const binary_data_iterator &Itr) -> bool { + return Itr->second->getSection().getName() == SectionName; + }; + return make_range(FilteredBinaryDataIterator(pred, Begin, End), + FilteredBinaryDataIterator(pred, End, End)); + } + + /// Clear the global symbol address -> name(s) map. + void clearBinaryData() { + GlobalSymbols.clear(); + for (auto &Entry : BinaryDataMap) { + delete Entry.second; + } + BinaryDataMap.clear(); + } + + + /// Return a global symbol registered at a given \p Address and \p Size. + /// If no symbol exists, create one with unique name using \p Prefix. + /// If there are multiple symbols registered at the \p Address, then + /// return the first one. + MCSymbol *getOrCreateGlobalSymbol(uint64_t Address, + uint64_t Size, + uint16_t Alignment, + Twine Prefix); + + /// Register a symbol with \p Name at a given \p Address and \p Size. + MCSymbol *registerNameAtAddress(StringRef Name, + uint64_t Address, + BinaryData* BD); + + /// Register a symbol with \p Name at a given \p Address and \p Size. + MCSymbol *registerNameAtAddress(StringRef Name, + uint64_t Address, + uint64_t Size, + uint16_t Alignment); + + /// Return BinaryData registered at a given \p Address or nullptr if no + /// global symbol was registered at the location. + const BinaryData *getBinaryDataAtAddress(uint64_t Address) const { + auto NI = BinaryDataMap.find(Address); + return NI != BinaryDataMap.end() ? NI->second : nullptr; + } + + BinaryData *getBinaryDataAtAddress(uint64_t Address) { + auto NI = BinaryDataMap.find(Address); + return NI != BinaryDataMap.end() ? NI->second : nullptr; + } + + /// Look up the symbol entry that contains the given \p Address (based on + /// the start address and size for each symbol). Returns a pointer to + /// the BinaryData for that symbol. If no data is found, nullptr is returned. + const BinaryData *getBinaryDataContainingAddress(uint64_t Address, + bool IncludeEnd = false, + bool BestFit = false) const { + return getBinaryDataContainingAddressImpl(Address, IncludeEnd, BestFit); + } + + BinaryData *getBinaryDataContainingAddress(uint64_t Address, + bool IncludeEnd = false, + bool BestFit = false) { + return const_cast(getBinaryDataContainingAddressImpl(Address, + IncludeEnd, + BestFit)); + } + + /// Return BinaryData for the given \p Name or nullptr if no /// global symbol with that name exists. - MCSymbol *getGlobalSymbolByName(const std::string &Name) const; + const BinaryData *getBinaryDataByName(StringRef Name) const { + auto Itr = GlobalSymbols.find(Name); + return Itr != GlobalSymbols.end() ? Itr->second : nullptr; + } + + BinaryData *getBinaryDataByName(StringRef Name) { + auto Itr = GlobalSymbols.find(Name); + return Itr != GlobalSymbols.end() ? Itr->second : nullptr; + } + + /// Perform any necessary post processing on the symbol table after + /// function disassembly is complete. This processing fixes top + /// level data holes and makes sure the symbol table is valid. + /// It also assigns all memory profiling info to the appropriate + /// BinaryData objects. + void postProcessSymbolTable(); + + /// Set the size of the global symbol located at \p Address. Return + /// false if no symbol exists, true otherwise. + bool setBinaryDataSize(uint64_t Address, uint64_t Size); /// Print the global symbol table. void printGlobalSymbols(raw_ostream& OS) const; @@ -269,15 +454,62 @@ public: bool deregisterSection(BinarySection &Section); /// Iterate over all registered sections. - iterator_range sections() { - return make_range(Sections.begin(), Sections.end()); + iterator_range sections() { + auto notNull = [](const SectionIterator &Itr) { + return (bool)*Itr; + }; + return make_range(FilteredSectionIterator(notNull, + Sections.begin(), + Sections.end()), + FilteredSectionIterator(notNull, + Sections.end(), + Sections.end())); } /// Iterate over all registered sections. - iterator_range sections() const { - return make_range(Sections.begin(), Sections.end()); + iterator_range sections() const { + return const_cast(this)->sections(); } + /// Iterate over all registered allocatable sections. + iterator_range allocatableSections() { + auto isAllocatable = [](const SectionIterator &Itr) { + return *Itr && Itr->isAllocatable(); + }; + return make_range(FilteredSectionIterator(isAllocatable, + Sections.begin(), + Sections.end()), + FilteredSectionIterator(isAllocatable, + Sections.end(), + Sections.end())); + } + + /// Iterate over all registered allocatable sections. + iterator_range allocatableSections() const { + return const_cast(this)->allocatableSections(); + } + + /// Iterate over all registered non-allocatable sections. + iterator_range nonAllocatableSections() { + auto notAllocated = [](const SectionIterator &Itr) { + return *Itr && !Itr->isAllocatable(); + }; + return make_range(FilteredSectionIterator(notAllocated, + Sections.begin(), + Sections.end()), + FilteredSectionIterator(notAllocated, + Sections.end(), + Sections.end())); + } + + /// Iterate over all registered non-allocatable sections. + iterator_range nonAllocatableSections() const { + return const_cast(this)->nonAllocatableSections(); + } + + /// Return section name containing the given \p Address. + ErrorOr getSectionNameForAddress(uint64_t Address) const; + /// Print all sections. void printSections(raw_ostream& OS) const; @@ -321,28 +553,6 @@ public: /// the binary. ErrorOr extractPointerAtAddress(uint64_t Address) const; - /// Register a symbol with \p Name at a given \p Address. - MCSymbol *registerNameAtAddress(const std::string &Name, uint64_t Address) { - // Check if the Name was already registered. - const auto GSI = GlobalSymbols.find(Name); - if (GSI != GlobalSymbols.end()) { - assert(GSI->second == Address && "addresses do not match"); - auto *Symbol = Ctx->lookupSymbol(Name); - assert(Symbol && "symbol should be registered with MCContext"); - - return Symbol; - } - - // Add the name to global symbols map. - GlobalSymbols[Name] = Address; - - // Add to the reverse map. There could multiple names at the same address. - GlobalAddresses.emplace(std::make_pair(Address, Name)); - - // Register the name with MCContext. - return Ctx->getOrCreateSymbol(Name); - } - /// Replaces all references to \p ChildBF with \p ParentBF. \p ChildBF is then /// removed from the list of functions \p BFs. The profile data of \p ChildBF /// is merged into that of \p ParentBF. @@ -371,6 +581,12 @@ public: return BFI == SymbolToFunctionMap.end() ? nullptr : BFI->second; } + /// Associate the symbol \p Sym with the function \p BF for lookups with + /// getFunctionForSymbol(). + void setSymbolToFunctionMap(const MCSymbol *Sym, BinaryFunction *BF) { + SymbolToFunctionMap[Sym] = BF; + } + /// Populate some internal data structures with debug info. void preprocessDebugInfo( std::map &BinaryFunctions); diff --git a/bolt/BinaryData.cpp b/bolt/BinaryData.cpp new file mode 100644 index 000000000000..ea27bead6f9c --- /dev/null +++ b/bolt/BinaryData.cpp @@ -0,0 +1,132 @@ +//===--- BinaryData.cpp - Representation of section data objects ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +//===----------------------------------------------------------------------===// + +#include "BinaryData.h" +#include "BinarySection.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Regex.h" + +using namespace llvm; +using namespace bolt; + +#undef DEBUG_TYPE +#define DEBUG_TYPE "bolt" + +namespace opts { +extern cl::OptionCategory BoltCategory; +extern cl::opt Verbosity; + +cl::opt +PrintSymbolAliases("print-aliases", + cl::desc("print aliases when printing objects"), + cl::Hidden, + cl::ZeroOrMore, + cl::cat(BoltCategory)); +} + +bool BinaryData::isMoveable() const { + return (!isAbsolute() && + (IsMoveable && + (!Parent || isTopLevelJumpTable()))); +} + +void BinaryData::merge(const BinaryData *Other) { + assert(!Size || !Other->Size || Size == Other->Size); + assert(Address == Other->Address); + assert(*Section == *Other->Section); + assert(OutputOffset == Other->OutputOffset); + assert(OutputSection == Other->OutputSection); + Names.insert(Names.end(), Other->Names.begin(), Other->Names.end()); + Symbols.insert(Symbols.end(), Other->Symbols.begin(), Other->Symbols.end()); + MemData.insert(MemData.end(), Other->MemData.begin(), Other->MemData.end()); + if (!Size) + Size = Other->Size; +} + +bool BinaryData::hasNameRegex(StringRef NameRegex) const { + Regex MatchName(NameRegex); + for (auto &Name : Names) + if (MatchName.match(Name)) + return true; + return false; +} + +StringRef BinaryData::getSectionName() const { + return getSection().getName(); +} + +uint64_t BinaryData::computeOutputOffset() const { + return Address - getSection().getAddress(); +} + +void BinaryData::setSection(BinarySection &NewSection) { + Section = &NewSection; + if (OutputSection.empty()) + OutputSection = getSection().getName(); +} + +bool BinaryData::isMoved() const { + return (computeOutputOffset() != OutputOffset || + OutputSection != getSectionName()); +} + +void BinaryData::print(raw_ostream &OS) const { + printBrief(OS); +} + +void BinaryData::printBrief(raw_ostream &OS) const { + OS << "("; + + if (isJumpTable()) + OS << "jump-table: "; + else + OS << "object: "; + + OS << getName(); + + if ((opts::PrintSymbolAliases || opts::Verbosity > 1) && Names.size() > 1) { + OS << ", aliases:"; + for (unsigned I = 1u; I < Names.size(); ++I) { + OS << (I == 1 ? " (" : ", ") << Names[I]; + } + OS << ")"; + } + + if (opts::Verbosity > 1 && Parent) { + OS << " (" << Parent->getName() << "/" << Parent->getSize() << ")"; + } + + OS << ", 0x" << Twine::utohexstr(getAddress()) + << ":0x" << Twine::utohexstr(getEndAddress()) + << "/" << getSize(); + + if (opts::Verbosity > 1) { + for (auto &MI : memData()) { + OS << ", " << MI; + } + } + + OS << ")"; +} + +BinaryData::BinaryData(StringRef Name, + uint64_t Address, + uint64_t Size, + uint16_t Alignment, + BinarySection &Section) +: Names({Name}), + Section(&Section), + Address(Address), + Size(Size), + Alignment(Alignment), + OutputSection(Section.getName()), + OutputOffset(computeOutputOffset()) +{ } diff --git a/bolt/BinaryData.h b/bolt/BinaryData.h new file mode 100644 index 000000000000..0acace0ca7ae --- /dev/null +++ b/bolt/BinaryData.h @@ -0,0 +1,207 @@ +//===--- BinaryData.h - Representation of section data objects -----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVM_BOLT_BINARY_DATA_H +#define LLVM_TOOLS_LLVM_BOLT_BINARY_DATA_H + +#include "DataReader.h" +#include "llvm/MC/MCSymbol.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include +#include + +namespace llvm { +namespace bolt { + +struct BinarySection; + +/// \p BinaryData represents an indivisible part of a data section section. +/// BinaryData's may contain sub-components, e.g. jump tables but they are +/// considered to be part of the parent symbol in terms of divisibility and +/// reordering. +class BinaryData { + friend class BinaryContext; + /// Non-null if this BinaryData is contained in a larger BinaryData object, + /// i.e. the start and end addresses are contained within another object. + BinaryData *Parent{nullptr}; + + // non-copyable + BinaryData() = delete; + BinaryData(const BinaryData &) = delete; + BinaryData &operator=(const BinaryData &) = delete; + +protected: + /// All names associated with this data. The first name is the primary one. + std::vector Names; + /// All symbols associated with this data. This vector should have one entry + /// corresponding to every entry in \p Names. + std::vector Symbols; + + /// Section this data belongs to. + BinarySection *Section; + /// Start address of this symbol. + uint64_t Address{0}; + /// Size of this data (can be 0). + uint64_t Size{0}; + /// Alignment of this data. + uint16_t Alignment{1}; + + /// Output section for this data if it has been moved from the original + /// section. + std::string OutputSection; + /// The offset of this symbol in the output section. This is different + /// from \p Address - Section.getAddress() when the data has been reordered. + uint64_t OutputOffset{0}; + + /// Memory profiling data associated with this object. + std::vector MemData; + + bool IsMoveable{true}; + + void addMemData(const MemInfo &MI) { + MemData.push_back(MI); + } + + BinaryData *getRootData() { + auto *BD = this; + while (BD->Parent) + BD = BD->Parent; + return BD; + } + + BinaryData *getAtomicRoot() { + auto *BD = this; + while (!BD->isAtomic() && BD->Parent) + BD = BD->Parent; + return BD; + } + + uint64_t computeOutputOffset() const; + +public: + BinaryData(BinaryData &&) = default; + BinaryData(StringRef Name, + uint64_t Address, + uint64_t Size, + uint16_t Alignment, + BinarySection &Section); + virtual ~BinaryData() { } + + virtual bool isJumpTable() const { return false; } + virtual bool isObject() const { return !isJumpTable(); } + virtual void merge(const BinaryData *Other); + + bool isTopLevelJumpTable() const { + return (isJumpTable() && + (!Parent || (!Parent->Parent && Parent->isObject()))); + } + + // BinaryData that is considered atomic and potentially moveable. All + // MemInfo data and relocations should be wrt. to atomic data. + bool isAtomic() const { + return isTopLevelJumpTable() || !Parent; + } + + iterator_range::const_iterator> names() const { + return make_range(Names.begin(), Names.end()); + } + + iterator_range::const_iterator> symbols() const { + return make_range(Symbols.begin(), Symbols.end()); + } + + iterator_range::const_iterator> memData() const { + return make_range(MemData.begin(), MemData.end()); + } + + StringRef getName() const { return Names.front(); } + const std::vector &getNames() const { return Names; } + MCSymbol *getSymbol() { return Symbols.front(); } + const MCSymbol *getSymbol() const { return Symbols.front(); } + + bool hasName(StringRef Name) const { + return std::find(Names.begin(), Names.end(), Name) != Names.end(); + } + bool hasNameRegex(StringRef Name) const; + bool nameStartsWith(StringRef Prefix) const { + for (const auto &Name : Names) { + if (StringRef(Name).startswith(Prefix)) + return true; + } + return false; + } + + bool hasSymbol(const MCSymbol *Symbol) const { + return std::find(Symbols.begin(), Symbols.end(), Symbol) != Symbols.end(); + } + + bool isAbsolute() const { return getSymbol()->isAbsolute(); } + bool isMoveable() const; + + uint64_t getAddress() const { return Address; } + uint64_t getEndAddress() const { return Address + Size; } + uint64_t getSize() const { return Size; } + uint16_t getAlignment() const { return Alignment; } + uint64_t getOutputOffset() const { return OutputOffset; } + uint64_t getOutputSize() const { return Size; } + + BinarySection &getSection() { return *Section; } + const BinarySection &getSection() const { return *Section; } + StringRef getSectionName() const; + StringRef getOutputSection() const { return OutputSection; } + + bool isMoved() const; + bool containsAddress(uint64_t Address) const { + return ((getAddress() <= Address && Address < getEndAddress()) || + (getAddress() == Address && !getSize())); + } + bool containsRange(uint64_t Address, uint64_t Size) const { + return (getAddress() <= Address && Address + Size <= getEndAddress()); + } + + const BinaryData *getParent() const { + return Parent; + } + + const BinaryData *getRootData() const { + auto *BD = this; + while (BD->Parent) + BD = BD->Parent; + return BD; + } + + const BinaryData *getAtomicRoot() const { + auto *BD = this; + while (!BD->isAtomic() && BD->Parent) + BD = BD->Parent; + return BD; + } + + void setIsMoveable(bool Flag) { IsMoveable = Flag; } + void setOutputOffset(uint64_t Offset) { OutputOffset = Offset; } + void setOutputSection(StringRef Name) { OutputSection = Name; } + void setSection(BinarySection &NewSection); + + virtual void printBrief(raw_ostream &OS) const; + virtual void print(raw_ostream &OS) const; +}; + +inline raw_ostream &operator<<(raw_ostream &OS, const BinaryData &BD) { + BD.printBrief(OS); + return OS; +} + +} // namespace bolt +} // namespace llvm + +#endif diff --git a/bolt/BinaryFunction.cpp b/bolt/BinaryFunction.cpp index 0f59195919a5..7a12d8eeaf7b 100644 --- a/bolt/BinaryFunction.cpp +++ b/bolt/BinaryFunction.cpp @@ -129,7 +129,7 @@ PrintOnlyRegex("print-only-regex", cl::Hidden, cl::cat(BoltCategory)); -cl::opt +static cl::opt TimeBuild("time-build", cl::desc("print time spent constructing binary functions"), cl::ZeroOrMore, @@ -364,9 +364,9 @@ bool BinaryFunction::isForwardCall(const MCSymbol *CalleeSymbol) const { } } else { // Absolute symbol. - auto const CalleeSI = BC.GlobalSymbols.find(CalleeSymbol->getName()); - assert(CalleeSI != BC.GlobalSymbols.end() && "unregistered symbol found"); - return CalleeSI->second > getAddress(); + auto *CalleeSI = BC.getBinaryDataByName(CalleeSymbol->getName()); + assert(CalleeSI && "unregistered symbol found"); + return CalleeSI->getAddress() > getAddress(); } } @@ -563,7 +563,7 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation, // Print all jump tables. for (auto &JTI : JumpTables) { - JTI.second.print(OS); + JTI.second->print(OS); } OS << "DWARF CFI Instructions:\n"; @@ -675,9 +675,8 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, if (BC.TheTriple->getArch() == llvm::Triple::aarch64) { const auto *Sym = BC.MIA->getTargetSymbol(*PCRelBaseInstr, 1); assert (Sym && "Symbol extraction failed"); - auto SI = BC.GlobalSymbols.find(Sym->getName()); - if (SI != BC.GlobalSymbols.end()) { - PCRelAddr = SI->second; + if (auto *BD = BC.getBinaryDataByName(Sym->getName())) { + PCRelAddr = BD->getAddress(); } else { for (auto &Elmt : Labels) { if (Elmt.second == Sym) { @@ -708,10 +707,12 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, // RIP-relative addressing should be converted to symbol form by now // in processed instructions (but not in jump). if (DispExpr) { - auto SI = - BC.GlobalSymbols.find(BC.MIA->getTargetSymbol(DispExpr)->getName()); - assert(SI != BC.GlobalSymbols.end() && "global symbol needs a value"); - ArrayStart = SI->second; + const MCSymbol *TargetSym; + uint64_t TargetOffset; + std::tie(TargetSym, TargetOffset) = BC.MIA->getTargetSymbolInfo(DispExpr); + auto *BD = BC.getBinaryDataByName(TargetSym->getName()); + assert(BD && "global symbol needs a value"); + ArrayStart = BD->getAddress() + TargetOffset; BaseRegNum = 0; if (BC.TheTriple->getArch() == llvm::Triple::aarch64) { ArrayStart &= ~0xFFFULL; @@ -729,13 +730,13 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, // Check if there's already a jump table registered at this address. if (auto *JT = getJumpTableContainingAddress(ArrayStart)) { - auto JTOffset = ArrayStart - JT->Address; + auto JTOffset = ArrayStart - JT->getAddress(); if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && JTOffset != 0) { // Adjust the size of this jump table and create a new one if necessary. // We cannot re-use the entries since the offsets are relative to the // table start. DEBUG(dbgs() << "BOLT-DEBUG: adjusting size of jump table at 0x" - << Twine::utohexstr(JT->Address) << '\n'); + << Twine::utohexstr(JT->getAddress()) << '\n'); JT->OffsetEntries.resize(JTOffset / JT->EntrySize); } else { // Re-use an existing jump table. Perhaps parts of it. @@ -750,8 +751,10 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, // Get or create a new label for the table. auto LI = JT->Labels.find(JTOffset); if (LI == JT->Labels.end()) { - auto *JTStartLabel = BC.getOrCreateGlobalSymbol(ArrayStart, - "JUMP_TABLEat"); + auto *JTStartLabel = BC.registerNameAtAddress(generateJumpTableName(ArrayStart), + ArrayStart, + 0, + JT->EntrySize); auto Result = JT->Labels.emplace(JTOffset, JTStartLabel); assert(Result.second && "error adding jump table label"); LI = Result.first; @@ -827,20 +830,33 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE) { assert(JTOffsetCandidates.size() > 2 && "expected more than 2 jump table entries"); - auto *JTStartLabel = BC.getOrCreateGlobalSymbol(ArrayStart, "JUMP_TABLEat"); - DEBUG(dbgs() << "BOLT-DEBUG: creating jump table " - << JTStartLabel->getName() - << " in function " << *this << " with " - << JTOffsetCandidates.size() << " entries.\n"); + + auto JumpTableName = generateJumpTableName(ArrayStart); auto JumpTableType = Type == IndirectBranchType::POSSIBLE_JUMP_TABLE ? JumpTable::JTT_NORMAL : JumpTable::JTT_PIC; - JumpTables.emplace(ArrayStart, JumpTable{ArrayStart, - EntrySize, - JumpTableType, - std::move(JTOffsetCandidates), - {{0, JTStartLabel}}}); + + auto *JTStartLabel = BC.Ctx->getOrCreateSymbol(JumpTableName); + + auto JT = llvm::make_unique(JumpTableName, + ArrayStart, + EntrySize, + JumpTableType, + std::move(JTOffsetCandidates), + JumpTable::LabelMapType{{0, JTStartLabel}}, + *BC.getSectionForAddress(ArrayStart)); + + auto *JTLabel = BC.registerNameAtAddress(JumpTableName, + ArrayStart, + JT.get()); + assert(JTLabel == JTStartLabel); + + DEBUG(dbgs() << "BOLT-DEBUG: creating jump table " + << JTStartLabel->getName() + << " in function " << *this << " with " + << JTOffsetCandidates.size() << " entries.\n"); + JumpTables.emplace(ArrayStart, JT.release()); BC.MIA->replaceMemOperandDisp(const_cast(*MemLocInstr), JTStartLabel, BC.Ctx.get()); BC.MIA->setJumpTable(BC.Ctx.get(), Instruction, ArrayStart, IndexRegNum); @@ -849,6 +865,7 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, return Type; } + assert(!Value || BC.getSectionForAddress(Value)); BC.InterproceduralReferences.insert(Value); return IndirectBranchType::POSSIBLE_TAIL_CALL; } @@ -865,9 +882,9 @@ MCSymbol *BinaryFunction::getOrCreateLocalLabel(uint64_t Address, // Check if there's a global symbol registered at given address. // If so - reuse it since we want to keep the symbol value updated. if (Offset != 0) { - if (auto *Symbol = BC.getGlobalSymbolAtAddress(Address)) { - Labels[Offset] = Symbol; - return Symbol; + if (auto *BD = BC.getBinaryDataAtAddress(Address)) { + Labels[Offset] = BD->getSymbol(); + return BD->getSymbol(); } } @@ -903,6 +920,7 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { auto handlePCRelOperand = [&](MCInst &Instruction, uint64_t Address, uint64_t Size) { uint64_t TargetAddress{0}; + uint64_t TargetOffset{0}; MCSymbol *TargetSymbol{nullptr}; if (!MIA->evaluateMemOperandTarget(Instruction, TargetAddress, Address, Size)) { @@ -970,13 +988,31 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { BC.InterproceduralReferences.insert(TargetAddress); } } - if (!TargetSymbol) - TargetSymbol = BC.getOrCreateGlobalSymbol(TargetAddress, "DATAat"); + if (!TargetSymbol) { + auto *BD = BC.getBinaryDataContainingAddress(TargetAddress); + if (BD) { + TargetSymbol = BD->getSymbol(); + TargetOffset = TargetAddress - BD->getAddress(); + } else { + // TODO: use DWARF info to get size/alignment here? + TargetSymbol = BC.getOrCreateGlobalSymbol(TargetAddress, 0, 0, "DATAat"); + DEBUG(if (opts::Verbosity >= 2) { + dbgs() << "Created DATAat sym: " << TargetSymbol->getName() + << " in section " << BD->getSectionName() << "\n"; + }); + } + } + const MCExpr *Expr = MCSymbolRefExpr::create(TargetSymbol, + MCSymbolRefExpr::VK_None, + *BC.Ctx); + if (TargetOffset) { + auto *Offset = MCConstantExpr::create(TargetOffset, *BC.Ctx); + Expr = MCBinaryExpr::createAdd(Expr, Offset, *BC.Ctx); + } MIA->replaceMemOperandDisp( Instruction, MCOperand::createExpr(BC.MIA->getTargetExprFor( Instruction, - MCSymbolRefExpr::create( - TargetSymbol, MCSymbolRefExpr::VK_None, *BC.Ctx), + Expr, *BC.Ctx, 0))); return true; }; @@ -1050,33 +1086,39 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { // Check if there's a relocation associated with this instruction. bool UsedReloc{false}; - if (!Relocations.empty()) { - auto RI = Relocations.lower_bound(Offset); - if (RI != Relocations.end() && RI->first < Offset + Size) { - const auto &Relocation = RI->second; - DEBUG(dbgs() << "BOLT-DEBUG: replacing immediate with relocation" - " against " << Relocation.Symbol->getName() - << " in function " << *this - << " for instruction at offset 0x" - << Twine::utohexstr(Offset) << '\n'); - int64_t Value; - const auto Result = BC.MIA->replaceImmWithSymbol( - Instruction, Relocation.Symbol, Relocation.Addend, Ctx.get(), Value, - Relocation.Type); - (void)Result; - assert(Result && "cannot replace immediate with relocation"); - // For aarch, if we replaced an immediate with a symbol from a - // relocation, we mark it so we do not try to further process a - // pc-relative operand. All we need is the symbol. - if (BC.TheTriple->getArch() == llvm::Triple::aarch64) - UsedReloc = true; + for (auto Itr = Relocations.lower_bound(Offset); + Itr != Relocations.upper_bound(Offset + Size); + ++Itr) { + const auto &Relocation = Itr->second; + if (Relocation.Offset >= Offset + Size) + continue; - // Make sure we replaced the correct immediate (instruction - // can have multiple immediate operands). - assert((BC.TheTriple->getArch() == llvm::Triple::aarch64 || - static_cast(Value) == Relocation.Value) && - "immediate value mismatch in function"); - } + DEBUG(dbgs() << "BOLT-DEBUG: replacing immediate with relocation" + " against " << Relocation.Symbol->getName() + << "+" << Relocation.Addend + << " in function " << *this + << " for instruction at offset 0x" + << Twine::utohexstr(Offset) << '\n'); + int64_t Value = Relocation.Value; + const auto Result = BC.MIA->replaceImmWithSymbol(Instruction, + Relocation.Symbol, + Relocation.Addend, + Ctx.get(), + Value, + Relocation.Type); + (void)Result; + assert(Result && "cannot replace immediate with relocation"); + // For aarch, if we replaced an immediate with a symbol from a + // relocation, we mark it so we do not try to further process a + // pc-relative operand. All we need is the symbol. + if (BC.TheTriple->getArch() == llvm::Triple::aarch64) + UsedReloc = true; + + // Make sure we replaced the correct immediate (instruction + // can have multiple immediate operands). + assert((BC.TheTriple->getArch() == llvm::Triple::aarch64 || + static_cast(Value) == Relocation.Value) && + "immediate value mismatch in function"); } // Convert instruction to a shorter version that could be relaxed if needed. @@ -1157,6 +1199,8 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { } TargetSymbol = BC.getOrCreateGlobalSymbol(TargetAddress, + 0, + 0, "FUNCat"); if (TargetAddress == 0) { // We actually see calls to address 0 in presence of weak symbols @@ -1288,12 +1332,13 @@ add_instruction: void BinaryFunction::postProcessJumpTables() { // Create labels for all entries. for (auto &JTI : JumpTables) { - auto &JT = JTI.second; + auto &JT = *JTI.second; for (auto Offset : JT.OffsetEntries) { auto *Label = getOrCreateLocalLabel(getAddress() + Offset, /*CreatePastEnd*/ true); JT.Entries.push_back(Label); } + BC.setBinaryDataSize(JT.getAddress(), JT.getSize()); } // Add TakenBranches from JumpTables. @@ -1305,7 +1350,7 @@ void BinaryFunction::postProcessJumpTables() { const auto JTAddress = JTSite.second; const auto *JT = getJumpTableContainingAddress(JTAddress); assert(JT && "cannot find jump table for address"); - auto EntryOffset = JTAddress - JT->Address; + auto EntryOffset = JTAddress - JT->getAddress(); while (EntryOffset < JT->getSize()) { auto TargetOffset = JT->OffsetEntries[EntryOffset / JT->EntrySize]; if (TargetOffset < getSize()) @@ -1313,7 +1358,7 @@ void BinaryFunction::postProcessJumpTables() { // Take ownership of jump table relocations. if (BC.HasRelocations) { - auto EntryAddress = JT->Address + EntryOffset; + auto EntryAddress = JT->getAddress() + EntryOffset; auto Res = BC.removeRelocationAt(EntryAddress); (void)Res; DEBUG( @@ -1335,7 +1380,7 @@ void BinaryFunction::postProcessJumpTables() { // Free memory used by jump table offsets. for (auto &JTI : JumpTables) { - auto &JT = JTI.second; + auto &JT = *JTI.second; clearList(JT.OffsetEntries); } @@ -1755,7 +1800,8 @@ void BinaryFunction::addEntryPoint(uint64_t Address) { << " at offset 0x" << Twine::utohexstr(Address - getAddress()) << '\n'); - auto *EntrySymbol = BC.getGlobalSymbolAtAddress(Address); + auto *EntryBD = BC.getBinaryDataAtAddress(Address); + auto *EntrySymbol = EntryBD ? EntryBD->getSymbol() : nullptr; // If we haven't disassembled the function yet we can add a new entry point // even if it doesn't have an associated entry in the symbol table. @@ -2905,26 +2951,28 @@ bool BinaryFunction::isIdenticalWith(const BinaryFunction &OtherBF, } // Check if symbols are jump tables. - auto SIA = BC.GlobalSymbols.find(A->getName()); - if (SIA == BC.GlobalSymbols.end()) + auto *SIA = BC.getBinaryDataByName(A->getName()); + if (!SIA) return false; - auto SIB = BC.GlobalSymbols.find(B->getName()); - if (SIB == BC.GlobalSymbols.end()) + auto *SIB = BC.getBinaryDataByName(B->getName()); + if (!SIB) return false; - assert((SIA->second != SIB->second) && + assert((SIA->getAddress() != SIB->getAddress()) && "different symbols should not have the same value"); - const auto *JumpTableA = getJumpTableContainingAddress(SIA->second); + const auto *JumpTableA = + getJumpTableContainingAddress(SIA->getAddress()); if (!JumpTableA) return false; + const auto *JumpTableB = - OtherBF.getJumpTableContainingAddress(SIB->second); + OtherBF.getJumpTableContainingAddress(SIB->getAddress()); if (!JumpTableB) return false; - if ((SIA->second - JumpTableA->Address) != - (SIB->second - JumpTableB->Address)) + if ((SIA->getAddress() - JumpTableA->getAddress()) != + (SIB->getAddress() - JumpTableB->getAddress())) return false; return equalJumpTables(JumpTableA, JumpTableB, OtherBF); @@ -2955,6 +3003,24 @@ bool BinaryFunction::isIdenticalWith(const BinaryFunction &OtherBF, return true; } +std::string BinaryFunction::generateJumpTableName(uint64_t Address) const { + auto *JT = getJumpTableContainingAddress(Address); + size_t Id; + uint64_t Offset = 0; + if (JT) { + Offset = Address - JT->getAddress(); + auto Itr = JT->Labels.find(Offset); + if (Itr != JT->Labels.end()) { + return Itr->second->getName(); + } + Id = JumpTableIds.at(JT->getAddress()); + } else { + Id = JumpTableIds[Address] = JumpTables.size(); + } + return ("JUMP_TABLE/" + Names[0] + "." + std::to_string(Id) + + (Offset ? ("." + std::to_string(Offset)) : "")); +} + bool BinaryFunction::equalJumpTables(const JumpTable *JumpTableA, const JumpTable *JumpTableB, const BinaryFunction &BFB) const { @@ -3282,17 +3348,18 @@ void BinaryFunction::emitJumpTables(MCStreamer *Streamer) { outs() << "BOLT-INFO: jump tables for function " << *this << ":\n"; } for (auto &JTI : JumpTables) { - auto &JT = JTI.second; + auto &JT = *JTI.second; if (opts::PrintJumpTables) JT.print(outs()); if (opts::JumpTables == JTS_BASIC && BC.HasRelocations) { - JT.updateOriginal(BC); + JT.updateOriginal(); } else { MCSection *HotSection, *ColdSection; if (opts::JumpTables == JTS_BASIC) { - JT.SectionName = - ".local.JUMP_TABLEat0x" + Twine::utohexstr(JT.Address).str(); - HotSection = BC.Ctx->getELFSection(JT.SectionName, + std::string Name = JT.Labels[0]->getName().str(); + std::replace(Name.begin(), Name.end(), '/', '.'); + JT.setOutputSection(".local." + Name); + HotSection = BC.Ctx->getELFSection(JT.getOutputSection(), ELF::SHT_PROGBITS, ELF::SHF_ALLOC); ColdSection = HotSection; @@ -3311,157 +3378,6 @@ void BinaryFunction::emitJumpTables(MCStreamer *Streamer) { } } -std::pair -BinaryFunction::JumpTable::getEntriesForAddress(const uint64_t Addr) const { - const uint64_t InstOffset = Addr - Address; - size_t StartIndex = 0, EndIndex = 0; - uint64_t Offset = 0; - - for (size_t I = 0; I < Entries.size(); ++I) { - auto LI = Labels.find(Offset); - if (LI != Labels.end()) { - const auto NextLI = std::next(LI); - const auto NextOffset = - NextLI == Labels.end() ? getSize() : NextLI->first; - if (InstOffset >= LI->first && InstOffset < NextOffset) { - StartIndex = I; - EndIndex = I; - while (Offset < NextOffset) { - ++EndIndex; - Offset += EntrySize; - } - break; - } - } - Offset += EntrySize; - } - - return std::make_pair(StartIndex, EndIndex); -} - -bool BinaryFunction::JumpTable::replaceDestination(uint64_t JTAddress, - const MCSymbol *OldDest, - MCSymbol *NewDest) { - bool Patched{false}; - const auto Range = getEntriesForAddress(JTAddress); - for (auto I = &Entries[Range.first], E = &Entries[Range.second]; - I != E; ++I) { - auto &Entry = *I; - if (Entry == OldDest) { - Patched = true; - Entry = NewDest; - } - } - return Patched; -} - -void BinaryFunction::JumpTable::updateOriginal(BinaryContext &BC) { - // In non-relocation mode we have to emit jump tables in local sections. - // This way we only overwrite them when a corresponding function is - // overwritten. - assert(BC.HasRelocations && "relocation mode expected"); - auto Section = BC.getSectionForAddress(Address); - assert(Section && "section not found for jump table"); - uint64_t Offset = Address - Section->getAddress(); - StringRef SectionName = Section->getName(); - for (auto *Entry : Entries) { - const auto RelType = (Type == JTT_NORMAL) ? ELF::R_X86_64_64 - : ELF::R_X86_64_PC32; - const uint64_t RelAddend = (Type == JTT_NORMAL) - ? 0 : Offset - (Address - Section->getAddress()); - DEBUG(dbgs() << "adding relocation to section " << SectionName - << " at offset " << Twine::utohexstr(Offset) << " for symbol " - << Entry->getName() << " with addend " - << Twine::utohexstr(RelAddend) << '\n'); - Section->addRelocation(Offset, Entry, RelType, RelAddend); - Offset += EntrySize; - } -} - -uint64_t BinaryFunction::JumpTable::emit(MCStreamer *Streamer, - MCSection *HotSection, - MCSection *ColdSection) { - // Pre-process entries for aggressive splitting. - // Each label represents a separate switch table and gets its own count - // determining its destination. - std::map LabelCounts; - if (opts::JumpTables > JTS_SPLIT && !Counts.empty()) { - MCSymbol *CurrentLabel = Labels[0]; - uint64_t CurrentLabelCount = 0; - for (unsigned Index = 0; Index < Entries.size(); ++Index) { - auto LI = Labels.find(Index * EntrySize); - if (LI != Labels.end()) { - LabelCounts[CurrentLabel] = CurrentLabelCount; - CurrentLabel = LI->second; - CurrentLabelCount = 0; - } - CurrentLabelCount += Counts[Index].Count; - } - LabelCounts[CurrentLabel] = CurrentLabelCount; - } else { - Streamer->SwitchSection(Count > 0 ? HotSection : ColdSection); - Streamer->EmitValueToAlignment(EntrySize); - } - MCSymbol *LastLabel = nullptr; - uint64_t Offset = 0; - for (auto *Entry : Entries) { - auto LI = Labels.find(Offset); - if (LI != Labels.end()) { - DEBUG(dbgs() << "BOLT-DEBUG: emitting jump table " - << LI->second->getName() << " (originally was at address 0x" - << Twine::utohexstr(Address + Offset) - << (Offset ? "as part of larger jump table\n" : "\n")); - if (!LabelCounts.empty()) { - DEBUG(dbgs() << "BOLT-DEBUG: jump table count: " - << LabelCounts[LI->second] << '\n'); - if (LabelCounts[LI->second] > 0) { - Streamer->SwitchSection(HotSection); - } else { - Streamer->SwitchSection(ColdSection); - } - Streamer->EmitValueToAlignment(EntrySize); - } - Streamer->EmitLabel(LI->second); - LastLabel = LI->second; - } - if (Type == JTT_NORMAL) { - Streamer->EmitSymbolValue(Entry, OutputEntrySize); - } else { // JTT_PIC - auto JT = MCSymbolRefExpr::create(LastLabel, Streamer->getContext()); - auto E = MCSymbolRefExpr::create(Entry, Streamer->getContext()); - auto Value = MCBinaryExpr::createSub(E, JT, Streamer->getContext()); - Streamer->EmitValue(Value, EntrySize); - } - Offset += EntrySize; - } - - return Offset; -} - -void BinaryFunction::JumpTable::print(raw_ostream &OS) const { - uint64_t Offset = 0; - for (const auto *Entry : Entries) { - auto LI = Labels.find(Offset); - if (LI != Labels.end()) { - OS << "Jump Table " << LI->second->getName() << " at @0x" - << Twine::utohexstr(Address+Offset); - if (Offset) { - OS << " (possibly part of larger jump table):\n"; - } else { - OS << " with total count of " << Count << ":\n"; - } - } - OS << format(" 0x%04" PRIx64 " : ", Offset) << Entry->getName(); - if (!Counts.empty()) { - OS << " : " << Counts[Offset / EntrySize].Mispreds - << "/" << Counts[Offset / EntrySize].Count; - } - OS << '\n'; - Offset += EntrySize; - } - OS << "\n\n"; -} - void BinaryFunction::calculateLoopInfo() { // Discover loops. BinaryDominatorTree DomTree; diff --git a/bolt/BinaryFunction.h b/bolt/BinaryFunction.h index 87bfb15ad654..99e9cb7bfb52 100644 --- a/bolt/BinaryFunction.h +++ b/bolt/BinaryFunction.h @@ -22,6 +22,7 @@ #include "BinaryLoop.h" #include "DataReader.h" #include "DebugData.h" +#include "JumpTable.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/iterator.h" @@ -51,8 +52,6 @@ class DWARFDebugInfoEntryMinimal; namespace bolt { -struct SectionInfo; - using DWARFUnitLineTable = std::pair; @@ -150,14 +149,6 @@ inline raw_ostream &operator<<(raw_ostream &OS, const DynoStats &Stats) { DynoStats operator+(const DynoStats &A, const DynoStats &B); -enum JumpTableSupportLevel : char { - JTS_NONE = 0, /// Disable jump tables support. - JTS_BASIC = 1, /// Enable basic jump tables support (in-place). - JTS_MOVE = 2, /// Move jump tables to a separate section. - JTS_SPLIT = 3, /// Enable hot/cold splitting of jump tables. - JTS_AGGRESSIVE = 4, /// Aggressive splitting of jump tables. -}; - enum IndirectCallPromotionType : char { ICP_NONE, /// Don't perform ICP. ICP_CALLS, /// Perform ICP on indirect calls. @@ -231,12 +222,6 @@ public: ST_ALL, /// Split all functions }; - /// Branch statistics for jump table entries. - struct JumpInfo { - uint64_t Mispreds{0}; - uint64_t Count{0}; - }; - static constexpr uint64_t COUNT_NO_PROFILE = BinaryBasicBlock::COUNT_NO_PROFILE; @@ -567,90 +552,17 @@ private: /// function and that apply before the entry basic block). CFIInstrMapType CIEFrameInstructions; -public: - /// Representation of a jump table. - /// - /// The jump table may include other jump tables that are referenced by - /// a different label at a different offset in this jump table. - struct JumpTable { - enum JumpTableType : char { - JTT_NORMAL, - JTT_PIC, - }; - - /// Original address. - uint64_t Address; - - /// Size of the entry used for storage. - std::size_t EntrySize; - - /// Size of the entry size we will write (we may use a more compact layout) - std::size_t OutputEntrySize; - - /// The type of this jump table. - JumpTableType Type; - - /// All the entries as labels. - std::vector Entries; - - /// All the entries as offsets into a function. Invalid after CFG is built. - std::vector OffsetEntries; - - /// Map ->