diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h index 7419bc5fb8bc..111fa9f9ec01 100644 --- a/bolt/include/bolt/Core/BinaryBasicBlock.h +++ b/bolt/include/bolt/Core/BinaryBasicBlock.h @@ -6,11 +6,14 @@ // //===----------------------------------------------------------------------===// // +// Sequence of MC(Plus) instructions. Call/invoke does not terminate the block. +// //===----------------------------------------------------------------------===// #ifndef BOLT_CORE_BINARY_BASIC_BLOCK_H #define BOLT_CORE_BINARY_BASIC_BLOCK_H +#include "bolt/Core/MCPlus.h" #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/StringRef.h" #include "llvm/MC/MCInst.h" @@ -27,8 +30,6 @@ namespace bolt { class BinaryFunction; -/// The intention is to keep the structure similar to MachineBasicBlock as -/// we might switch to it at some point. class BinaryBasicBlock { public: @@ -56,18 +57,27 @@ public: static constexpr uint32_t INVALID_OFFSET = std::numeric_limits::max(); + using BranchInfoType = SmallVector; + private: /// Vector of all instructions in the block. - std::vector Instructions; + InstructionListType Instructions; /// CFG information. - std::vector Predecessors; - std::vector Successors; - std::vector Throwers; - std::vector LandingPads; + using EdgeListType = SmallVector; + EdgeListType Predecessors; + EdgeListType Successors; /// Each successor has a corresponding BranchInfo entry in the list. - std::vector BranchInfo; + BranchInfoType BranchInfo; + + using ExceptionListType = SmallVector; + + /// List of blocks that this landing pad is handling. + ExceptionListType Throwers; + + /// List of blocks that can catch exceptions thrown by code in this block. + ExceptionListType LandingPads; /// Function that owns this basic block. BinaryFunction *Function; @@ -134,7 +144,9 @@ private: private: BinaryBasicBlock() = delete; BinaryBasicBlock(const BinaryBasicBlock &) = delete; - BinaryBasicBlock& operator=(const BinaryBasicBlock &) = delete; + BinaryBasicBlock(const BinaryBasicBlock &&) = delete; + BinaryBasicBlock &operator=(const BinaryBasicBlock &) = delete; + BinaryBasicBlock &operator=(const BinaryBasicBlock &&) = delete; explicit BinaryBasicBlock( BinaryFunction *Function, @@ -162,9 +174,9 @@ public: std::numeric_limits::max(); // Instructions iterators. - using iterator = std::vector::iterator; - using const_iterator = std::vector::const_iterator; - using reverse_iterator = std::reverse_iterator; + using iterator = InstructionListType::iterator; + using const_iterator = InstructionListType::const_iterator; + using reverse_iterator = std::reverse_iterator; using const_reverse_iterator = std::reverse_iterator; bool empty() const { assert(hasInstructions()); @@ -198,14 +210,14 @@ public: return Instructions.rend(); } // CFG iterators. - using pred_iterator = std::vector::iterator; - using const_pred_iterator = std::vector::const_iterator; - using succ_iterator = std::vector::iterator; - using const_succ_iterator = std::vector::const_iterator; - using throw_iterator = decltype(Throwers)::iterator; + using pred_iterator = EdgeListType::iterator; + using const_pred_iterator = EdgeListType::const_iterator; + using succ_iterator = EdgeListType::iterator; + using const_succ_iterator = EdgeListType::const_iterator; + using throw_iterator = decltype(Throwers)::iterator; using const_throw_iterator = decltype(Throwers)::const_iterator; - using lp_iterator = decltype(LandingPads)::iterator; - using const_lp_iterator = decltype(LandingPads)::const_iterator; + using lp_iterator = decltype(LandingPads)::iterator; + using const_lp_iterator = decltype(LandingPads)::const_iterator; using pred_reverse_iterator = std::reverse_iterator; using const_pred_reverse_iterator = @@ -309,9 +321,8 @@ public: } // BranchInfo iterators. - using branch_info_iterator = std::vector::iterator; - using const_branch_info_iterator = - std::vector::const_iterator; + using branch_info_iterator = BranchInfoType::iterator; + using const_branch_info_iterator = BranchInfoType::const_iterator; using branch_info_reverse_iterator = std::reverse_iterator; using const_branch_info_reverse_iterator = @@ -351,12 +362,10 @@ public: } /// Get instruction at given index. - MCInst &getInstructionAtIndex(unsigned Index) { - return Instructions.at(Index); - } + MCInst &getInstructionAtIndex(unsigned Index) { return Instructions[Index]; } const MCInst &getInstructionAtIndex(unsigned Index) const { - return Instructions.at(Index); + return Instructions[Index]; } /// Return symbol marking the start of this basic block. @@ -503,7 +512,7 @@ public: template Itr insertPseudoInstr(Itr Pos, MCInst &Instr) { ++NumPseudos; - return Instructions.emplace(Pos, Instr); + return Instructions.insert(Pos, Instr); } /// Return the number of pseudo instructions in the basic block. @@ -774,7 +783,7 @@ public: } iterator replaceInstruction(iterator II, - const std::vector &Replacement) { + const InstructionListType &Replacement) { return replaceInstruction(II, Replacement.begin(), Replacement.end()); } @@ -787,7 +796,7 @@ public: iterator insertInstruction(iterator At, MCInst &NewInst) { adjustNumPseudos(NewInst, 1); - return Instructions.emplace(At, NewInst); + return Instructions.insert(At, NewInst); } /// Helper to retrieve any terminators in \p BB before \p Pos. This is used @@ -802,8 +811,8 @@ public: /// Split apart the instructions in this basic block starting at Inst. /// The instructions following Inst are removed and returned in a vector. - std::vector splitInstructions(const MCInst *Inst) { - std::vector SplitInst; + InstructionListType splitInstructions(const MCInst *Inst) { + InstructionListType SplitInst; assert(!Instructions.empty()); while(&Instructions.back() != Inst) { diff --git a/bolt/include/bolt/Core/MCPlus.h b/bolt/include/bolt/Core/MCPlus.h index 2d92e15f8317..15dd0bd751d7 100644 --- a/bolt/include/bolt/Core/MCPlus.h +++ b/bolt/include/bolt/Core/MCPlus.h @@ -15,10 +15,14 @@ #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" #include "llvm/Support/Casting.h" +#include namespace llvm { namespace bolt { +// NOTE: using SmallVector for instruction list results in a memory regression. +using InstructionListType = std::vector; + namespace MCPlus { /// This type represents C++ EH info for a callsite. The symbol is the landing diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h index 049ef8006f87..6197ce2d297f 100644 --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -253,30 +253,30 @@ public: Itr.reset(Other.Itr->Copy()); return *this; } - InstructionIterator() { } + InstructionIterator() {} InstructionIterator(const InstructionIterator &Other) - : Itr(Other.Itr->Copy()) { } + : Itr(Other.Itr->Copy()) {} InstructionIterator(InstructionIterator &&Other) - : Itr(std::move(Other.Itr)) { } + : Itr(std::move(Other.Itr)) {} explicit InstructionIterator(std::unique_ptr Itr) - : Itr(std::move(Itr)) { } + : Itr(std::move(Itr)) {} - InstructionIterator(std::vector::iterator Itr) - : Itr(new SeqImpl::iterator>(Itr)) { } + InstructionIterator(InstructionListType::iterator Itr) + : Itr(new SeqImpl(Itr)) {} template - InstructionIterator(T *Itr) - : Itr(new SeqImpl(Itr)) { } + InstructionIterator(T *Itr) : Itr(new SeqImpl(Itr)) {} InstructionIterator(ArrayRef::iterator Itr) - : Itr(new SeqImpl::iterator>(Itr)) { } + : Itr(new SeqImpl::iterator>(Itr)) {} InstructionIterator(MutableArrayRef::iterator Itr) - : Itr(new SeqImpl::iterator>(Itr)) { } + : Itr(new SeqImpl::iterator>(Itr)) {} // TODO: it would be nice to templatize this on the key type. InstructionIterator(std::map::iterator Itr) - : Itr(new MapImpl::iterator>(Itr)) { } + : Itr(new MapImpl::iterator>(Itr)) {} + private: std::unique_ptr Itr; }; @@ -429,7 +429,7 @@ public: } /// Create increment contents of target by 1 for Instrumentation - virtual void createInstrIncMemory(std::vector &Instrs, + virtual void createInstrIncMemory(InstructionListType &Instrs, const MCSymbol *Target, MCContext *Ctx, bool IsLeaf) const { llvm_unreachable("not implemented"); @@ -1357,12 +1357,12 @@ public: return false; } - virtual void createLongJmp(std::vector &Seq, const MCSymbol *Target, + virtual void createLongJmp(InstructionListType &Seq, const MCSymbol *Target, MCContext *Ctx, bool IsTailCall = false) { llvm_unreachable("not implemented"); } - virtual void createShortJmp(std::vector &Seq, const MCSymbol *Target, + virtual void createShortJmp(InstructionListType &Seq, const MCSymbol *Target, MCContext *Ctx, bool IsTailCall = false) { llvm_unreachable("not implemented"); } @@ -1409,7 +1409,7 @@ public: } /// Store \p Target absolute adddress to \p RegName - virtual std::vector materializeAddress(const MCSymbol *Target, + virtual InstructionListType materializeAddress(const MCSymbol *Target, MCContext *Ctx, MCPhysReg RegName, int64_t Addend = 0) const { @@ -1447,7 +1447,7 @@ public: return false; } - virtual void createLongTailCall(std::vector &Seq, + virtual void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target, MCContext *Ctx) { llvm_unreachable("not implemented"); } @@ -1535,23 +1535,23 @@ public: } /// Create an inline version of memcpy(dest, src, 1). - virtual std::vector createOneByteMemcpy() const { + virtual InstructionListType createOneByteMemcpy() const { llvm_unreachable("not implemented"); return {}; } /// Create a sequence of instructions to compare contents of a register /// \p RegNo to immediate \Imm and jump to \p Target if they are equal. - virtual std::vector - createCmpJE(MCPhysReg RegNo, int64_t Imm, const MCSymbol *Target, - MCContext *Ctx) const { + virtual InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm, + const MCSymbol *Target, + MCContext *Ctx) const { llvm_unreachable("not implemented"); return {}; } /// Creates inline memcpy instruction. If \p ReturnEnd is true, then return /// (dest + n) instead of dest. - virtual std::vector createInlineMemcpy(bool ReturnEnd) const { + virtual InstructionListType createInlineMemcpy(bool ReturnEnd) const { llvm_unreachable("not implemented"); return {}; } @@ -1816,62 +1816,62 @@ public: /// Remove meta-data, but don't destroy it. void stripAnnotations(MCInst &Inst, bool KeepTC = false); - virtual std::vector + virtual InstructionListType createInstrumentedIndirectCall(const MCInst &CallInst, bool TailCall, MCSymbol *HandlerFuncAddr, int CallSiteID, MCContext *Ctx) { llvm_unreachable("not implemented"); - return std::vector(); + return InstructionListType(); } - virtual std::vector createInstrumentedIndCallHandlerExitBB() const { + virtual InstructionListType createInstrumentedIndCallHandlerExitBB() const { llvm_unreachable("not implemented"); - return std::vector(); + return InstructionListType(); } - virtual std::vector + virtual InstructionListType createInstrumentedIndTailCallHandlerExitBB() const { llvm_unreachable("not implemented"); - return std::vector(); + return InstructionListType(); } - virtual std::vector + virtual InstructionListType createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline, const MCSymbol *IndCallHandler, MCContext *Ctx) { llvm_unreachable("not implemented"); - return std::vector(); + return InstructionListType(); } - virtual std::vector createNumCountersGetter(MCContext *Ctx) const { + virtual InstructionListType createNumCountersGetter(MCContext *Ctx) const { llvm_unreachable("not implemented"); return {}; } - virtual std::vector createInstrLocationsGetter(MCContext *Ctx) const { + virtual InstructionListType createInstrLocationsGetter(MCContext *Ctx) const { llvm_unreachable("not implemented"); return {}; } - virtual std::vector createInstrTablesGetter(MCContext *Ctx) const { + virtual InstructionListType createInstrTablesGetter(MCContext *Ctx) const { llvm_unreachable("not implemented"); return {}; } - virtual std::vector createInstrNumFuncsGetter(MCContext *Ctx) const { + virtual InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const { llvm_unreachable("not implemented"); return {}; } - virtual std::vector createSymbolTrampoline(const MCSymbol *TgtSym, + virtual InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym, MCContext *Ctx) const { llvm_unreachable("not implemented"); - return std::vector(); + return InstructionListType(); } - virtual std::vector createDummyReturnFunction(MCContext *Ctx) const { + virtual InstructionListType createDummyReturnFunction(MCContext *Ctx) const { llvm_unreachable("not implemented"); - return std::vector(); + return InstructionListType(); } /// This method takes an indirect call instruction and splits it up into an @@ -1892,7 +1892,8 @@ public: /// empty vector of instructions. The label is meant to indicate the basic /// block where all previous snippets are joined, i.e. the instructions that /// would immediate follow the original call. - using BlocksVectorTy = std::vector>>; + using BlocksVectorTy = + std::vector>; struct MultiBlocksCode { BlocksVectorTy Blocks; std::vector Successors; diff --git a/bolt/include/bolt/Passes/Instrumentation.h b/bolt/include/bolt/Passes/Instrumentation.h index c9cc03ee8546..697b70b015e9 100644 --- a/bolt/include/bolt/Passes/Instrumentation.h +++ b/bolt/include/bolt/Passes/Instrumentation.h @@ -62,7 +62,7 @@ private: void createLeafNodeDescription(FunctionDescription &FuncDesc, uint32_t Node); /// Create the sequence of instructions to increment a counter - std::vector createInstrumentationSnippet(BinaryContext &BC, + InstructionListType createInstrumentationSnippet(BinaryContext &BC, bool IsLeaf); // Critical edges worklist @@ -73,7 +73,7 @@ private: // instrumentOneTarget() populates this, instrumentFunction() consumes. using SplitWorklistTy = std::vector>; - using SplitInstrsTy = std::vector>; + using SplitInstrsTy = std::vector; /// Instrument the branch or call in \p Iter. \p TargetBB should be non-null /// if this is a local branch and null if it is a call. Return true if the diff --git a/bolt/lib/Core/Exceptions.cpp b/bolt/lib/Core/Exceptions.cpp index 19bbc59f4368..a4c2cf156938 100644 --- a/bolt/lib/Core/Exceptions.cpp +++ b/bolt/lib/Core/Exceptions.cpp @@ -186,12 +186,11 @@ void BinaryFunction::parseLSDA(ArrayRef LSDASectionData, // Create a handler entry if necessary. MCSymbol *LPSymbol = nullptr; if (LandingPad) { - if (Instructions.find(LandingPad) == Instructions.end()) { - if (opts::Verbosity >= 1) { + if (!getInstructionAtOffset(LandingPad)) { + if (opts::Verbosity >= 1) errs() << "BOLT-WARNING: landing pad " << Twine::utohexstr(LandingPad) - << " not pointing to an instruction in function " - << *this << " - ignoring.\n"; - } + << " not pointing to an instruction in function " << *this + << " - ignoring.\n"; } else { auto Label = Labels.find(LandingPad); if (Label != Labels.end()) { diff --git a/bolt/lib/Passes/ADRRelaxationPass.cpp b/bolt/lib/Passes/ADRRelaxationPass.cpp index 76c04cd8e664..72b847ff5eda 100644 --- a/bolt/lib/Passes/ADRRelaxationPass.cpp +++ b/bolt/lib/Passes/ADRRelaxationPass.cpp @@ -51,7 +51,7 @@ void ADRRelaxationPass::runOnFunction(BinaryFunction &BF) { MCPhysReg Reg; BC.MIB->getADRReg(Inst, Reg); int64_t Addend = BC.MIB->getTargetAddend(Inst); - std::vector Addr = + InstructionListType Addr = BC.MIB->materializeAddress(Symbol, BC.Ctx.get(), Reg, Addend); It = BB->replaceInstruction(It, Addr); } diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp index 9b939c272630..8664d962cc53 100644 --- a/bolt/lib/Passes/BinaryPasses.cpp +++ b/bolt/lib/Passes/BinaryPasses.cpp @@ -1697,7 +1697,7 @@ void InlineMemcpy::runOnFunctions(BinaryContext &BC) { const bool IsMemcpy8 = (CalleeSymbol->getName() == "_memcpy8"); const bool IsTailCall = BC.MIB->isTailCall(Inst); - const std::vector NewCode = + const InstructionListType NewCode = BC.MIB->createInlineMemcpy(IsMemcpy8); II = BB.replaceInstruction(II, NewCode); std::advance(II, NewCode.size() - 1); @@ -1816,7 +1816,7 @@ void SpecializeMemcpy1::runOnFunctions(BinaryContext &BC) { BinaryBasicBlock *MemcpyBB = Function.addBasicBlock(CurBB->getInputOffset()); - std::vector CmpJCC = + InstructionListType CmpJCC = BC.MIB->createCmpJE(BC.MIB->getIntArgRegister(2), 1, OneByteMemcpyBB->getLabel(), BC.Ctx.get()); CurBB->addInstructions(CmpJCC); @@ -1832,7 +1832,7 @@ void SpecializeMemcpy1::runOnFunctions(BinaryContext &BC) { if (CurBB->getKnownExecutionCount() > 0) MemcpyBB->setExecutionCount(1); - std::vector OneByteMemcpy = BC.MIB->createOneByteMemcpy(); + InstructionListType OneByteMemcpy = BC.MIB->createOneByteMemcpy(); OneByteMemcpyBB->addInstructions(OneByteMemcpy); ++NumSpecialized; diff --git a/bolt/lib/Passes/IndirectCallPromotion.cpp b/bolt/lib/Passes/IndirectCallPromotion.cpp index f7d68a4625ce..7e39b92320e5 100644 --- a/bolt/lib/Passes/IndirectCallPromotion.cpp +++ b/bolt/lib/Passes/IndirectCallPromotion.cpp @@ -792,7 +792,7 @@ IndirectCallPromotion::rewriteCall( // Remember any pseudo instructions following a tail call. These // must be preserved and moved to the original block. - std::vector TailInsts; + InstructionListType TailInsts; const MCInst *TailInst = &CallInst; if (IsTailCallOrJT) { while (TailInst + 1 < &(*IndCallBlock.end()) && @@ -801,7 +801,7 @@ IndirectCallPromotion::rewriteCall( } } - std::vector MovedInst = IndCallBlock.splitInstructions(&CallInst); + InstructionListType MovedInst = IndCallBlock.splitInstructions(&CallInst); // Link new BBs to the original input offset of the BB where the indirect // call site is, so we can map samples recorded in new BBs back to the // original BB seen in the input binary (if using BAT) @@ -821,7 +821,7 @@ IndirectCallPromotion::rewriteCall( for (auto Itr = ICPcode.begin() + 1; Itr != ICPcode.end(); ++Itr) { MCSymbol *&Sym = Itr->first; - std::vector &Insts = Itr->second; + InstructionListType &Insts = Itr->second; assert(Sym); std::unique_ptr TBB = Function.createBasicBlock(OrigOffset, Sym); @@ -860,8 +860,8 @@ IndirectCallPromotion::fixCFG(BinaryBasicBlock &IndCallBlock, } if (TotalIndirectBranches == 0) TotalIndirectBranches = 1; - std::vector BBI; - std::vector ScaledBBI; + BinaryBasicBlock::BranchInfoType BBI; + BinaryBasicBlock::BranchInfoType ScaledBBI; for (const Callsite &Target : Targets) { const size_t NumEntries = std::max(static_cast(1UL), Target.JTIndices.size()); @@ -1421,7 +1421,7 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) { dbgs() << "BOLT-INFO: ICP indirect call code:\n"; for (const auto &entry : ICPcode) { const MCSymbol *const &Sym = entry.first; - const std::vector &Insts = entry.second; + const InstructionListType &Insts = entry.second; if (Sym) dbgs() << Sym->getName() << ":\n"; Offset = BC.printInstructions(dbgs(), Insts.begin(), diff --git a/bolt/lib/Passes/Instrumentation.cpp b/bolt/lib/Passes/Instrumentation.cpp index 920710dcc394..48d16725be33 100644 --- a/bolt/lib/Passes/Instrumentation.cpp +++ b/bolt/lib/Passes/Instrumentation.cpp @@ -170,13 +170,13 @@ void Instrumentation::createLeafNodeDescription(FunctionDescription &FuncDesc, FuncDesc.LeafNodes.emplace_back(IN); } -std::vector +InstructionListType Instrumentation::createInstrumentationSnippet(BinaryContext &BC, bool IsLeaf) { auto L = BC.scopeLock(); MCSymbol *Label; Label = BC.Ctx->createNamedTempSymbol("InstrEntry"); Summary->Counters.emplace_back(Label); - std::vector CounterInstrs; + InstructionListType CounterInstrs; BC.MIB->createInstrIncMemory(CounterInstrs, Label, &*BC.Ctx, IsLeaf); return CounterInstrs; } @@ -184,17 +184,15 @@ Instrumentation::createInstrumentationSnippet(BinaryContext &BC, bool IsLeaf) { namespace { // Helper instruction sequence insertion function -BinaryBasicBlock::iterator -insertInstructions(std::vector& Instrs, - BinaryBasicBlock &BB, - BinaryBasicBlock::iterator Iter) { +BinaryBasicBlock::iterator insertInstructions(InstructionListType &Instrs, + BinaryBasicBlock &BB, + BinaryBasicBlock::iterator Iter) { for (MCInst &NewInst : Instrs) { Iter = BB.insertInstruction(Iter, NewInst); ++Iter; } return Iter; } - } void Instrumentation::instrumentLeafNode(BinaryBasicBlock &BB, @@ -203,7 +201,7 @@ void Instrumentation::instrumentLeafNode(BinaryBasicBlock &BB, FunctionDescription &FuncDesc, uint32_t Node) { createLeafNodeDescription(FuncDesc, Node); - std::vector CounterInstrs = createInstrumentationSnippet( + InstructionListType CounterInstrs = createInstrumentationSnippet( BB.getFunction()->getBinaryContext(), IsLeaf); insertInstructions(CounterInstrs, BB, Iter); } @@ -218,7 +216,7 @@ void Instrumentation::instrumentIndirectTarget(BinaryBasicBlock &BB, BinaryContext &BC = FromFunction.getBinaryContext(); bool IsTailCall = BC.MIB->isTailCall(*Iter); - std::vector CounterInstrs = BC.MIB->createInstrumentedIndirectCall( + InstructionListType CounterInstrs = BC.MIB->createInstrumentedIndirectCall( *Iter, IsTailCall, IsTailCall ? IndTailCallHandlerExitBBFunction->getSymbol() : IndCallHandlerExitBBFunction->getSymbol(), @@ -249,8 +247,8 @@ bool Instrumentation::instrumentOneTarget( return false; } - std::vector CounterInstrs = - createInstrumentationSnippet(FromFunction.getBinaryContext(), IsLeaf); + InstructionListType CounterInstrs = + createInstrumentationSnippet(FromFunction.getBinaryContext(), IsLeaf); BinaryContext &BC = FromFunction.getBinaryContext(); const MCInst &Inst = *Iter; @@ -596,7 +594,7 @@ void Instrumentation::runOnFunctions(BinaryContext &BC) { void Instrumentation::createAuxiliaryFunctions(BinaryContext &BC) { auto createSimpleFunction = - [&](StringRef Title, std::vector Instrs) -> BinaryFunction * { + [&](StringRef Title, InstructionListType Instrs) -> BinaryFunction * { BinaryFunction *Func = BC.createInjectedBinaryFunction(std::string(Title)); std::vector> BBs; diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp index cd66fb7d6ba5..d6f126e1ea1d 100644 --- a/bolt/lib/Passes/LongJmp.cpp +++ b/bolt/lib/Passes/LongJmp.cpp @@ -39,7 +39,7 @@ constexpr unsigned ColdFragAlign = 16; void relaxStubToShortJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) { const BinaryContext &BC = StubBB.getFunction()->getBinaryContext(); - std::vector Seq; + InstructionListType Seq; BC.MIB->createShortJmp(Seq, Tgt, BC.Ctx.get()); StubBB.clear(); StubBB.addInstructions(Seq.begin(), Seq.end()); @@ -47,7 +47,7 @@ void relaxStubToShortJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) { void relaxStubToLongJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) { const BinaryContext &BC = StubBB.getFunction()->getBinaryContext(); - std::vector Seq; + InstructionListType Seq; BC.MIB->createLongJmp(Seq, Tgt, BC.Ctx.get()); StubBB.clear(); StubBB.addInstructions(Seq.begin(), Seq.end()); diff --git a/bolt/lib/Passes/PatchEntries.cpp b/bolt/lib/Passes/PatchEntries.cpp index a0f64065b5bb..21e4c09bda06 100644 --- a/bolt/lib/Passes/PatchEntries.cpp +++ b/bolt/lib/Passes/PatchEntries.cpp @@ -57,7 +57,7 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) { // Calculate the size of the patch. static size_t PatchSize = 0; if (!PatchSize) { - std::vector Seq; + InstructionListType Seq; BC.MIB->createLongTailCall(Seq, BC.Ctx->createTempSymbol(), BC.Ctx.get()); PatchSize = BC.computeCodeSize(Seq.begin(), Seq.end()); } @@ -124,7 +124,7 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) { PatchFunction->setFileOffset(Patch.FileOffset); PatchFunction->setOriginSection(Patch.Section); - std::vector Seq; + InstructionListType Seq; BC.MIB->createLongTailCall(Seq, Patch.Symbol, BC.Ctx.get()); PatchFunction->addBasicBlock(0)->addInstructions(Seq); diff --git a/bolt/lib/Passes/RetpolineInsertion.cpp b/bolt/lib/Passes/RetpolineInsertion.cpp index 09d7da7d09ba..d24a128167ff 100644 --- a/bolt/lib/Passes/RetpolineInsertion.cpp +++ b/bolt/lib/Passes/RetpolineInsertion.cpp @@ -120,7 +120,7 @@ BinaryFunction *createNewRetpoline(BinaryContext &BC, BB1.addInstruction(Lfence); } - std::vector Seq; + InstructionListType Seq; MIB.createShortJmp(Seq, BB1.getLabel(), &Ctx); BB1.addInstructions(Seq.begin(), Seq.end()); @@ -227,10 +227,10 @@ BinaryFunction *RetpolineInsertion::getOrCreateRetpoline( } void createBranchReplacement(BinaryContext &BC, - const IndirectBranchInfo &BrInfo, - bool R11Available, - std::vector &Replacement, - const MCSymbol *RetpolineSymbol) { + const IndirectBranchInfo &BrInfo, + bool R11Available, + InstructionListType &Replacement, + const MCSymbol *RetpolineSymbol) { auto &MIB = *BC.MIB; // Load the branch address in r11 if available if (BrInfo.isMem() && R11Available) { @@ -293,7 +293,7 @@ void RetpolineInsertion::runOnFunctions(BinaryContext &BC) { IndirectBranchInfo BrInfo(Inst, MIB); bool R11Available = false; BinaryFunction *TargetRetpoline; - std::vector Replacement; + InstructionListType Replacement; // Determine if r11 is available before this instruction if (BrInfo.isMem()) { diff --git a/bolt/lib/Passes/ShrinkWrapping.cpp b/bolt/lib/Passes/ShrinkWrapping.cpp index fcf03f000dbb..8574a1e82f20 100644 --- a/bolt/lib/Passes/ShrinkWrapping.cpp +++ b/bolt/lib/Passes/ShrinkWrapping.cpp @@ -1583,13 +1583,13 @@ void ShrinkWrapping::insertUpdatedCFI(unsigned CSR, int SPValPush, auto InsertionIter = InstIter; ++InsertionIter; InAffectedZone = CurZone; - if (InAffectedZone) { - InstIter = --insertCFIsForPushOrPop(*BB, InsertionIter, CSR, true, 0, - SPValPop); - } else { - InstIter = --insertCFIsForPushOrPop(*BB, InsertionIter, CSR, false, 0, - SPValPush); - } + if (InAffectedZone) + InstIter = insertCFIsForPushOrPop(*BB, InsertionIter, CSR, true, 0, + SPValPop); + else + InstIter = insertCFIsForPushOrPop(*BB, InsertionIter, CSR, false, 0, + SPValPush); + --InstIter; } } // Are we at the first basic block or hot-cold split point? @@ -1744,10 +1744,11 @@ BBIterTy ShrinkWrapping::insertCFIsForPushOrPop(BinaryBasicBlock &BB, updateCFIInstOffset(*Pos++, NewOffset); } if (HasDeletedOffsetCFIs[Reg]) { - Pos = ++BF.addCFIInstruction( + Pos = BF.addCFIInstruction( &BB, Pos, MCCFIInstruction::createOffset( nullptr, BC.MRI->getDwarfRegNum(Reg, false), NewOffset)); + ++Pos; } } else { for (uint32_t Idx : DeletedPopCFIs[Reg]) { @@ -1755,10 +1756,11 @@ BBIterTy ShrinkWrapping::insertCFIsForPushOrPop(BinaryBasicBlock &BB, updateCFIInstOffset(*Pos++, NewOffset); } if (HasDeletedOffsetCFIs[Reg]) { - Pos = ++BF.addCFIInstruction( + Pos = BF.addCFIInstruction( &BB, Pos, MCCFIInstruction::createSameValue( nullptr, BC.MRI->getDwarfRegNum(Reg, false))); + ++Pos; } } return Pos; @@ -1803,7 +1805,9 @@ BBIterTy ShrinkWrapping::processInsertion(BBIterTy InsertionPoint, dbgs() << "the following inst: "; NewInst.dump(); }); - return ++CurBB->insertInstruction(InsertionPoint, std::move(NewInst)); + BBIterTy Iter = + CurBB->insertInstruction(InsertionPoint, std::move(NewInst)); + return ++Iter; } CurBB->addInstruction(std::move(NewInst)); LLVM_DEBUG(dbgs() << "Adding to BB!\n"); diff --git a/bolt/lib/Passes/ValidateInternalCalls.cpp b/bolt/lib/Passes/ValidateInternalCalls.cpp index c9e40fce20a1..f0ebc06e9765 100644 --- a/bolt/lib/Passes/ValidateInternalCalls.cpp +++ b/bolt/lib/Passes/ValidateInternalCalls.cpp @@ -99,7 +99,7 @@ bool ValidateInternalCalls::fixCFGForPIC(BinaryFunction &Function) const { continue; BC.MIB->addAnnotation(Inst, getProcessedICTag(), 0U); - std::vector MovedInsts = BB.splitInstructions(&Inst); + InstructionListType MovedInsts = BB.splitInstructions(&Inst); if (!MovedInsts.empty()) { // Split this block at the call instruction. Create an unreachable // block. diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp index ef5519b8353b..fcc4086547bc 100644 --- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp +++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp @@ -816,7 +816,7 @@ public: return true; } - void createLongTailCall(std::vector &Seq, const MCSymbol *Target, + void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target, MCContext *Ctx) override { createShortJmp(Seq, Target, Ctx, /*IsTailCall*/ true); } @@ -911,7 +911,7 @@ public: return true; } - void createLongJmp(std::vector &Seq, const MCSymbol *Target, + void createLongJmp(InstructionListType &Seq, const MCSymbol *Target, MCContext *Ctx, bool IsTailCall) override { // ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call // Standard for the ARM 64-bit Architecture (AArch64)". @@ -968,7 +968,7 @@ public: Seq.emplace_back(Inst); } - void createShortJmp(std::vector &Seq, const MCSymbol *Target, + void createShortJmp(InstructionListType &Seq, const MCSymbol *Target, MCContext *Ctx, bool IsTailCall) override { // ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call // Standard for the ARM 64-bit Architecture (AArch64)". @@ -977,7 +977,7 @@ public: // add ip0, ip0, imm // br ip0 MCPhysReg Reg = AArch64::X16; - std::vector Insts = materializeAddress(Target, Ctx, Reg); + InstructionListType Insts = materializeAddress(Target, Ctx, Reg); Insts.emplace_back(); MCInst &Inst = Insts.back(); Inst.clear(); @@ -1097,11 +1097,11 @@ public: return true; } - std::vector materializeAddress(const MCSymbol *Target, MCContext *Ctx, + InstructionListType materializeAddress(const MCSymbol *Target, MCContext *Ctx, MCPhysReg RegName, int64_t Addend = 0) const override { // Get page-aligned address and add page offset - std::vector Insts(2); + InstructionListType Insts(2); Insts[0].setOpcode(AArch64::ADRP); Insts[0].clear(); Insts[0].addOperand(MCOperand::createReg(RegName)); diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp index 2042100cc04b..d588155fc014 100644 --- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp +++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp @@ -2874,8 +2874,8 @@ public: return true; } - std::vector createInlineMemcpy(bool ReturnEnd) const override { - std::vector Code; + InstructionListType createInlineMemcpy(bool ReturnEnd) const override { + InstructionListType Code; if (ReturnEnd) { Code.emplace_back(MCInstBuilder(X86::LEA64r) .addReg(X86::RAX) @@ -2897,8 +2897,8 @@ public: return Code; } - std::vector createOneByteMemcpy() const override { - std::vector Code; + InstructionListType createOneByteMemcpy() const override { + InstructionListType Code; Code.emplace_back(MCInstBuilder(X86::MOV8rm) .addReg(X86::CL) .addReg(X86::RSI) @@ -2919,10 +2919,10 @@ public: return Code; } - std::vector - createCmpJE(MCPhysReg RegNo, int64_t Imm, const MCSymbol *Target, - MCContext *Ctx) const override { - std::vector Code; + InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm, + const MCSymbol *Target, + MCContext *Ctx) const override { + InstructionListType Code; Code.emplace_back(MCInstBuilder(X86::CMP64ri8) .addReg(RegNo) .addImm(Imm)); @@ -3280,7 +3280,7 @@ public: return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true); } - void createLongTailCall(std::vector &Seq, const MCSymbol *Target, + void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target, MCContext *Ctx) override { Seq.clear(); Seq.emplace_back(); @@ -3394,7 +3394,7 @@ public: return true; } - void createShortJmp(std::vector &Seq, const MCSymbol *Target, + void createShortJmp(InstructionListType &Seq, const MCSymbol *Target, MCContext *Ctx, bool IsTailCall) override { Seq.clear(); MCInst Inst; @@ -3541,7 +3541,7 @@ public: Inst.clear(); } - void createInstrIncMemory(std::vector &Instrs, const MCSymbol *Target, + void createInstrIncMemory(InstructionListType &Instrs, const MCSymbol *Target, MCContext *Ctx, bool IsLeaf) const override { unsigned int I = 0; @@ -3596,10 +3596,11 @@ public: Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg } - std::vector - createInstrumentedIndirectCall(const MCInst &CallInst, bool TailCall, - MCSymbol *HandlerFuncAddr, int CallSiteID, - MCContext *Ctx) override { + InstructionListType createInstrumentedIndirectCall(const MCInst &CallInst, + bool TailCall, + MCSymbol *HandlerFuncAddr, + int CallSiteID, + MCContext *Ctx) override { // Check if the target address expression used in the original indirect call // uses the stack pointer, which we are going to clobber. static BitVector SPAliases(getAliases(X86::RSP)); @@ -3614,7 +3615,7 @@ public: } } - std::vector Insts; + InstructionListType Insts; MCPhysReg TempReg = getIntArgRegister(0); // Code sequence used to enter indirect call instrumentation helper: // push %rdi @@ -3658,7 +3659,7 @@ public: return Insts; } - std::vector createInstrumentedIndCallHandlerExitBB() const override { + InstructionListType createInstrumentedIndCallHandlerExitBB() const override { const MCPhysReg TempReg = getIntArgRegister(0); // We just need to undo the sequence created for every ind call in // instrumentIndirectTarget(), which can be accomplished minimally with: @@ -3667,7 +3668,7 @@ public: // add $16, %rsp // xchg (%rsp), %rdi // jmp *-8(%rsp) - std::vector Insts(5); + InstructionListType Insts(5); createPopFlags(Insts[0], 8); createPopRegister(Insts[1], TempReg, 8); createStackPointerDecrement(Insts[2], 16, /*NoFlagsClobber=*/false); @@ -3676,7 +3677,7 @@ public: return Insts; } - std::vector + InstructionListType createInstrumentedIndTailCallHandlerExitBB() const override { const MCPhysReg TempReg = getIntArgRegister(0); // Same thing as above, but for tail calls @@ -3684,7 +3685,7 @@ public: // add $16, %rsp // pop %rdi // jmp *-16(%rsp) - std::vector Insts(4); + InstructionListType Insts(4); createPopFlags(Insts[0], 8); createStackPointerDecrement(Insts[1], 16, /*NoFlagsClobber=*/false); createPopRegister(Insts[2], TempReg, 8); @@ -3692,7 +3693,7 @@ public: return Insts; } - std::vector + InstructionListType createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline, const MCSymbol *IndCallHandler, MCContext *Ctx) override { @@ -3705,12 +3706,12 @@ public: // je IndCallHandler // callq *%rdi // jmpq IndCallHandler - std::vector Insts; + InstructionListType Insts; Insts.emplace_back(); createPushFlags(Insts.back(), 8); Insts.emplace_back(); createMove(Insts.back(), InstrTrampoline, TempReg, Ctx); - std::vector cmpJmp = createCmpJE(TempReg, 0, IndCallHandler, Ctx); + InstructionListType cmpJmp = createCmpJE(TempReg, 0, IndCallHandler, Ctx); Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end()); Insts.emplace_back(); Insts.back().setOpcode(X86::CALL64r); @@ -3720,48 +3721,48 @@ public: return Insts; } - std::vector createNumCountersGetter(MCContext *Ctx) const override { - std::vector Insts(2); + InstructionListType createNumCountersGetter(MCContext *Ctx) const override { + InstructionListType Insts(2); MCSymbol *NumLocs = Ctx->getOrCreateSymbol("__bolt_num_counters"); createMove(Insts[0], NumLocs, X86::EAX, Ctx); createReturn(Insts[1]); return Insts; } - std::vector + InstructionListType createInstrLocationsGetter(MCContext *Ctx) const override { - std::vector Insts(2); + InstructionListType Insts(2); MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_locations"); createLea(Insts[0], Locs, X86::EAX, Ctx); createReturn(Insts[1]); return Insts; } - std::vector createInstrTablesGetter(MCContext *Ctx) const override { - std::vector Insts(2); + InstructionListType createInstrTablesGetter(MCContext *Ctx) const override { + InstructionListType Insts(2); MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_tables"); createLea(Insts[0], Locs, X86::EAX, Ctx); createReturn(Insts[1]); return Insts; } - std::vector createInstrNumFuncsGetter(MCContext *Ctx) const override { - std::vector Insts(2); + InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override { + InstructionListType Insts(2); MCSymbol *NumFuncs = Ctx->getOrCreateSymbol("__bolt_instr_num_funcs"); createMove(Insts[0], NumFuncs, X86::EAX, Ctx); createReturn(Insts[1]); return Insts; } - std::vector createSymbolTrampoline(const MCSymbol *TgtSym, + InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym, MCContext *Ctx) const override { - std::vector Insts(1); + InstructionListType Insts(1); createUncondBranch(Insts[0], TgtSym, Ctx); return Insts; } - std::vector createDummyReturnFunction(MCContext *Ctx) const override { - std::vector Insts(1); + InstructionListType createDummyReturnFunction(MCContext *Ctx) const override { + InstructionListType Insts(1); createReturn(Insts[0]); return Insts; } @@ -3779,11 +3780,11 @@ public: BlocksVectorTy Results; // Label for the current code block. - MCSymbol* NextTarget = nullptr; + MCSymbol *NextTarget = nullptr; // The join block which contains all the instructions following CallInst. // MergeBlock remains null if CallInst is a tail call. - MCSymbol* MergeBlock = nullptr; + MCSymbol *MergeBlock = nullptr; unsigned FuncAddrReg = X86::R10; @@ -3810,7 +3811,7 @@ public: return Results; } - const auto jumpToMergeBlock = [&](std::vector &NewCall) { + const auto jumpToMergeBlock = [&](InstructionListType &NewCall) { assert(MergeBlock); NewCall.push_back(CallInst); MCInst &Merge = NewCall.back(); @@ -3819,8 +3820,8 @@ public: }; for (unsigned int i = 0; i < Targets.size(); ++i) { - Results.emplace_back(NextTarget, std::vector()); - std::vector* NewCall = &Results.back().second; + Results.emplace_back(NextTarget, InstructionListType()); + InstructionListType *NewCall = &Results.back().second; if (MinimizeCodeSize && !LoadElim) { // Load the call target into FuncAddrReg. @@ -3942,7 +3943,7 @@ public: // Call specific target directly. Results.emplace_back(Ctx->createNamedTempSymbol(), - std::vector()); + InstructionListType()); NewCall = &Results.back().second; NewCall->push_back(CallInst); MCInst &CallOrJmp = NewCall->back(); @@ -3999,8 +4000,8 @@ public: } // Cold call block. - Results.emplace_back(NextTarget, std::vector()); - std::vector &NewCall = Results.back().second; + Results.emplace_back(NextTarget, InstructionListType()); + InstructionListType &NewCall = Results.back().second; for (const MCInst *Inst : MethodFetchInsns) { if (Inst != &CallInst) NewCall.push_back(*Inst); @@ -4012,7 +4013,7 @@ public: jumpToMergeBlock(NewCall); // Record merge block - Results.emplace_back(MergeBlock, std::vector()); + Results.emplace_back(MergeBlock, InstructionListType()); } return Results; @@ -4032,11 +4033,11 @@ public: BlocksVectorTy Results; // Label for the current code block. - MCSymbol* NextTarget = nullptr; + MCSymbol *NextTarget = nullptr; for (unsigned int i = 0; i < Targets.size(); ++i) { - Results.emplace_back(NextTarget, std::vector()); - std::vector* CurBB = &Results.back().second; + Results.emplace_back(NextTarget, InstructionListType()); + InstructionListType *CurBB = &Results.back().second; // Compare current index to a specific index. CurBB->emplace_back(MCInst()); @@ -4069,8 +4070,8 @@ public: } // Cold call block. - Results.emplace_back(NextTarget, std::vector()); - std::vector &CurBB = Results.back().second; + Results.emplace_back(NextTarget, InstructionListType()); + InstructionListType &CurBB = Results.back().second; for (const MCInst *Inst : TargetFetchInsns) { if (Inst != &IJmpInst) CurBB.push_back(*Inst);