diff --git a/bolt/BinaryBasicBlock.cpp b/bolt/BinaryBasicBlock.cpp index f64a1b4ee338..05d0d6c2f321 100644 --- a/bolt/BinaryBasicBlock.cpp +++ b/bolt/BinaryBasicBlock.cpp @@ -66,7 +66,7 @@ bool BinaryBasicBlock::validateSuccessorInvariants() { // Note: for now we assume that successors do not reference labels from // any overlapping jump tables. We only look at the entries for the jump // table that is referenced at the last instruction. - const auto Range = JT->getEntriesForAddress(BC.MIA->getJumpTable(*Inst)); + const auto Range = JT->getEntriesForAddress(BC.MIB->getJumpTable(*Inst)); const std::vector Entries(&JT->Entries[Range.first], &JT->Entries[Range.second]); std::set UniqueSyms(Entries.begin(), Entries.end()); @@ -108,7 +108,7 @@ bool BinaryBasicBlock::validateSuccessorInvariants() { break; case 1: { const bool HasCondBlock = CondBranch && - Function->getBasicBlockForLabel(BC.MIA->getTargetSymbol(*CondBranch)); + Function->getBasicBlockForLabel(BC.MIB->getTargetSymbol(*CondBranch)); Valid = !CondBranch || !HasCondBlock; break; } @@ -128,7 +128,7 @@ bool BinaryBasicBlock::validateSuccessorInvariants() { << getName() << "\n"; if (JT) { errs() << "Jump Table instruction addr = 0x" - << Twine::utohexstr(BC.MIA->getJumpTable(*Inst)) << "\n"; + << Twine::utohexstr(BC.MIB->getJumpTable(*Inst)) << "\n"; JT->print(errs()); } getFunction()->dump(); @@ -188,7 +188,7 @@ int32_t BinaryBasicBlock::getCFIStateAtInstr(const MCInst *Instr) const { InstrSeen = (&*RII == Instr); continue; } - if (Function->getBinaryContext().MIA->isCFI(*RII)) { + if (Function->getBinaryContext().MIB->isCFI(*RII)) { LastCFI = &*RII; break; } @@ -322,8 +322,8 @@ bool BinaryBasicBlock::analyzeBranch(const MCSymbol *&TBB, const MCSymbol *&FBB, MCInst *&CondBranch, MCInst *&UncondBranch) { - auto &MIA = Function->getBinaryContext().MIA; - return MIA->analyzeBranch(Instructions.begin(), + auto &MIB = Function->getBinaryContext().MIB; + return MIB->analyzeBranch(Instructions.begin(), Instructions.end(), TBB, FBB, @@ -343,7 +343,7 @@ MCInst *BinaryBasicBlock::getTerminatorBefore(MCInst *Pos) { ++Itr; continue; } - if (BC.MIA->isTerminator(*Itr)) + if (BC.MIB->isTerminator(*Itr)) FirstTerminator = &*Itr; ++Itr; } @@ -356,7 +356,7 @@ bool BinaryBasicBlock::hasTerminatorAfter(MCInst *Pos) { while (Itr != rend()) { if (&*Itr == Pos) return false; - if (BC.MIA->isTerminator(*Itr)) + if (BC.MIB->isTerminator(*Itr)) return true; ++Itr; } @@ -376,14 +376,14 @@ void BinaryBasicBlock::addBranchInstruction(const BinaryBasicBlock *Successor) { assert(isSuccessor(Successor)); auto &BC = Function->getBinaryContext(); MCInst NewInst; - BC.MIA->createUncondBranch(NewInst, Successor->getLabel(), BC.Ctx.get()); + BC.MIB->createUncondBranch(NewInst, Successor->getLabel(), BC.Ctx.get()); Instructions.emplace_back(std::move(NewInst)); } void BinaryBasicBlock::addTailCallInstruction(const MCSymbol *Target) { auto &BC = Function->getBinaryContext(); MCInst NewInst; - BC.MIA->createTailCall(NewInst, Target, BC.Ctx.get()); + BC.MIB->createTailCall(NewInst, Target, BC.Ctx.get()); Instructions.emplace_back(std::move(NewInst)); } @@ -391,7 +391,7 @@ uint32_t BinaryBasicBlock::getNumCalls() const { uint32_t N{0}; auto &BC = Function->getBinaryContext(); for (auto &Instr : Instructions) { - if (BC.MIA->isCall(Instr)) + if (BC.MIB->isCall(Instr)) ++N; } return N; diff --git a/bolt/BinaryContext.cpp b/bolt/BinaryContext.cpp index 09ebe6b7d58c..897e05b37953 100644 --- a/bolt/BinaryContext.cpp +++ b/bolt/BinaryContext.cpp @@ -658,12 +658,12 @@ void BinaryContext::printInstruction(raw_ostream &OS, bool PrintMCInst, bool PrintMemData, bool PrintRelocations) const { - if (MIA->isEHLabel(Instruction)) { - OS << " EH_LABEL: " << *MIA->getTargetSymbol(Instruction) << '\n'; + if (MIB->isEHLabel(Instruction)) { + OS << " EH_LABEL: " << *MIB->getTargetSymbol(Instruction) << '\n'; return; } OS << format(" %08" PRIx64 ": ", Offset); - if (MIA->isCFI(Instruction)) { + if (MIB->isCFI(Instruction)) { uint32_t Offset = Instruction.getOperand(0).getImm(); OS << "\t!CFI\t$" << Offset << "\t; "; if (Function) @@ -672,31 +672,31 @@ void BinaryContext::printInstruction(raw_ostream &OS, return; } InstPrinter->printInst(&Instruction, OS, "", *STI); - if (MIA->isCall(Instruction)) { - if (MIA->isTailCall(Instruction)) + if (MIB->isCall(Instruction)) { + if (MIB->isTailCall(Instruction)) OS << " # TAILCALL "; - if (MIA->isInvoke(Instruction)) { + if (MIB->isInvoke(Instruction)) { const MCSymbol *LP; uint64_t Action; - std::tie(LP, Action) = MIA->getEHInfo(Instruction); + std::tie(LP, Action) = MIB->getEHInfo(Instruction); OS << " # handler: "; if (LP) OS << *LP; else OS << '0'; OS << "; action: " << Action; - auto GnuArgsSize = MIA->getGnuArgsSize(Instruction); + auto GnuArgsSize = MIB->getGnuArgsSize(Instruction); if (GnuArgsSize >= 0) OS << "; GNU_args_size = " << GnuArgsSize; } } - if (MIA->isIndirectBranch(Instruction)) { - if (auto JTAddress = MIA->getJumpTable(Instruction)) { + if (MIB->isIndirectBranch(Instruction)) { + if (auto JTAddress = MIB->getJumpTable(Instruction)) { OS << " # JUMPTABLE @0x" << Twine::utohexstr(JTAddress); } } - MIA->forEachAnnotation( + MIB->forEachAnnotation( Instruction, [&OS](const MCAnnotation *Annotation) { OS << " # " << Annotation->getName() << ": "; @@ -726,7 +726,7 @@ void BinaryContext::printInstruction(raw_ostream &OS, if ((opts::PrintMemData || PrintMemData) && Function) { const auto *MD = Function->getMemData(); const auto MemDataOffset = - MIA->tryGetAnnotationAs(Instruction, "MemDataOffset"); + MIB->tryGetAnnotationAs(Instruction, "MemDataOffset"); if (MD && MemDataOffset) { bool DidPrint = false; for (auto &MI : MD->getMemInfoRange(MemDataOffset.get())) { diff --git a/bolt/BinaryContext.h b/bolt/BinaryContext.h index 5cb67ad4fe16..8e4dcd2b15f3 100644 --- a/bolt/BinaryContext.h +++ b/bolt/BinaryContext.h @@ -17,6 +17,7 @@ #include "BinaryData.h" #include "BinarySection.h" #include "DebugData.h" +#include "MCPlusBuilder.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/Triple.h" #include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h" @@ -222,6 +223,8 @@ public: std::unique_ptr MIA; + std::unique_ptr MIB; + std::unique_ptr MRI; std::unique_ptr DisAsm; @@ -270,6 +273,7 @@ public: std::unique_ptr STI, std::unique_ptr InstPrinter, std::unique_ptr MIA, + std::unique_ptr MIB, std::unique_ptr MRI, std::unique_ptr DisAsm, DataReader &DR) : @@ -285,6 +289,7 @@ public: STI(std::move(STI)), InstPrinter(std::move(InstPrinter)), MIA(std::move(MIA)), + MIB(std::move(MIB)), MRI(std::move(MRI)), DisAsm(std::move(DisAsm)), DR(DR) { @@ -612,7 +617,7 @@ public: SmallString<256> Code; SmallVector Fixups; raw_svector_ostream VecOS(Code); - if (MIA->isCFI(*Beg) || MIA->isEHLabel(*Beg)) { + if (MIB->isCFI(*Beg) || MIB->isEHLabel(*Beg)) { ++Beg; continue; } @@ -637,8 +642,8 @@ public: /// Return true if instruction \p Inst requires an offset for further /// processing (e.g. assigning a profile). bool keepOffsetForInstruction(const MCInst &Inst) const { - if (MIA->isCall(Inst) || MIA->isBranch(Inst) || MIA->isReturn(Inst) || - MIA->isPrefix(Inst) || MIA->isIndirectBranch(Inst)) { + if (MIB->isCall(Inst) || MIB->isBranch(Inst) || MIB->isReturn(Inst) || + MIB->isPrefix(Inst) || MIB->isIndirectBranch(Inst)) { return true; } return false; diff --git a/bolt/BinaryFunction.cpp b/bolt/BinaryFunction.cpp index a2a68b2e3ab5..169d9765dd8e 100644 --- a/bolt/BinaryFunction.cpp +++ b/bolt/BinaryFunction.cpp @@ -13,6 +13,7 @@ #include "BinaryBasicBlock.h" #include "BinaryFunction.h" #include "DataReader.h" +#include "MCPlusBuilder.h" #include "llvm/ADT/edit_distance.h" #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" @@ -668,7 +669,7 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, } } - auto Type = BC.MIA->analyzeIndirectBranch(Instruction, + auto Type = BC.MIB->analyzeIndirectBranch(Instruction, Begin, End, PtrSize, @@ -686,7 +687,7 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, IndexRegNum = 0; if (BC.TheTriple->getArch() == llvm::Triple::aarch64) { - const auto *Sym = BC.MIA->getTargetSymbol(*PCRelBaseInstr, 1); + const auto *Sym = BC.MIB->getTargetSymbol(*PCRelBaseInstr, 1); assert (Sym && "Symbol extraction failed"); if (auto *BD = BC.getBinaryDataByName(Sym->getName())) { PCRelAddr = BD->getAddress(); @@ -710,7 +711,7 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, // function (for example, if the indirect jump lives in the last basic // block of the function, it will create a reference to the next function). // This replaces a symbol reference with an immediate. - BC.MIA->replaceMemOperandDisp(*PCRelBaseInstr, + BC.MIB->replaceMemOperandDisp(*PCRelBaseInstr, MCOperand::createImm(PCRelAddr - InstrAddr)); // FIXME: Disable full jump table processing for AArch64 until we have a // proper way of determining the jump table limits. @@ -722,7 +723,7 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, if (DispExpr) { const MCSymbol *TargetSym; uint64_t TargetOffset; - std::tie(TargetSym, TargetOffset) = BC.MIA->getTargetSymbolInfo(DispExpr); + std::tie(TargetSym, TargetOffset) = BC.MIB->getTargetSymbolInfo(DispExpr); auto *BD = BC.getBinaryDataByName(TargetSym->getName()); assert(BD && "global symbol needs a value"); ArrayStart = BD->getAddress() + TargetOffset; @@ -773,9 +774,9 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, LI = Result.first; } - BC.MIA->replaceMemOperandDisp(const_cast(*MemLocInstr), + BC.MIB->replaceMemOperandDisp(const_cast(*MemLocInstr), LI->second, BC.Ctx.get()); - BC.MIA->setJumpTable(BC.Ctx.get(), Instruction, ArrayStart, IndexRegNum); + BC.MIB->setJumpTable(BC.Ctx.get(), Instruction, ArrayStart, IndexRegNum); JTSites.emplace_back(Offset, ArrayStart); @@ -870,9 +871,9 @@ IndirectBranchType BinaryFunction::processIndirectBranch(MCInst &Instruction, << " in function " << *this << " with " << JTOffsetCandidates.size() << " entries.\n"); JumpTables.emplace(ArrayStart, JT.release()); - BC.MIA->replaceMemOperandDisp(const_cast(*MemLocInstr), + BC.MIB->replaceMemOperandDisp(const_cast(*MemLocInstr), JTStartLabel, BC.Ctx.get()); - BC.MIA->setJumpTable(BC.Ctx.get(), Instruction, ArrayStart, IndexRegNum); + BC.MIB->setJumpTable(BC.Ctx.get(), Instruction, ArrayStart, IndexRegNum); JTSites.emplace_back(Offset, ArrayStart); @@ -919,7 +920,7 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { "function size does not match raw data size"); auto &Ctx = BC.Ctx; - auto &MIA = BC.MIA; + auto &MIB = BC.MIB; DWARFUnitLineTable ULT = getDWARFUnitLineTable(); @@ -935,7 +936,7 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { uint64_t TargetAddress{0}; uint64_t TargetOffset{0}; MCSymbol *TargetSymbol{nullptr}; - if (!MIA->evaluateMemOperandTarget(Instruction, TargetAddress, Address, + if (!MIB->evaluateMemOperandTarget(Instruction, TargetAddress, Address, Size)) { errs() << "BOLT-ERROR: PC-relative operand can't be evaluated:\n"; BC.InstPrinter->printInst(&Instruction, errs(), "", *BC.STI); @@ -985,7 +986,7 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { // without its supporting relocation. if (!TargetSymbol && Section && Section->isText() && (BC.TheTriple->getArch() != llvm::Triple::aarch64 || - !BC.MIA->isADRP(Instruction))) { + !BC.MIB->isADRP(Instruction))) { if (containsAddress(TargetAddress, /*UseMaxSize=*/ BC.TheTriple->getArch() == llvm::Triple::aarch64)) { if (TargetAddress != getAddress()) { @@ -1022,8 +1023,8 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { auto *Offset = MCConstantExpr::create(TargetOffset, *BC.Ctx); Expr = MCBinaryExpr::createAdd(Expr, Offset, *BC.Ctx); } - MIA->replaceMemOperandDisp( - Instruction, MCOperand::createExpr(BC.MIA->getTargetExprFor( + MIB->replaceMemOperandDisp( + Instruction, MCOperand::createExpr(BC.MIB->getTargetExprFor( Instruction, Expr, *BC.Ctx, 0))); @@ -1080,7 +1081,7 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { } // Cannot process functions with AVX-512 instructions. - if (MIA->hasEVEXEncoding(Instruction)) { + if (MIB->hasEVEXEncoding(Instruction)) { if (opts::Verbosity >= 1) { errs() << "BOLT-WARNING: function " << *this << " uses instruction" " encoded with EVEX (AVX-512) at offset 0x" @@ -1113,7 +1114,7 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { << " for instruction at offset 0x" << Twine::utohexstr(Offset) << '\n'); int64_t Value = Relocation.Value; - const auto Result = BC.MIA->replaceImmWithSymbol(Instruction, + const auto Result = BC.MIB->replaceImmWithSymbol(Instruction, Relocation.Symbol, Relocation.Addend, Ctx.get(), @@ -1135,11 +1136,11 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { } // Convert instruction to a shorter version that could be relaxed if needed. - MIA->shortenInstruction(Instruction); + MIB->shortenInstruction(Instruction); - if (MIA->isBranch(Instruction) || MIA->isCall(Instruction)) { + if (MIB->isBranch(Instruction) || MIB->isCall(Instruction)) { uint64_t TargetAddress = 0; - if (MIA->evaluateBranch(Instruction, + if (MIB->evaluateBranch(Instruction, AbsoluteInstrAddr, Size, TargetAddress)) { @@ -1148,8 +1149,8 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { // // If the target *is* the function address it could be either a branch // or a recursive call. - bool IsCall = MIA->isCall(Instruction); - const bool IsCondBranch = MIA->isConditionalBranch(Instruction); + bool IsCall = MIB->isCall(Instruction); + const bool IsCondBranch = MIB->isConditionalBranch(Instruction); MCSymbol *TargetSymbol = nullptr; if (IsCall && containsAddress(TargetAddress)) { @@ -1177,7 +1178,7 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { << Twine::utohexstr(AbsoluteInstrAddr) << " in function " << *this << " : replacing with nop.\n"); - BC.MIA->createNoop(Instruction); + BC.MIB->createNoop(Instruction); if (IsCondBranch) { // Register branch offset for profile validation. IgnoredBranches.emplace_back(Offset, Offset + Size); @@ -1193,14 +1194,14 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { << ". Code size will be increased.\n"; } - assert(!MIA->isTailCall(Instruction) && + assert(!MIB->isTailCall(Instruction) && "synthetic tail call instruction found"); // This is a call regardless of the opcode. // Assign proper opcode for tail calls, so that they could be // treated as calls. if (!IsCall) { - if (!MIA->convertJmpToTailCall(Instruction, BC.Ctx.get())) { + if (!MIB->convertJmpToTailCall(Instruction, BC.Ctx.get())) { assert(IsCondBranch && "unknown tail call instruction"); if (opts::Verbosity >= 2) { errs() << "BOLT-WARNING: conditional tail call detected in " @@ -1261,16 +1262,16 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { // Add taken branch info. TakenBranches.emplace_back(Offset, TargetAddress - getAddress()); } - BC.MIA->replaceBranchTarget(Instruction, TargetSymbol, &*Ctx); + BC.MIB->replaceBranchTarget(Instruction, TargetSymbol, &*Ctx); // Mark CTC. if (IsCondBranch && IsCall) { - MIA->setConditionalTailCall(Instruction, TargetAddress); + MIB->setConditionalTailCall(Instruction, TargetAddress); } } else { // Could not evaluate branch. Should be an indirect call or an // indirect branch. Bail out on the latter case. - if (MIA->isIndirectBranch(Instruction)) { + if (MIB->isIndirectBranch(Instruction)) { auto Result = processIndirectBranch(Instruction, Size, Offset); switch (Result) { default: @@ -1278,7 +1279,7 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { case IndirectBranchType::POSSIBLE_TAIL_CALL: { auto Result = - MIA->convertJmpToTailCall(Instruction, BC.Ctx.get()); + MIB->convertJmpToTailCall(Instruction, BC.Ctx.get()); (void)Result; assert(Result); } @@ -1295,7 +1296,7 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { }; } // Indirect call. We only need to fix it if the operand is RIP-relative - if (IsSimple && MIA->hasPCRelOperand(Instruction)) { + if (IsSimple && MIB->hasPCRelOperand(Instruction)) { if (!handlePCRelOperand(Instruction, AbsoluteInstrAddr, Size)) { errs() << "BOLT-ERROR: cannot handle PC-relative operand at 0x" << Twine::utohexstr(AbsoluteInstrAddr) @@ -1307,7 +1308,7 @@ void BinaryFunction::disassemble(ArrayRef FunctionData) { } } } else { - if (MIA->hasPCRelOperand(Instruction) && !UsedReloc) { + if (MIB->hasPCRelOperand(Instruction) && !UsedReloc) { if (!handlePCRelOperand(Instruction, AbsoluteInstrAddr, Size)) { errs() << "BOLT-ERROR: cannot handle PC-relative operand at 0x" << Twine::utohexstr(AbsoluteInstrAddr) @@ -1327,11 +1328,11 @@ add_instruction: // Record offset of the instruction for profile matching. if (BC.keepOffsetForInstruction(Instruction)) { - MIA->addAnnotation(Ctx.get(), Instruction, "Offset", Offset); + MIB->addAnnotation(Ctx.get(), Instruction, "Offset", Offset); } if (MemData && !emptyRange(MemData->getMemInfoRange(Offset))) { - MIA->addAnnotation(Ctx.get(), Instruction, "MemDataOffset", Offset); + MIB->addAnnotation(Ctx.get(), Instruction, "MemDataOffset", Offset); } addInstruction(Offset, std::move(Instruction)); @@ -1408,19 +1409,19 @@ void BinaryFunction::postProcessJumpTables() { bool BinaryFunction::postProcessIndirectBranches() { for (auto *BB : layout()) { for (auto &Instr : *BB) { - if (!BC.MIA->isIndirectBranch(Instr)) + if (!BC.MIB->isIndirectBranch(Instr)) continue; // If there's an indirect branch in a single-block function - // it must be a tail call. if (layout_size() == 1) { - BC.MIA->convertJmpToTailCall(Instr, BC.Ctx.get()); + BC.MIB->convertJmpToTailCall(Instr, BC.Ctx.get()); return true; } // Validate the tail call or jump table assumptions. - if (BC.MIA->isTailCall(Instr) || BC.MIA->getJumpTable(Instr)) { - if (BC.MIA->getMemoryOperandNo(Instr) != -1) { + if (BC.MIB->isTailCall(Instr) || BC.MIB->getJumpTable(Instr)) { + if (BC.MIB->getMemoryOperandNo(Instr) != -1) { // We have validated memory contents addressed by the jump // instruction already. continue; @@ -1441,7 +1442,7 @@ bool BinaryFunction::postProcessIndirectBranches() { if (PrevInstr == BB->rend()) { if (opts::Verbosity >= 2) { outs() << "BOLT-INFO: rejected potential " - << (BC.MIA->isTailCall(Instr) ? "indirect tail call" + << (BC.MIB->isTailCall(Instr) ? "indirect tail call" : "jump table") << " in function " << *this << " because the jump-on register was not defined in " @@ -1452,9 +1453,9 @@ bool BinaryFunction::postProcessIndirectBranches() { return false; } // In case of PIC jump table we need to do more checks. - if (BC.MIA->isMoveMem2Reg(*PrevInstr)) + if (BC.MIB->isMoveMem2Reg(*PrevInstr)) continue; - assert(BC.MIA->isADD64rr(*PrevInstr) && "add instruction expected"); + assert(BC.MIB->isADD64rr(*PrevInstr) && "add instruction expected"); auto R2 = PrevInstr->getOperand(2).getReg(); // Make sure both regs are set in the same basic block prior to ADD. bool IsR1Set = false; @@ -1478,7 +1479,7 @@ bool BinaryFunction::postProcessIndirectBranches() { // what it is and conservatively reject the function's CFG. bool IsEpilogue = false; for (const auto &Instr : *BB) { - if (BC.MIA->isLeave(Instr) || BC.MIA->isPop(Instr)) { + if (BC.MIB->isLeave(Instr) || BC.MIB->isPop(Instr)) { IsEpilogue = true; break; } @@ -1493,7 +1494,7 @@ bool BinaryFunction::postProcessIndirectBranches() { } return false; } - BC.MIA->convertJmpToTailCall(Instr, BC.Ctx.get()); + BC.MIB->convertJmpToTailCall(Instr, BC.Ctx.get()); } } return true; @@ -1510,12 +1511,12 @@ void BinaryFunction::recomputeLandingPads() { for (auto *BB : BasicBlocks) { std::unordered_set BBLandingPads; for (auto &Instr : *BB) { - if (!BC.MIA->isInvoke(Instr)) + if (!BC.MIB->isInvoke(Instr)) continue; const MCSymbol *LPLabel; uint64_t Action; - std::tie(LPLabel, Action) = BC.MIA->getEHInfo(Instr); + std::tie(LPLabel, Action) = BC.MIB->getEHInfo(Instr); if (!LPLabel) continue; @@ -1532,7 +1533,7 @@ void BinaryFunction::recomputeLandingPads() { bool BinaryFunction::buildCFG() { NamedRegionTimer T("buildcfg", "Build CFG", TimerGroupName, TimerGroupDesc, opts::TimeBuild); - auto &MIA = BC.MIA; + auto &MIB = BC.MIB; if (!isSimple()) { assert(!BC.HasRelocations && @@ -1581,8 +1582,8 @@ bool BinaryFunction::buildCFG() { auto updateOffset = [&](uint64_t Offset) { assert(PrevBB && PrevBB != InsertBB && "invalid previous block"); auto *PrevInstr = PrevBB->getLastNonPseudoInstr(); - if (PrevInstr && !MIA->hasAnnotation(*PrevInstr, "Offset")) - MIA->addAnnotation(BC.Ctx.get(), *PrevInstr, "Offset", Offset); + if (PrevInstr && !MIB->hasAnnotation(*PrevInstr, "Offset")) + MIB->addAnnotation(BC.Ctx.get(), *PrevInstr, "Offset", Offset); }; for (auto I = Instructions.begin(), E = Instructions.end(); I != E; ++I) { @@ -1603,7 +1604,7 @@ bool BinaryFunction::buildCFG() { // Ignore nops. We use nops to derive alignment of the next basic block. // It will not always work, as some blocks are naturally aligned, but // it's just part of heuristic for block alignment. - if (MIA->isNoop(Instr) && !PreserveNops) { + if (MIB->isNoop(Instr) && !PreserveNops) { IsLastInstrNop = true; continue; } @@ -1614,9 +1615,9 @@ bool BinaryFunction::buildCFG() { assert(PrevBB && "no previous basic block for a fall through"); auto *PrevInstr = PrevBB->getLastNonPseudoInstr(); assert(PrevInstr && "no previous instruction for a fall through"); - if (MIA->isUnconditionalBranch(Instr) && - !MIA->isUnconditionalBranch(*PrevInstr) && - !MIA->getConditionalTailCall(*PrevInstr)) { + if (MIB->isUnconditionalBranch(Instr) && + !MIB->isUnconditionalBranch(*PrevInstr) && + !MIB->getConditionalTailCall(*PrevInstr)) { // Temporarily restore inserter basic block. InsertBB = PrevBB; } else { @@ -1631,8 +1632,8 @@ bool BinaryFunction::buildCFG() { addCFIPlaceholders(0, InsertBB); } - const auto IsBlockEnd = MIA->isTerminator(Instr); - IsLastInstrNop = MIA->isNoop(Instr); + const auto IsBlockEnd = MIB->isTerminator(Instr); + IsLastInstrNop = MIB->isNoop(Instr); LastInstrOffset = Offset; InsertBB->addInstruction(std::move(Instr)); @@ -1702,10 +1703,10 @@ bool BinaryFunction::buildCFG() { // // Conditional tail call is a special case since we don't add a taken // branch successor for it. - IsPrevFT = !MIA->isTerminator(*LastInstr) || - MIA->getConditionalTailCall(*LastInstr); + IsPrevFT = !MIB->isTerminator(*LastInstr) || + MIB->getConditionalTailCall(*LastInstr); } else if (BB->succ_size() == 1) { - IsPrevFT = MIA->isConditionalBranch(*LastInstr); + IsPrevFT = MIB->isConditionalBranch(*LastInstr); } else { IsPrevFT = false; } @@ -1784,7 +1785,7 @@ void BinaryFunction::postProcessCFG() { // Remove "Offset" annotations. for (auto *BB : layout()) for (auto &Inst : *BB) - BC.MIA->removeAnnotation(Inst, "Offset"); + BC.MIB->removeAnnotation(Inst, "Offset"); assert((!isSimple() || validateCFG()) && "Invalid CFG detected after post-processing CFG"); @@ -1868,7 +1869,7 @@ void BinaryFunction::removeConditionalTailCalls() { if (!CTCInstr) continue; - auto TargetAddressOrNone = BC.MIA->getConditionalTailCall(*CTCInstr); + auto TargetAddressOrNone = BC.MIB->getConditionalTailCall(*CTCInstr); if (!TargetAddressOrNone) continue; @@ -1879,24 +1880,24 @@ void BinaryFunction::removeConditionalTailCalls() { uint64_t CTCMispredCount = BinaryBasicBlock::COUNT_NO_PROFILE; if (hasValidProfile()) { CTCTakenCount = - BC.MIA->getAnnotationWithDefault(*CTCInstr, "CTCTakenCount"); + BC.MIB->getAnnotationWithDefault(*CTCInstr, "CTCTakenCount"); CTCMispredCount = - BC.MIA->getAnnotationWithDefault(*CTCInstr, + BC.MIB->getAnnotationWithDefault(*CTCInstr, "CTCMispredCount"); } // Assert that the tail call does not throw. const MCSymbol *LP; uint64_t Action; - std::tie(LP, Action) = BC.MIA->getEHInfo(*CTCInstr); + std::tie(LP, Action) = BC.MIB->getEHInfo(*CTCInstr); assert(!LP && "found tail call with associated landing pad"); // Create a basic block with an unconditional tail call instruction using // the same destination. - const auto *CTCTargetLabel = BC.MIA->getTargetSymbol(*CTCInstr); + const auto *CTCTargetLabel = BC.MIB->getTargetSymbol(*CTCInstr); assert(CTCTargetLabel && "symbol expected for conditional tail call"); MCInst TailCallInstr; - BC.MIA->createTailCall(TailCallInstr, CTCTargetLabel, BC.Ctx.get()); + BC.MIB->createTailCall(TailCallInstr, CTCTargetLabel, BC.Ctx.get()); auto TailCallBB = createBasicBlock(BinaryBasicBlock::INVALID_OFFSET, BC.Ctx->createTempSymbol("TC", true)); TailCallBB->addInstruction(TailCallInstr); @@ -1908,9 +1909,9 @@ void BinaryFunction::removeConditionalTailCalls() { // Add execution count for the block. TailCallBB->setExecutionCount(CTCTakenCount); - BC.MIA->convertTailCallToJmp(*CTCInstr); + BC.MIB->convertTailCallToJmp(*CTCInstr); - BC.MIA->replaceBranchTarget(*CTCInstr, TailCallBB->getLabel(), + BC.MIB->replaceBranchTarget(*CTCInstr, TailCallBB->getLabel(), BC.Ctx.get()); // Add basic block to the list that will be added to the end. @@ -1920,7 +1921,7 @@ void BinaryFunction::removeConditionalTailCalls() { BB.swapConditionalSuccessors(); // This branch is no longer a conditional tail call. - BC.MIA->unsetConditionalTailCall(*CTCInstr); + BC.MIB->unsetConditionalTailCall(*CTCInstr); } insertBasicBlocks(std::prev(end()), @@ -2079,7 +2080,7 @@ bool BinaryFunction::fixCFIState() { int32_t OldState = BB->getCFIState(); // Remember state at function entry point (our reference state). auto InsertIt = FDEStartBB->begin(); - while (InsertIt != FDEStartBB->end() && BC.MIA->isCFI(*InsertIt)) + while (InsertIt != FDEStartBB->end() && BC.MIB->isCFI(*InsertIt)) ++InsertIt; addCFIPseudo(FDEStartBB, InsertIt, FrameInstructions.size()); FrameInstructions.emplace_back( @@ -2106,7 +2107,7 @@ bool BinaryFunction::fixCFIState() { } } auto Pos = BB->begin(); - while (Pos != BB->end() && BC.MIA->isCFI(*Pos)) { + while (Pos != BB->end() && BC.MIB->isCFI(*Pos)) { auto CFI = getCFIFor(*Pos); if (CFI->getOperation() == MCCFIInstruction::OpRememberState) ++StackOffset; @@ -2171,14 +2172,14 @@ void BinaryFunction::emitBody(MCStreamer &Streamer, bool EmitColdPart) { for (auto I = BB->begin(), E = BB->end(); I != E; ++I) { auto &Instr = *I; // Handle pseudo instructions. - if (BC.MIA->isEHLabel(Instr)) { - const auto *Label = BC.MIA->getTargetSymbol(Instr); + if (BC.MIB->isEHLabel(Instr)) { + const auto *Label = BC.MIB->getTargetSymbol(Instr); assert(Instr.getNumOperands() == 1 && Label && "bad EH_LABEL instruction"); Streamer.EmitLabel(const_cast(Label)); continue; } - if (BC.MIA->isCFI(Instr)) { + if (BC.MIB->isCFI(Instr)) { Streamer.EmitCFIInstruction(*getCFIFor(Instr)); continue; } @@ -2187,8 +2188,8 @@ void BinaryFunction::emitBody(MCStreamer &Streamer, bool EmitColdPart) { } // Emit GNU_args_size CFIs as necessary. - if (usesGnuArgsSize() && BC.MIA->isInvoke(Instr)) { - auto NewGnuArgsSize = BC.MIA->getGnuArgsSize(Instr); + if (usesGnuArgsSize() && BC.MIB->isInvoke(Instr)) { + auto NewGnuArgsSize = BC.MIB->getGnuArgsSize(Instr); assert(NewGnuArgsSize >= 0 && "expected non-negative GNU_args_size"); if (NewGnuArgsSize != CurrentGnuArgsSize) { CurrentGnuArgsSize = NewGnuArgsSize; @@ -2267,7 +2268,7 @@ void BinaryFunction::setTrapOnEntry() { for (const auto EntryOffset : EntryOffsets) { MCInst TrapInstr; - BC.MIA->createTrap(TrapInstr); + BC.MIB->createTrap(TrapInstr); addInstruction(EntryOffset, std::move(TrapInstr)); } @@ -2420,11 +2421,11 @@ void BinaryFunction::duplicateConstantIslands() { ++OpNum; continue; } - const auto *Symbol = BC.MIA->getTargetSymbol(Inst, OpNum); + const auto *Symbol = BC.MIB->getTargetSymbol(Inst, OpNum); auto ISym = ColdIslandSymbols.find(Symbol); if (ISym == ColdIslandSymbols.end()) continue; - Operand = MCOperand::createExpr(BC.MIA->getTargetExprFor( + Operand = MCOperand::createExpr(BC.MIB->getTargetExprFor( Inst, MCSymbolRefExpr::create(ISym->second, MCSymbolRefExpr::VK_None, *BC.Ctx), @@ -2524,7 +2525,7 @@ void BinaryFunction::dumpGraph(raw_ostream& OS) const { UncondBranch); const auto *LastInstr = BB->getLastNonPseudoInstr(); - const bool IsJumpTable = LastInstr && BC.MIA->getJumpTable(*LastInstr); + const bool IsJumpTable = LastInstr && BC.MIB->getJumpTable(*LastInstr); auto BI = BB->branch_info_begin(); for (auto *Succ : BB->successors()) { @@ -2662,7 +2663,7 @@ bool BinaryFunction::validateCFG() const { } void BinaryFunction::fixBranches() { - auto &MIA = BC.MIA; + auto &MIB = BC.MIB; auto *Ctx = BC.Ctx.get(); for (unsigned I = 0, E = BasicBlocksLayout.size(); I != E; ++I) { @@ -2698,19 +2699,19 @@ void BinaryFunction::fixBranches() { const auto *TSuccessor = BB->getConditionalSuccessor(true); const auto *FSuccessor = BB->getConditionalSuccessor(false); if (NextBB && NextBB == TSuccessor && - !BC.MIA->hasAnnotation(*CondBranch, "DoNotChangeTarget")) { + !BC.MIB->hasAnnotation(*CondBranch, "DoNotChangeTarget")) { std::swap(TSuccessor, FSuccessor); - MIA->reverseBranchCondition(*CondBranch, TSuccessor->getLabel(), Ctx); + MIB->reverseBranchCondition(*CondBranch, TSuccessor->getLabel(), Ctx); BB->swapConditionalSuccessors(); } else { - MIA->replaceBranchTarget(*CondBranch, TSuccessor->getLabel(), Ctx); + MIB->replaceBranchTarget(*CondBranch, TSuccessor->getLabel(), Ctx); } if (TSuccessor == FSuccessor) { BB->removeDuplicateConditionalSuccessor(CondBranch); } if (!NextBB || ((NextBB != TSuccessor || - BC.MIA->hasAnnotation(*CondBranch, "DoNotChangeTarget")) && + BC.MIB->hasAnnotation(*CondBranch, "DoNotChangeTarget")) && NextBB != FSuccessor)) { BB->addBranchInstruction(FSuccessor); } @@ -2737,7 +2738,7 @@ void BinaryFunction::propagateGnuArgsSizeInfo() { for (auto BB : BasicBlocks) { for (auto II = BB->begin(); II != BB->end(); ) { auto &Instr = *II; - if (BC.MIA->isCFI(Instr)) { + if (BC.MIB->isCFI(Instr)) { auto CFI = getCFIFor(Instr); if (CFI->getOperation() == MCCFIInstruction::OpGnuArgsSize) { CurrentGnuArgsSize = CFI->getOffset(); @@ -2747,9 +2748,9 @@ void BinaryFunction::propagateGnuArgsSizeInfo() { II = BB->erasePseudoInstruction(II); continue; } - } else if (BC.MIA->isInvoke(Instr)) { + } else if (BC.MIB->isInvoke(Instr)) { // Add the value of GNU_args_size as an extra operand to invokes. - BC.MIA->addGnuArgsSize(Instr, CurrentGnuArgsSize); + BC.MIB->addGnuArgsSize(Instr, CurrentGnuArgsSize); } ++II; } @@ -2763,7 +2764,7 @@ void BinaryFunction::postProcessBranches() { auto LastInstrRI = BB->getLastNonPseudo(); if (BB->succ_size() == 1) { if (LastInstrRI != BB->rend() && - BC.MIA->isConditionalBranch(*LastInstrRI)) { + BC.MIB->isConditionalBranch(*LastInstrRI)) { // __builtin_unreachable() could create a conditional branch that // falls-through into the next function - hence the block will have only // one valid successor. Such behaviour is undefined and thus we remove @@ -2784,12 +2785,12 @@ void BinaryFunction::postProcessBranches() { << BB->getName() << " in function " << *this << '\n'); continue; } - if (!BC.MIA->isTerminator(*LastInstrRI) && - !BC.MIA->isCall(*LastInstrRI)) { + if (!BC.MIB->isTerminator(*LastInstrRI) && + !BC.MIB->isCall(*LastInstrRI)) { DEBUG(dbgs() << "BOLT-DEBUG: adding return to basic block " << BB->getName() << " in function " << *this << '\n'); MCInst ReturnInstr; - BC.MIA->createReturn(ReturnInstr); + BC.MIB->createReturn(ReturnInstr); BB->addInstruction(ReturnInstr); } } @@ -2867,7 +2868,7 @@ BinaryFunction::BasicBlockOrderType BinaryFunction::dfs() const { MCInst *UncondBranch = nullptr; if (BB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch) && CondBranch && BB->succ_size() == 2) { - if (BC.MIA->getCanonicalBranchOpcode(CondBranch->getOpcode()) == + if (BC.MIB->getCanonicalBranchOpcode(CondBranch->getOpcode()) == CondBranch->getOpcode()) { Stack.push(BB->getConditionalSuccessor(true)); Stack.push(BB->getConditionalSuccessor(false)); @@ -3005,7 +3006,7 @@ bool BinaryFunction::isIdenticalWith(const BinaryFunction &OtherBF, // is ignored for CFG purposes. auto *TrailingInstr = (I != E ? &(*I) : (OtherI != OtherE ? &(*OtherI) : 0)); - if (TrailingInstr && !BC.MIA->isUnconditionalBranch(*TrailingInstr)) { + if (TrailingInstr && !BC.MIB->isUnconditionalBranch(*TrailingInstr)) { return false; } @@ -3097,7 +3098,7 @@ std::size_t BinaryFunction::hash(bool Recompute, bool UseDFS) const { // Ignore unconditional jumps since we check CFG consistency by processing // basic blocks in order and do not rely on branches to be in-sync with // CFG. Note that we still use condition code of conditional jumps. - if (BC.MIA->isUnconditionalBranch(Inst)) + if (BC.MIB->isUnconditionalBranch(Inst)) continue; if (Opcode == 0) { @@ -3206,9 +3207,9 @@ bool BinaryFunction::replaceJumpTableEntryIn(BinaryBasicBlock *BB, BinaryBasicBlock *OldDest, BinaryBasicBlock *NewDest) { auto *Instr = BB->getLastNonPseudoInstr(); - if (!Instr || !BC.MIA->isIndirectBranch(*Instr)) + if (!Instr || !BC.MIB->isIndirectBranch(*Instr)) return false; - auto JTAddress = BC.MIA->getJumpTable(*Instr); + auto JTAddress = BC.MIB->getJumpTable(*Instr); assert(JTAddress && "Invalid jump table address"); auto *JT = getJumpTableContainingAddress(JTAddress); assert(JT && "No jump table structure for this indirect branch"); @@ -3585,7 +3586,7 @@ MCInst *BinaryFunction::getInstructionAtOffset(uint64_t Offset) { for (auto &Inst : *BB) { constexpr auto InvalidOffset = std::numeric_limits::max(); - if (Offset == BC.MIA->getAnnotationWithDefault(Inst, "Offset", + if (Offset == BC.MIB->getAnnotationWithDefault(Inst, "Offset", InvalidOffset)) return &Inst; } @@ -3773,23 +3774,23 @@ DynoStats BinaryFunction::getDynoStats() const { // Count the number of calls by iterating through all instructions. for (const auto &Instr : *BB) { - if (BC.MIA->isStore(Instr)) { + if (BC.MIB->isStore(Instr)) { Stats[DynoStats::STORES] += BBExecutionCount; } - if (BC.MIA->isLoad(Instr)) { + if (BC.MIB->isLoad(Instr)) { Stats[DynoStats::LOADS] += BBExecutionCount; } - if (!BC.MIA->isCall(Instr)) + if (!BC.MIB->isCall(Instr)) continue; uint64_t CallFreq = BBExecutionCount; - if (BC.MIA->getConditionalTailCall(Instr)) { + if (BC.MIB->getConditionalTailCall(Instr)) { CallFreq = - BC.MIA->getAnnotationWithDefault(Instr, "CTCTakenCount"); + BC.MIB->getAnnotationWithDefault(Instr, "CTCTakenCount"); } Stats[DynoStats::FUNCTION_CALLS] += CallFreq; - if (BC.MIA->isIndirectCall(Instr)) { + if (BC.MIB->isIndirectCall(Instr)) { Stats[DynoStats::INDIRECT_CALLS] += CallFreq; - } else if (const auto *CallSymbol = BC.MIA->getTargetSymbol(Instr)) { + } else if (const auto *CallSymbol = BC.MIB->getTargetSymbol(Instr)) { const auto *BF = BC.getFunctionForSymbol(CallSymbol); if (BF && BF->isPLTFunction()) { Stats[DynoStats::PLT_CALLS] += CallFreq; @@ -3806,7 +3807,7 @@ DynoStats BinaryFunction::getDynoStats() const { // Jump tables. const auto *LastInstr = BB->getLastNonPseudoInstr(); - if (BC.MIA->getJumpTable(*LastInstr)) { + if (BC.MIB->getJumpTable(*LastInstr)) { Stats[DynoStats::JUMP_TABLE_BRANCHES] += BBExecutionCount; DEBUG( static uint64_t MostFrequentJT; @@ -3840,7 +3841,7 @@ DynoStats BinaryFunction::getDynoStats() const { } // CTCs - if (BC.MIA->getConditionalTailCall(*CondBranch)) { + if (BC.MIB->getConditionalTailCall(*CondBranch)) { if (BB->branch_info_begin() != BB->branch_info_end()) Stats[DynoStats::UNCOND_BRANCHES] += BB->branch_info_begin()->Count; continue; diff --git a/bolt/BinaryFunction.h b/bolt/BinaryFunction.h index 2ff68d673958..7db9825525e9 100644 --- a/bolt/BinaryFunction.h +++ b/bolt/BinaryFunction.h @@ -32,7 +32,6 @@ #include "llvm/MC/MCDisassembler/MCDisassembler.h" #include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCInst.h" -#include "llvm/MC/MCInstrAnalysis.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Object/ObjectFile.h" @@ -433,8 +432,8 @@ private: // NB: there's no need to compare jump table indirect jump instructions // separately as jump tables are handled by comparing corresponding // symbols. - const auto EHInfoA = BC.MIA->getEHInfo(InstA); - const auto EHInfoB = BC.MIA->getEHInfo(InstB); + const auto EHInfoA = BC.MIB->getEHInfo(InstA); + const auto EHInfoB = BC.MIB->getEHInfo(InstB); // Action indices should match. if (EHInfoA.second != EHInfoB.second) @@ -928,8 +927,8 @@ public: /// that is in \p BB. Return nullptr if none exists BinaryBasicBlock *getLandingPadBBFor(const BinaryBasicBlock &BB, const MCInst &InvokeInst) { - assert(BC.MIA->isInvoke(InvokeInst) && "must be invoke instruction"); - MCLandingPad LP = BC.MIA->getEHInfo(InvokeInst); + assert(BC.MIB->isInvoke(InvokeInst) && "must be invoke instruction"); + MCLandingPad LP = BC.MIB->getEHInfo(InvokeInst); if (LP.first) { auto *LBB = BB.getLandingPad(LP.first); assert (LBB && "Landing pad should be defined"); @@ -1259,12 +1258,12 @@ public: } const JumpTable *getJumpTable(const MCInst &Inst) const { - const auto Address = BC.MIA->getJumpTable(Inst); + const auto Address = BC.MIB->getJumpTable(Inst); return getJumpTableContainingAddress(Address); } JumpTable *getJumpTable(const MCInst &Inst) { - const auto Address = BC.MIA->getJumpTable(Inst); + const auto Address = BC.MIB->getJumpTable(Inst); return getJumpTableContainingAddress(Address); } @@ -1498,7 +1497,7 @@ public: } --I; - while (I != Instructions.begin() && BC.MIA->isNoop(I->second)) { + while (I != Instructions.begin() && BC.MIB->isNoop(I->second)) { Offset = I->first; --I; } @@ -1522,13 +1521,13 @@ public: BinaryBasicBlock::iterator Pos, uint32_t Offset) { MCInst CFIPseudo; - BC.MIA->createCFI(CFIPseudo, Offset); + BC.MIB->createCFI(CFIPseudo, Offset); return BB->insertPseudoInstr(Pos, CFIPseudo); } /// Retrieve the MCCFIInstruction object associated with a CFI pseudo. MCCFIInstruction* getCFIFor(const MCInst &Instr) { - if (!BC.MIA->isCFI(Instr)) + if (!BC.MIB->isCFI(Instr)) return nullptr; uint32_t Offset = Instr.getOperand(0).getImm(); assert(Offset < FrameInstructions.size() && "Invalid CFI offset"); @@ -1536,7 +1535,7 @@ public: } const MCCFIInstruction* getCFIFor(const MCInst &Instr) const { - if (!BC.MIA->isCFI(Instr)) + if (!BC.MIB->isCFI(Instr)) return nullptr; uint32_t Offset = Instr.getOperand(0).getImm(); assert(Offset < FrameInstructions.size() && "Invalid CFI offset"); diff --git a/bolt/BinaryFunctionProfile.cpp b/bolt/BinaryFunctionProfile.cpp index 77592d8a7743..9317aa41634c 100644 --- a/bolt/BinaryFunctionProfile.cpp +++ b/bolt/BinaryFunctionProfile.cpp @@ -112,7 +112,7 @@ bool BinaryFunction::recordTrace( auto *PrevBB = BasicBlocksLayout[FromBB->getIndex() - 1]; if (PrevBB->getSuccessor(FromBB->getLabel())) { const auto *Instr = PrevBB->getLastNonPseudoInstr(); - if (Instr && BC.MIA->isCall(*Instr)) { + if (Instr && BC.MIB->isCall(*Instr)) { FromBB = PrevBB; } else { DEBUG(dbgs() << "invalid incoming LBR (no call): " << FirstLBR << '\n'); @@ -156,7 +156,7 @@ bool BinaryFunction::recordTrace( const auto *Instr = BB->getLastNonPseudoInstr(); uint64_t Offset{0}; if (Instr) { - Offset = BC.MIA->getAnnotationWithDefault(*Instr, "Offset"); + Offset = BC.MIB->getAnnotationWithDefault(*Instr, "Offset"); } else { Offset = BB->getOffset(); } @@ -186,7 +186,7 @@ bool BinaryFunction::recordBranch(uint64_t From, uint64_t To, if (!opts::CompatMode) return true; auto *Instr = getInstructionAtOffset(0); - if (Instr && BC.MIA->isCall(*Instr)) + if (Instr && BC.MIB->isCall(*Instr)) return true; return false; } @@ -211,10 +211,10 @@ bool BinaryFunction::recordBranch(uint64_t From, uint64_t To, const auto *LastInstr = ToBB->getLastNonPseudoInstr(); if (LastInstr) { const auto LastInstrOffset = - BC.MIA->getAnnotationWithDefault(*LastInstr, "Offset"); + BC.MIB->getAnnotationWithDefault(*LastInstr, "Offset"); // With old .fdata we are getting FT branches for "jcc,jmp" sequences. - if (To == LastInstrOffset && BC.MIA->isUnconditionalBranch(*LastInstr)) { + if (To == LastInstrOffset && BC.MIB->isUnconditionalBranch(*LastInstr)) { return true; } @@ -249,8 +249,8 @@ bool BinaryFunction::recordBranch(uint64_t From, uint64_t To, if (!FromBB->getSuccessor(ToBB->getLabel())) { // Check if this is a recursive call or a return from a recursive call. - if (ToBB->isEntryPoint() && (BC.MIA->isCall(*FromInstruction) || - BC.MIA->isIndirectBranch(*FromInstruction))) { + if (ToBB->isEntryPoint() && (BC.MIB->isCall(*FromInstruction) || + BC.MIB->isIndirectBranch(*FromInstruction))) { // Execution count is already accounted for. return true; } @@ -371,7 +371,7 @@ void BinaryFunction::postProcessProfile() { const auto *LastInstr = BB->getLastNonPseudoInstr(); if (!LastInstr) continue; - const auto JTAddress = BC.MIA->getJumpTable(*LastInstr); + const auto JTAddress = BC.MIB->getJumpTable(*LastInstr); if (!JTAddress) continue; auto *JT = getJumpTableContainingAddress(JTAddress); @@ -492,27 +492,27 @@ void BinaryFunction::convertBranchData() { auto *Instr = getInstructionAtOffset(BI.From.Offset); if (!Instr || - (!BC.MIA->isCall(*Instr) && !BC.MIA->isIndirectBranch(*Instr))) + (!BC.MIB->isCall(*Instr) && !BC.MIB->isIndirectBranch(*Instr))) continue; auto setOrUpdateAnnotation = [&](StringRef Name, uint64_t Count) { - if (opts::Verbosity >= 1 && BC.MIA->hasAnnotation(*Instr, Name)) { + if (opts::Verbosity >= 1 && BC.MIB->hasAnnotation(*Instr, Name)) { errs() << "BOLT-WARNING: duplicate " << Name << " info for offset 0x" << Twine::utohexstr(BI.From.Offset) << " in function " << *this << '\n'; } - auto &Value = BC.MIA->getOrCreateAnnotationAs(BC.Ctx.get(), + auto &Value = BC.MIB->getOrCreateAnnotationAs(BC.Ctx.get(), *Instr, Name); Value += Count; }; - if (BC.MIA->isIndirectCall(*Instr) || BC.MIA->isIndirectBranch(*Instr)) { + if (BC.MIB->isIndirectCall(*Instr) || BC.MIB->isIndirectBranch(*Instr)) { IndirectCallSiteProfile &CSP = - BC.MIA->getOrCreateAnnotationAs(BC.Ctx.get(), + BC.MIB->getOrCreateAnnotationAs(BC.Ctx.get(), *Instr, "CallProfile"); CSP.emplace_back(BI.To.IsSymbol, BI.To.Name, BI.Branches, BI.Mispreds); - } else if (BC.MIA->getConditionalTailCall(*Instr)) { + } else if (BC.MIB->getConditionalTailCall(*Instr)) { setOrUpdateAnnotation("CTCTakenCount", BI.Branches); setOrUpdateAnnotation("CTCMispredCount", BI.Mispreds); } else { @@ -659,9 +659,9 @@ void BinaryFunction::inferFallThroughCounts() { // Get taken count of conditional tail call if the block ends with one. uint64_t CTCTakenCount = 0; const auto CTCInstr = BB->getLastNonPseudoInstr(); - if (CTCInstr && BC.MIA->getConditionalTailCall(*CTCInstr)) { + if (CTCInstr && BC.MIB->getConditionalTailCall(*CTCInstr)) { CTCTakenCount = - BC.MIA->getAnnotationWithDefault(*CTCInstr, "CTCTakenCount"); + BC.MIB->getAnnotationWithDefault(*CTCInstr, "CTCTakenCount"); } // Calculate frequency of throws from this node according to LBR data @@ -696,8 +696,8 @@ void BinaryFunction::inferFallThroughCounts() { // Skip if the last instruction is an unconditional jump. const auto *LastInstr = BB->getLastNonPseudoInstr(); if (LastInstr && - (BC.MIA->isUnconditionalBranch(*LastInstr) || - BC.MIA->isIndirectBranch(*LastInstr))) + (BC.MIB->isUnconditionalBranch(*LastInstr) || + BC.MIB->isIndirectBranch(*LastInstr))) continue; // If there is an FT it will be the last successor. auto &SuccBI = *BB->branch_info_rbegin(); @@ -832,12 +832,12 @@ float BinaryFunction::evaluateProfileData(const FuncBranchData &BranchData) { // by regular branch instructions and we need isBranch() here. auto *Instr = getInstructionAtOffset(BI.From.Offset); // If it's a prefix - skip it. - if (Instr && BC.MIA->isPrefix(*Instr)) + if (Instr && BC.MIB->isPrefix(*Instr)) Instr = getInstructionAtOffset(BI.From.Offset + 1); if (Instr && - (BC.MIA->isCall(*Instr) || - BC.MIA->isBranch(*Instr) || - BC.MIA->isReturn(*Instr))) { + (BC.MIB->isCall(*Instr) || + BC.MIB->isBranch(*Instr) || + BC.MIB->isReturn(*Instr))) { IsValid = true; } } diff --git a/bolt/CMakeLists.txt b/bolt/CMakeLists.txt index cc0095b10467..3804b16f2e8f 100644 --- a/bolt/CMakeLists.txt +++ b/bolt/CMakeLists.txt @@ -1,5 +1,6 @@ add_subdirectory(merge-fdata) add_subdirectory(Passes) +add_subdirectory(Target) # Get the current git revision for BOLT. function(get_version ofn) @@ -47,6 +48,8 @@ add_public_gen_version_target(GenBoltRevision) set(LLVM_LINK_COMPONENTS ${LLVM_TARGETS_TO_BUILD} BOLTPasses + BOLTTargetAArch64 + BOLTTargetX86 CodeGen Core DebugInfoDWARF @@ -75,6 +78,7 @@ add_llvm_tool(llvm-bolt DWARFRewriter.cpp Exceptions.cpp JumpTable.cpp + MCPlusBuilder.cpp ProfileReader.cpp ProfileWriter.cpp Relocation.cpp diff --git a/bolt/CacheMetrics.cpp b/bolt/CacheMetrics.cpp index 795706135a45..36c4832eb5b2 100644 --- a/bolt/CacheMetrics.cpp +++ b/bolt/CacheMetrics.cpp @@ -147,11 +147,11 @@ extractFunctionCalls(const std::vector &BinaryFunctions) { for (auto BB : SrcFunction->layout()) { // Find call instructions and extract target symbols from each one for (auto &Inst : *BB) { - if (!BC.MIA->isCall(Inst)) + if (!BC.MIB->isCall(Inst)) continue; // Call info - const MCSymbol* DstSym = BC.MIA->getTargetSymbol(Inst); + const MCSymbol* DstSym = BC.MIB->getTargetSymbol(Inst); auto Count = BB->getKnownExecutionCount(); // Ignore calls w/o information if (DstSym == nullptr || Count == 0) diff --git a/bolt/Exceptions.cpp b/bolt/Exceptions.cpp index b3ad09e3684b..58f77d8f9e6d 100644 --- a/bolt/Exceptions.cpp +++ b/bolt/Exceptions.cpp @@ -236,13 +236,13 @@ void BinaryFunction::parseLSDA(ArrayRef LSDASectionData, assert(II != IE && "exception range not pointing to an instruction"); do { auto &Instruction = II->second; - if (BC.MIA->isCall(Instruction) && - !BC.MIA->getConditionalTailCall(Instruction)) { - assert(!BC.MIA->isInvoke(Instruction) && + if (BC.MIB->isCall(Instruction) && + !BC.MIB->getConditionalTailCall(Instruction)) { + assert(!BC.MIB->isInvoke(Instruction) && "overlapping exception ranges detected"); // Add extra operands to a call instruction making it an invoke from // now on. - BC.MIA->addEHInfo(Instruction, + BC.MIB->addEHInfo(Instruction, MCLandingPad(LPSymbol, ActionEntry), BC.Ctx.get()); } @@ -408,11 +408,11 @@ void BinaryFunction::updateEHRanges() { for (auto II = BB->begin(); II != BB->end(); ++II) { auto Instr = *II; - if (!BC.MIA->isCall(Instr)) + if (!BC.MIB->isCall(Instr)) continue; // Instruction can throw an exception that should be handled. - const bool Throws = BC.MIA->isInvoke(Instr); + const bool Throws = BC.MIB->isInvoke(Instr); // Ignore the call if it's a continuation of a no-throw gap. if (!Throws && !StartRange) @@ -421,7 +421,7 @@ void BinaryFunction::updateEHRanges() { // Extract exception handling information from the instruction. const MCSymbol *LP = nullptr; uint64_t Action = 0; - std::tie(LP, Action) = BC.MIA->getEHInfo(Instr); + std::tie(LP, Action) = BC.MIB->getEHInfo(Instr); // No action if the exception handler has not changed. if (Throws && @@ -433,7 +433,7 @@ void BinaryFunction::updateEHRanges() { // Same symbol is used for the beginning and the end of the range. const MCSymbol *EHSymbol = BC.Ctx->createTempSymbol("EH", true); MCInst EHLabel; - BC.MIA->createEHLabel(EHLabel, EHSymbol, BC.Ctx.get()); + BC.MIB->createEHLabel(EHLabel, EHSymbol, BC.Ctx.get()); II = std::next(BB->insertPseudoInstr(II, EHLabel)); // At this point we could be in one of the following states: diff --git a/bolt/MCPlusBuilder.cpp b/bolt/MCPlusBuilder.cpp new file mode 100644 index 000000000000..e7d3ece50905 --- /dev/null +++ b/bolt/MCPlusBuilder.cpp @@ -0,0 +1,384 @@ +//===- MCPlusBuilder.cpp - main interface for MCPlus-level instructions ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Create/analyze/modify instructions at MC+ level. +// +//===----------------------------------------------------------------------===// + +#include "MCPlusBuilder.h" +#include "llvm/MC/MCInstrAnalysis.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/Support/Debug.h" +#include +#include + +#define DEBUG_TYPE "mcplus" + +using namespace llvm; +using namespace bolt; + +bool MCPlusBuilder::evaluateBranch(const MCInst &Inst, uint64_t Addr, + uint64_t Size, uint64_t &Target) const { + return Analysis->evaluateBranch(Inst, Addr, Size, Target); +} + +namespace { + +const MCLandingPad *findLandingPad(const MCInst &Inst) { + for (unsigned I = Inst.getNumOperands(); I > 0; --I) { + const auto &Op = Inst.getOperand(I - 1); + if (Op.isLandingPad()) { + return Op.getLandingPad(); + } + } + return nullptr; +} + +} + +bool MCPlusBuilder::hasEHInfo(const MCInst &Inst) const { + return findLandingPad(Inst) != nullptr; +} + +MCLandingPad MCPlusBuilder::getEHInfo(const MCInst &Inst) const { + const MCSymbol *LPSym = nullptr; + uint64_t Action = 0; + if (isCall(Inst)) { + if (auto LP = findLandingPad(Inst)) { + std::tie(LPSym, Action) = *LP; + } + } + + return std::make_pair(LPSym, Action); +} + +// Add handler and action info for call instruction. +void MCPlusBuilder::addEHInfo(MCInst &Inst, + const MCLandingPad &LP, + MCContext *Ctx) const { + if (isCall(Inst)) { + assert(!hasEHInfo(Inst)); + Inst.addOperand( + MCOperand::createLandingPad(new (*Ctx) MCLandingPad(LP))); + } +} + +int64_t MCPlusBuilder::getGnuArgsSize(const MCInst &Inst) const { + for (unsigned I = Inst.getNumOperands(); I > 0; --I) { + const auto &Op = Inst.getOperand(I - 1); + if (Op.isGnuArgsSize()) { + return Op.getGnuArgsSize(); + } + } + return -1LL; +} + +void MCPlusBuilder::addGnuArgsSize(MCInst &Inst, int64_t GnuArgsSize) const { + assert(GnuArgsSize >= 0 && "cannot set GNU_args_size to negative value"); + assert(getGnuArgsSize(Inst) == -1LL && "GNU_args_size already set"); + assert(isInvoke(Inst) && "GNU_args_size can only be set for invoke"); + + Inst.addOperand(MCOperand::createGnuArgsSize(GnuArgsSize)); +} + +uint64_t MCPlusBuilder::getJumpTable(const MCInst &Inst) const { + for (unsigned I = Inst.getNumOperands(); I > 0; --I) { + const auto &Op = Inst.getOperand(I - 1); + if (Op.isJumpTable()) { + return Op.getJumpTable(); + } + } + return 0; +} + +bool MCPlusBuilder::setJumpTable(MCContext *Ctx, MCInst &Inst, uint64_t Value, + uint16_t IndexReg) const { + if (!isIndirectBranch(Inst)) + return false; + assert(getJumpTable(Inst) == 0 && "jump table already set"); + Inst.addOperand(MCOperand::createJumpTable(Value)); + addAnnotation<>(Ctx, Inst, "JTIndexReg", IndexReg); + return true; +} + +Optional +MCPlusBuilder::getConditionalTailCall(const MCInst &Inst) const { + for (unsigned I = Inst.getNumOperands(); I > 0; --I) { + const auto &Op = Inst.getOperand(I - 1); + if (Op.isConditionalTailCall()) { + return Op.getConditionalTailCall(); + } + } + return NoneType(); +} + +bool +MCPlusBuilder::setConditionalTailCall(MCInst &Inst, uint64_t Dest) const { + if (!isConditionalBranch(Inst)) + return false; + + for (unsigned I = Inst.getNumOperands(); I > 0; --I) { + auto &Op = Inst.getOperand(I - 1); + if (Op.isConditionalTailCall()) { + Op.setConditionalTailCall(Dest); + return true; + } + } + + Inst.addOperand(MCOperand::createConditionalTailCall(Dest)); + return true; +} + +bool MCPlusBuilder::unsetConditionalTailCall(MCInst &Inst) const { + for (auto OpI = Inst.begin(), OpE = Inst.end(); OpI != OpE; ++OpI) { + if (OpI->isConditionalTailCall()) { + Inst.erase(OpI); + return true; + } + } + + return false; +} + +namespace { + +unsigned findAnnotationIndex(const MCInst &Inst, StringRef Name) { + for (unsigned I = Inst.getNumOperands(); I > 0; --I) { + const auto& Op = Inst.getOperand(I - 1); + if (Op.isAnnotation() && Op.getAnnotation()->getName() == Name) { + return I - 1; + } + } + return Inst.getNumOperands(); +} + +} + +bool MCPlusBuilder::hasAnnotation(const MCInst &Inst, StringRef Name) const { + return findAnnotationIndex(Inst, Name) < Inst.getNumOperands(); +} + +bool MCPlusBuilder::removeAnnotation(MCInst &Inst, StringRef Name) const { + const auto Idx = findAnnotationIndex(Inst, Name); + if (Idx < Inst.getNumOperands()) { + auto *Annotation = Inst.getOperand(Idx).getAnnotation(); + auto Itr = AnnotationPool.find(Annotation); + if (Itr != AnnotationPool.end()) { + AnnotationPool.erase(Itr); + Annotation->~MCAnnotation(); + } + Inst.erase(Inst.begin() + Idx); + return true; + } + return false; +} + +void MCPlusBuilder::removeAllAnnotations(MCInst &Inst) const { + for (auto Idx = Inst.getNumOperands(); Idx > 0; --Idx) { + auto &Op = Inst.getOperand(Idx - 1); + if (Op.isAnnotation()) { + auto *Annotation = Op.getAnnotation(); + auto Itr = AnnotationPool.find(Annotation); + if (Itr != AnnotationPool.end()) { + AnnotationPool.erase(Itr); + Annotation->~MCAnnotation(); + } + Inst.erase(Inst.begin() + Idx - 1); + } + } +} + +bool MCPlusBuilder::renameAnnotation(MCInst &Inst, + StringRef Before, + StringRef After) const { + const auto Idx = findAnnotationIndex(Inst, Before); + if (Idx >= Inst.getNumOperands()) { + return false; + } + auto *Annotation = Inst.getOperand(Idx).getAnnotation(); + auto PooledName = AnnotationNames.intern(After); + AnnotationNameRefs.insert(PooledName); + Annotation->setName(*PooledName); + return true; +} + +const MCAnnotation * +MCPlusBuilder::getAnnotation(const MCInst &Inst, StringRef Name) const { + const auto Idx = findAnnotationIndex(Inst, Name); + assert(Idx < Inst.getNumOperands()); + return Inst.getOperand(Idx).getAnnotation(); +} + +void MCPlusBuilder::getClobberedRegs(const MCInst &Inst, + BitVector &Regs) const { + if (isPrefix(Inst) || isCFI(Inst)) + return; + + const auto &InstInfo = Info->get(Inst.getOpcode()); + + const auto *ImplicitDefs = InstInfo.getImplicitDefs(); + for (unsigned I = 0, E = InstInfo.getNumImplicitDefs(); I != E; ++I) { + Regs |= getAliases(ImplicitDefs[I], /*OnlySmaller=*/false); + } + + for (unsigned I = 0, E = InstInfo.getNumDefs(); I != E; ++I) { + const auto &Operand = Inst.getOperand(I); + assert(Operand.isReg()); + Regs |= getAliases(Operand.getReg(), /*OnlySmaller=*/false); + } +} + +void MCPlusBuilder::getTouchedRegs(const MCInst &Inst, + BitVector &Regs) const { + if (isPrefix(Inst) || isCFI(Inst)) + return; + + const auto &InstInfo = Info->get(Inst.getOpcode()); + + const auto *ImplicitDefs = InstInfo.getImplicitDefs(); + for (unsigned I = 0, E = InstInfo.getNumImplicitDefs(); I != E; ++I) { + Regs |= getAliases(ImplicitDefs[I], /*OnlySmaller=*/false); + } + const auto *ImplicitUses = InstInfo.getImplicitUses(); + for (unsigned I = 0, E = InstInfo.getNumImplicitUses(); I != E; ++I) { + Regs |= getAliases(ImplicitUses[I], /*OnlySmaller=*/false); + } + + for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) { + if (!Inst.getOperand(I).isReg()) + continue; + Regs |= getAliases(Inst.getOperand(I).getReg(), /*OnlySmaller=*/false); + } +} + +void MCPlusBuilder::getWrittenRegs(const MCInst &Inst, + BitVector &Regs) const { + if (isPrefix(Inst) || isCFI(Inst)) + return; + + const auto &InstInfo = Info->get(Inst.getOpcode()); + + const auto *ImplicitDefs = InstInfo.getImplicitDefs(); + for (unsigned I = 0, E = InstInfo.getNumImplicitDefs(); I != E; ++I) { + Regs |= getAliases(ImplicitDefs[I], /*OnlySmaller=*/true); + } + + for (unsigned I = 0, E = InstInfo.getNumDefs(); I != E; ++I) { + const auto &Operand = Inst.getOperand(I); + assert(Operand.isReg()); + Regs |= getAliases(Operand.getReg(), /*OnlySmaller=*/true); + } +} + +void MCPlusBuilder::getUsedRegs(const MCInst &Inst, BitVector &Regs) const { + if (isPrefix(Inst) || isCFI(Inst)) + return; + + const auto &InstInfo = Info->get(Inst.getOpcode()); + + const auto *ImplicitUses = InstInfo.getImplicitUses(); + for (unsigned I = 0, E = InstInfo.getNumImplicitUses(); I != E; ++I) { + Regs |= getAliases(ImplicitUses[I], /*OnlySmaller=*/true); + } + + for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) { + if (!Inst.getOperand(I).isReg()) + continue; + Regs |= getAliases(Inst.getOperand(I).getReg(), /*OnlySmaller=*/true); + } +} + +const BitVector & +MCPlusBuilder::getAliases(MCPhysReg Reg, + bool OnlySmaller) const { + // AliasMap caches a mapping of registers to the set of registers that + // alias (are sub or superregs of itself, including itself). + static std::vector AliasMap; + static std::vector SuperReg; + + if (AliasMap.size() > 0) { + if (OnlySmaller) + return AliasMap[Reg]; + return AliasMap[SuperReg[Reg]]; + } + // Build alias map + for (MCPhysReg I = 0, E = RegInfo->getNumRegs(); I != E; ++I) { + BitVector BV(RegInfo->getNumRegs(), false); + BV.set(I); + AliasMap.emplace_back(std::move(BV)); + SuperReg.emplace_back(I); + } + std::queue Worklist; + // Propagate alias info upwards + for (MCPhysReg I = 0, E = RegInfo->getNumRegs(); I != E; ++I) { + Worklist.push(I); + } + while (!Worklist.empty()) { + MCPhysReg I = Worklist.front(); + Worklist.pop(); + for (MCSubRegIterator SI(I, RegInfo); SI.isValid(); ++SI) { + AliasMap[I] |= AliasMap[*SI]; + } + for (MCSuperRegIterator SI(I, RegInfo); SI.isValid(); ++SI) { + Worklist.push(*SI); + } + } + // Propagate parent reg downwards + for (MCPhysReg I = 0, E = RegInfo->getNumRegs(); I != E; ++I) { + Worklist.push(I); + } + while (!Worklist.empty()) { + MCPhysReg I = Worklist.front(); + Worklist.pop(); + for (MCSubRegIterator SI(I, RegInfo); SI.isValid(); ++SI) { + SuperReg[*SI] = SuperReg[I]; + Worklist.push(*SI); + } + } + + DEBUG({ + dbgs() << "Dumping reg alias table:\n"; + for (MCPhysReg I = 0, E = RegInfo->getNumRegs(); I != E; ++I) { + dbgs() << "Reg " << I << ": "; + const BitVector &BV = AliasMap[SuperReg[I]]; + int Idx = BV.find_first(); + while (Idx != -1) { + dbgs() << Idx << " "; + Idx = BV.find_next(Idx); + } + dbgs() << "\n"; + } + }); + + if (OnlySmaller) + return AliasMap[Reg]; + return AliasMap[SuperReg[Reg]]; +} + +uint8_t +MCPlusBuilder::getRegSize(MCPhysReg Reg) const { + // SizeMap caches a mapping of registers to their sizes + static std::vector SizeMap; + + if (SizeMap.size() > 0) { + return SizeMap[Reg]; + } + SizeMap = std::vector(RegInfo->getNumRegs()); + // Build size map + for (auto I = RegInfo->regclass_begin(), E = RegInfo->regclass_end(); I != E; + ++I) { + for (MCPhysReg Reg : *I) { + SizeMap[Reg] = I->getSize(); + } + } + + return SizeMap[Reg]; +} diff --git a/bolt/MCPlusBuilder.h b/bolt/MCPlusBuilder.h new file mode 100644 index 000000000000..0882317b333d --- /dev/null +++ b/bolt/MCPlusBuilder.h @@ -0,0 +1,1347 @@ +//===--- MCPlusBuilder.h - main interface for MCPlus-level instructions ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Create/analyze/modify instructions at MC+ level. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVM_BOLT_MCPLUSBUILDER_H +#define LLVM_TOOLS_LLVM_BOLT_MCPLUSBUILDER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/Optional.h" +#include "llvm/MC/MCAnnotation.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstrAnalysis.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSymbol.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/StringPool.h" +#include +#include +#include +#include +#include +#include + +namespace llvm { +namespace bolt { + +/// Different types of indirect branches encountered during disassembly. +enum class IndirectBranchType : char { + UNKNOWN = 0, /// Unable to determine type. + POSSIBLE_TAIL_CALL, /// Possibly a tail call. + POSSIBLE_JUMP_TABLE, /// Possibly a switch/jump table. + POSSIBLE_PIC_JUMP_TABLE, /// Possibly a jump table for PIC. + POSSIBLE_GOTO /// Possibly a gcc's computed goto. +}; + +class MCPlusBuilder { +protected: + const MCInstrAnalysis *Analysis; + const MCInstrInfo *Info; + const MCRegisterInfo *RegInfo; + + /// Hash a PooledStringPtr. It's ok to use the address since all these + /// strings are interned. + struct HashPooledStringPtr { + size_t operator()(const PooledStringPtr &Str) const { + return reinterpret_cast(Str.begin()); + } + }; + + // This map holds onto strings in the StringPool and non-trivial + // MCAnnotations. Instances of the MCAnnotation class can't hold Strings + // or non-trivial annotation values since they may contain memory that is + // not managed by MCContext. + mutable std::unordered_set AnnotationNameRefs; + mutable StringPool AnnotationNames; + + // Record all the annotations with non-trivial type. To prevent leaks, these + // will need destructors called when the annotation is removed or when all + // annotations are destroyed. + mutable std::unordered_set AnnotationPool; + +public: + class InstructionIterator + : public std::iterator { + public: + class Impl { + public: + virtual Impl *Copy() const = 0; + virtual void Next() = 0; + virtual void Prev() = 0; + virtual MCInst &Deref() = 0; + virtual bool Compare(const Impl &Other) const = 0; + virtual ~Impl() { } + }; + + template + class SeqImpl : public Impl { + public: + virtual Impl *Copy() const override { + return new SeqImpl(Itr); + } + virtual void Next() override { ++Itr; } + virtual void Prev() override { --Itr; } + virtual MCInst &Deref() override { + return const_cast(*Itr); + } + virtual bool Compare(const Impl &Other) const override { + // assumes that Other is same underlying type + return Itr == static_cast&>(Other).Itr; + } + explicit SeqImpl(T &&Itr) : Itr(std::move(Itr)) { } + explicit SeqImpl(const T &Itr) : Itr(Itr) { } + private: + T Itr; + }; + + template + class MapImpl : public Impl { + public: + virtual Impl *Copy() const override { + return new MapImpl(Itr); + } + virtual void Next() override { ++Itr; } + virtual void Prev() override { --Itr; } + virtual MCInst &Deref() override { + return const_cast(Itr->second); + } + virtual bool Compare(const Impl &Other) const override { + // assumes that Other is same underlying type + return Itr == static_cast&>(Other).Itr; + } + explicit MapImpl(T &&Itr) : Itr(std::move(Itr)) { } + explicit MapImpl(const T &Itr) : Itr(Itr) { } + private: + T Itr; + }; + + InstructionIterator &operator++() { + Itr->Next(); + return *this; + } + InstructionIterator &operator--() { + Itr->Prev(); + return *this; + } + InstructionIterator operator++(int) { + std::unique_ptr Tmp(Itr->Copy()); + Itr->Next(); + return InstructionIterator(std::move(Tmp)); + } + InstructionIterator operator--(int) { + std::unique_ptr Tmp(Itr->Copy()); + Itr->Prev(); + return InstructionIterator(std::move(Tmp)); + } + bool operator==(const InstructionIterator& Other) const { + return Itr->Compare(*Other.Itr); + } + bool operator!=(const InstructionIterator& Other) const { + return !Itr->Compare(*Other.Itr); + } + MCInst& operator*() { return Itr->Deref(); } + MCInst* operator->() { return &Itr->Deref(); } + + InstructionIterator &operator=(InstructionIterator &&Other) { + Itr = std::move(Other.Itr); + return *this; + } + InstructionIterator &operator=(const InstructionIterator &Other) { + if (this != &Other) + Itr.reset(Other.Itr->Copy()); + return *this; + } + InstructionIterator() { } + InstructionIterator(const InstructionIterator &Other) + : Itr(Other.Itr->Copy()) { } + InstructionIterator(InstructionIterator &&Other) + : Itr(std::move(Other.Itr)) { } + explicit InstructionIterator(std::unique_ptr Itr) + : Itr(std::move(Itr)) { } + + InstructionIterator(std::vector::iterator Itr) + : Itr(new SeqImpl::iterator>(Itr)) { } + + template + InstructionIterator(T *Itr) + : Itr(new SeqImpl(Itr)) { } + + InstructionIterator(ArrayRef::iterator Itr) + : Itr(new SeqImpl::iterator>(Itr)) { } + + InstructionIterator(MutableArrayRef::iterator Itr) + : Itr(new SeqImpl::iterator>(Itr)) { } + + // TODO: it would be nice to templatize this on the key type. + InstructionIterator(std::map::iterator Itr) + : Itr(new MapImpl::iterator>(Itr)) { } + private: + std::unique_ptr Itr; + }; + +public: + MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info, + const MCRegisterInfo *RegInfo) + : Analysis(Analysis), Info(Info), RegInfo(RegInfo) {} + + virtual ~MCPlusBuilder() { + AnnotationNameRefs.clear(); + for (auto *Annotation : AnnotationPool) { + Annotation->~MCAnnotation(); + } + } + + virtual bool isBranch(const MCInst &Inst) const { + return Analysis->isBranch(Inst); + } + + virtual bool isConditionalBranch(const MCInst &Inst) const { + return Analysis->isConditionalBranch(Inst); + } + + virtual bool isUnconditionalBranch(const MCInst &Inst) const { + return Analysis->isUnconditionalBranch(Inst); + } + + virtual bool isIndirectBranch(const MCInst &Inst) const { + return Analysis->isIndirectBranch(Inst); + } + + virtual bool isIndirectCall(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isCall(const MCInst &Inst) const { + return Analysis->isCall(Inst) || isTailCall(Inst); + } + + virtual bool isReturn(const MCInst &Inst) const { + return Analysis->isReturn(Inst); + } + + virtual bool isTerminator(const MCInst &Inst) const { + return Analysis->isTerminator(Inst); + } + + virtual bool isNoop(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isPrefix(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool deleteREPPrefix(MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isEHLabel(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isPop(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Return the width, in bytes, of the memory access performed by \p Inst, if + /// this is a pop instruction. Return zero otherwise. + virtual int getPopSize(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return 0; + } + + virtual bool isPush(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Return the width, in bytes, of the memory access performed by \p Inst, if + /// this is a push instruction. Return zero otherwise. + virtual int getPushSize(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return 0; + } + + virtual bool isADD64rr(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isSUB(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isLEA64r(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isMOVSX64rm32(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isLeave(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isEnter(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isADRP(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isMoveMem2Reg(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isLoad(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isStore(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isCleanRegXOR(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Interface and basic functionality of a MCInstMatcher. The idea is to make + /// it easy to match one or more MCInsts against a tree-like pattern and + /// extract the fragment operands. Example: + /// + /// auto IndJmpMatcher = + /// matchIndJmp(matchAdd(matchAnyOperand(), matchAnyOperand())); + /// if (!IndJmpMatcher->match(...)) + /// return false; + /// + /// This matches an indirect jump whose target register is defined by an + /// add to form the target address. Matchers should also allow extraction + /// of operands, for example: + /// + /// uint64_t Scale; + /// auto IndJmpMatcher = BC.MIA->matchIndJmp( + /// BC.MIA->matchAnyOperand(), BC.MIA->matchImm(Scale), + /// BC.MIA->matchReg(), BC.MIA->matchAnyOperand()); + /// if (!IndJmpMatcher->match(...)) + /// return false; + /// + /// Here we are interesting in extracting the scale immediate in an indirect + /// jump fragment. + /// + struct MCInstMatcher { + MutableArrayRef InstrWindow; + MutableArrayRef::iterator CurInst; + virtual ~MCInstMatcher() {} + + /// Returns true if the pattern is matched. Needs MCRegisterInfo and + /// MCInstrAnalysis for analysis. InstrWindow contains an array + /// where the last instruction is always the instruction to start matching + /// against a fragment, potentially matching more instructions before it. + /// If OpNum is greater than 0, we will not match against the last + /// instruction itself but against an operand of the last instruction given + /// by the index OpNum. If this operand is a register, we will immediately + /// look for a previous instruction defining this register and match against + /// it instead. This parent member function contains common bookkeeping + /// required to implement this behavior. + virtual bool match(const MCRegisterInfo &MRI, const MCPlusBuilder &MIA, + MutableArrayRef InInstrWindow, int OpNum) { + InstrWindow = InInstrWindow; + CurInst = InstrWindow.end(); + + if (!next()) + return false; + + if (OpNum < 0) + return true; + + if (static_cast(OpNum) >= CurInst->getNumOperands()) + return false; + + const auto &Op = CurInst->getOperand(OpNum); + if (!Op.isReg()) + return true; + + MCPhysReg Reg = Op.getReg(); + while (next()) { + const auto &InstrDesc = MIA.Info->get(CurInst->getOpcode()); + if (InstrDesc.hasDefOfPhysReg(*CurInst, Reg, MRI)) { + InstrWindow = InstrWindow.slice(0, CurInst - InstrWindow.begin() + 1); + return true; + } + } + return false; + } + + /// If successfully matched, calling this function will add an annotation + /// to all instructions that were matched. This is used to easily tag + /// instructions for deletion and implement match-and-replace operations. + virtual void annotate(const MCPlusBuilder &MIA, MCContext &Ctx, + StringRef Annotation) {} + + /// Moves internal instruction iterator to the next instruction, walking + /// backwards for pattern matching (effectively the previous instruction in + /// regular order). + bool next() { + if (CurInst == InstrWindow.begin()) + return false; + --CurInst; + return true; + } + }; + + /// Matches any operand + struct AnyOperandMatcher : MCInstMatcher { + MCOperand &Op; + AnyOperandMatcher(MCOperand &Op) : Op(Op) {} + + bool match(const MCRegisterInfo &MRI, const MCPlusBuilder &MIA, + MutableArrayRef InInstrWindow, int OpNum) override { + auto I = InInstrWindow.end(); + if (I == InInstrWindow.begin()) + return false; + --I; + if (OpNum < 0 || static_cast(OpNum) >= I->getNumOperands()) + return false; + Op = I->getOperand(OpNum); + return true; + } + + }; + + /// Matches operands that are immediates + struct ImmMatcher : MCInstMatcher { + uint64_t &Imm; + ImmMatcher(uint64_t &Imm) : Imm(Imm) {} + + bool match(const MCRegisterInfo &MRI, const MCPlusBuilder &MIA, + MutableArrayRef InInstrWindow, int OpNum) override { + if (!MCInstMatcher::match(MRI, MIA, InInstrWindow, OpNum)) + return false; + + if (OpNum < 0) + return false; + const auto &Op = CurInst->getOperand(OpNum); + if (!Op.isImm()) + return false; + Imm = Op.getImm(); + return true; + } + }; + + /// Matches operands that are MCSymbols + struct SymbolMatcher : MCInstMatcher { + const MCSymbol *&Sym; + SymbolMatcher(const MCSymbol *&Sym) : Sym(Sym) {} + + bool match(const MCRegisterInfo &MRI, const MCPlusBuilder &MIA, + MutableArrayRef InInstrWindow, int OpNum) override { + if (!MCInstMatcher::match(MRI, MIA, InInstrWindow, OpNum)) + return false; + + if (OpNum < 0) + return false; + Sym = MIA.getTargetSymbol(*CurInst, OpNum); + return Sym != nullptr; + } + }; + + /// Matches operands that are registers + struct RegMatcher : MCInstMatcher { + MCPhysReg &Reg; + RegMatcher(MCPhysReg &Reg) : Reg(Reg) {} + + bool match(const MCRegisterInfo &MRI, const MCPlusBuilder &MIA, + MutableArrayRef InInstrWindow, int OpNum) override { + auto I = InInstrWindow.end(); + if (I == InInstrWindow.begin()) + return false; + --I; + if (OpNum < 0 || static_cast(OpNum) >= I->getNumOperands()) + return false; + const auto &Op = I->getOperand(OpNum); + if (!Op.isReg()) + return false; + Reg = Op.getReg(); + return true; + } + }; + + std::unique_ptr matchAnyOperand(MCOperand &Op) const { + return std::unique_ptr(new AnyOperandMatcher(Op)); + } + + std::unique_ptr matchAnyOperand() const { + static MCOperand Unused; + return std::unique_ptr(new AnyOperandMatcher(Unused)); + } + + std::unique_ptr matchReg(MCPhysReg &Reg) const { + return std::unique_ptr(new RegMatcher(Reg)); + } + + std::unique_ptr matchReg() const { + static MCPhysReg Unused; + return std::unique_ptr(new RegMatcher(Unused)); + } + + std::unique_ptr matchImm(uint64_t &Imm) const { + return std::unique_ptr(new ImmMatcher(Imm)); + } + + std::unique_ptr matchImm() const { + static uint64_t Unused; + return std::unique_ptr(new ImmMatcher(Unused)); + } + + std::unique_ptr matchSymbol(const MCSymbol *&Sym) const { + return std::unique_ptr(new SymbolMatcher(Sym)); + } + + std::unique_ptr matchSymbol() const { + static const MCSymbol* Unused; + return std::unique_ptr(new SymbolMatcher(Unused)); + } + + virtual std::unique_ptr + matchIndJmp(std::unique_ptr Target) const { + llvm_unreachable("not implemented"); + return nullptr; + } + + virtual std::unique_ptr + matchIndJmp(std::unique_ptr Base, + std::unique_ptr Scale, + std::unique_ptr Index, + std::unique_ptr Offset) const { + llvm_unreachable("not implemented"); + return nullptr; + } + + virtual std::unique_ptr + matchAdd(std::unique_ptr A, + std::unique_ptr B) const { + llvm_unreachable("not implemented"); + return nullptr; + } + + virtual std::unique_ptr + matchLoadAddr(std::unique_ptr Target) const { + llvm_unreachable("not implemented"); + return nullptr; + } + + virtual std::unique_ptr + matchLoad(std::unique_ptr Base, + std::unique_ptr Scale, + std::unique_ptr Index, + std::unique_ptr Offset) const { + llvm_unreachable("not implemented"); + return nullptr; + } + + /// \brief Given a branch instruction try to get the address the branch + /// targets. Return true on success, and the address in Target. + virtual bool + evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size, + uint64_t &Target) const; + + /// Return true if one of the operands of the \p Inst instruction uses + /// PC-relative addressing. + /// Note that PC-relative branches do not fall into this category. + virtual bool hasPCRelOperand(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Return a number of the operand representing a memory. + /// Return -1 if the instruction doesn't have an explicit memory field. + virtual int getMemoryOperandNo(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return -1; + } + + /// Return true if the instruction is encoded using EVEX (AVX-512). + virtual bool hasEVEXEncoding(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Given an instruction with (compound) memory operand, evaluate and return + /// the corresponding values. Note that the operand could be in any position, + /// but there is an assumption there's only one compound memory operand. + /// Return true upon success, return false if the instruction does not have + /// a memory operand. + /// + /// Since a Displacement field could be either an immediate or an expression, + /// the function sets either \p DispImm or \p DispExpr value. + virtual bool evaluateX86MemoryOperand(const MCInst &Inst, + unsigned *BaseRegNum, + int64_t *ScaleImm, + unsigned *IndexRegNum, + int64_t *DispImm, + unsigned *SegmentRegNum, + const MCExpr **DispExpr = nullptr) const + { + llvm_unreachable("not implemented"); + return false; + } + + /// Given an instruction with memory addressing attempt to statically compute + /// the address being accessed. Return true on success, and the address in + /// \p Target. + /// + /// For RIP-relative addressing the caller is required to pass instruction + /// \p Address and \p Size. + virtual bool evaluateMemOperandTarget(const MCInst &Inst, + uint64_t &Target, + uint64_t Address = 0, + uint64_t Size = 0) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Return operand iterator pointing to displacement in the compound memory + /// operand if such exists. Return Inst.end() otherwise. + virtual MCInst::iterator getMemOperandDisp(MCInst &Inst) const { + llvm_unreachable("not implemented"); + return Inst.end(); + } + + /// Analyze \p Inst and return true if this instruction accesses \p Size + /// bytes of the stack frame at position \p StackOffset. \p IsLoad and + /// \p IsStore are set accordingly. If both are set, it means it is a + /// instruction that reads and updates the same memory location. \p Reg is set + /// to the source register in case of a store or destination register in case + /// of a load. If the store does not use a source register, \p SrcImm will + /// contain the source immediate and \p IsStoreFromReg will be set to false. + /// \p Simple is false if the instruction is not fully understood by + /// companion functions "replaceMemOperandWithImm" or + /// "replaceMemOperandWithReg". + virtual bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore, + bool &IsStoreFromReg, MCPhysReg &Reg, + int32_t &SrcImm, uint16_t &StackPtrReg, + int64_t &StackOffset, uint8_t &Size, + bool &IsSimple, bool &IsIndexed) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Convert a stack accessing load/store instruction in \p Inst to a PUSH + /// or POP saving/restoring the source/dest reg in \p Inst. The original + /// stack offset in \p Inst is ignored. + virtual void changeToPushOrPop(MCInst &Inst) const { + llvm_unreachable("not implemented"); + } + + /// Identify stack adjustment instructions -- those that change the stack + /// pointer by adding or subtracting an immediate. + virtual bool isStackAdjustment(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Use \p Input1 or Input2 as the current value for the input register and + /// put in \p Output the changes incurred by executing \p Inst. Return false + /// if it was not possible to perform the evaluation. + virtual bool evaluateSimple(const MCInst &Inst, int64_t &Output, + std::pair Input1, + std::pair Input2) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From, + MCPhysReg &To) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual MCPhysReg getStackPointer() const { + llvm_unreachable("not implemented"); + return 0; + } + + virtual MCPhysReg getFramePointer() const { + llvm_unreachable("not implemented"); + return 0; + } + + virtual MCPhysReg getFlagsReg() const { + llvm_unreachable("not implemented"); + return 0; + } + + /// Return true if \p Inst is a instruction that copies either the frame + /// pointer or the stack pointer to another general purpose register or + /// writes it to a memory location. + virtual bool escapesVariable(const MCInst &Inst, bool HasFramePointer) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Replace an immediate operand in the instruction \p Inst with a reference + /// of the passed \p Symbol plus \p Addend. If the instruction does not have + /// an immediate operand or has more than one - then return false. Otherwise + /// return true. + virtual bool replaceImmWithSymbol(MCInst &Inst, MCSymbol *Symbol, + int64_t Addend, MCContext *Ctx, + int64_t &Value, uint64_t RelType) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Add \p NewImm to the current immediate operand of \p Inst. If it is a + /// memory accessing instruction, this immediate is the memory address + /// displacement. Otherwise, the target operand is the first immediate + /// operand found in \p Inst. Return false if no imm operand found. + virtual bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Replace the compound memory operand of Inst with an immediate operand. + /// The value of the immediate operand is computed by reading the \p + /// ConstantData array starting from \p offset and assuming little-endianess. + /// Return true on success. The given instruction is modified in place. + virtual bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData, + uint32_t Offset) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Same as replaceMemOperandWithImm, but for registers. + virtual bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Return true if a move instruction moves a register to itself. + virtual bool isRedundantMove(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Return true if the instruction is a tail call. + virtual bool isTailCall(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Return true if the instruction is a call with an exception handling info. + virtual bool isInvoke(const MCInst &Inst) const { + return isCall(Inst) && hasEHInfo(Inst); + } + + /// Return true if \p Inst is an instruction that potentially traps when + /// working with addresses not aligned to the size of the operand. + virtual bool requiresAlignedAddress(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Return true if this instruction has handler and action info. + bool hasEHInfo(const MCInst &Inst) const; + + /// Return handler and action info for invoke instruction. + MCLandingPad getEHInfo(const MCInst &Inst) const; + + // Add handler and action info for call instruction. + void addEHInfo(MCInst &Inst, const MCLandingPad &LP, MCContext *Ctx) const; + + /// Return non-negative GNU_args_size associated with the instruction + /// or -1 if there's no associated info. + int64_t getGnuArgsSize(const MCInst &Inst) const; + + /// Return MCSymbol that represents a target of this instruction at a given + /// operand number \p OpNum. If there's no symbol associated with + /// the operand - return nullptr. + virtual const MCSymbol *getTargetSymbol(const MCInst &Inst, + unsigned OpNum = 0) const { + llvm_unreachable("not implemented"); + return nullptr; + } + + /// Return MCSymbol extracted from a target expression + virtual const MCSymbol *getTargetSymbol(const MCExpr *Expr) const { + return &Expr->getSymbol(); + } + + /// Return MCSymbol/offset extracted from a target expression + virtual std::pair + getTargetSymbolInfo(const MCExpr *Expr) const { + if (auto *SymExpr = dyn_cast(Expr)) { + return std::make_pair(&SymExpr->getSymbol(), 0); + } else if (auto *BinExpr = dyn_cast(Expr)) { + const auto *SymExpr = dyn_cast(BinExpr->getLHS()); + const auto *ConstExpr = dyn_cast(BinExpr->getRHS()); + if (BinExpr->getOpcode() == MCBinaryExpr::Add && SymExpr && ConstExpr) { + return std::make_pair(&SymExpr->getSymbol(), ConstExpr->getValue()); + } + } + return std::make_pair(nullptr, 0); + } + + /// Add the value of GNU_args_size to Inst if it already has EH info. + void addGnuArgsSize(MCInst &Inst, int64_t GnuArgsSize) const; + + /// Return jump table addressed by this instruction. + uint64_t getJumpTable(const MCInst &Inst) const; + + /// Set jump table addressed by this instruction. + bool setJumpTable(MCContext *Ctx, MCInst &Inst, uint64_t Value, + uint16_t IndexReg) const; + + /// Return destination of conditional tail call instruction if \p Inst is one. + Optional getConditionalTailCall(const MCInst &Inst) const; + + /// Mark the \p Instruction as a conditional tail call, and set its + /// destination address if it is known. If \p Instruction was already marked, + /// update its destination with \p Dest. + bool setConditionalTailCall(MCInst &Inst, uint64_t Dest = 0) const; + + /// If \p Inst was marked as a conditional tail call convert it to a regular + /// branch. Return true if the instruction was converted. + bool unsetConditionalTailCall(MCInst &Inst) const; + + /// Replace displacement in compound memory operand with given \p Operand. + virtual bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Return the MCExpr used for absolute references in this target + virtual const MCExpr *getTargetExprFor(MCInst &Inst, const MCExpr *Expr, + MCContext &Ctx, + uint64_t RelType) const { + return Expr; + } + + /// Return a BitVector marking all sub or super registers of \p Reg, including + /// itself. + virtual const BitVector &getAliases(MCPhysReg Reg, + bool OnlySmaller = false) const; + + /// Change \p Regs setting all registers used to pass parameters according + /// to the host abi. Do nothing if not implemented. + virtual BitVector getRegsUsedAsParams() const { + llvm_unreachable("not implemented"); + return BitVector(); + } + + /// Change \p Regs setting all registers used as callee-saved according + /// to the host abi. Do nothing if not implemented. + virtual void getCalleeSavedRegs(BitVector &Regs) const { + llvm_unreachable("not implemented"); + } + + /// Get the default def_in and live_out registers for the function + /// Currently only used for the Stoke optimzation + virtual void getDefaultDefIn(BitVector &Regs) const { + llvm_unreachable("not implemented"); + } + + /// Similar to getDefaultDefIn + virtual void getDefaultLiveOut(BitVector &Regs) const { + llvm_unreachable("not implemented"); + } + + /// Change \p Regs with a bitmask with all general purpose regs + virtual void getGPRegs(BitVector &Regs, bool IncludeAlias = true) const { + llvm_unreachable("not implemented"); + } + + /// Change \p Regs with a bitmask with all general purpose regs that can be + /// encoded without extra prefix bytes. For x86 only. + virtual void getClassicGPRegs(BitVector &Regs) const { + llvm_unreachable("not implemented"); + } + + /// Return the register width in bytes (1, 2, 4 or 8) + virtual uint8_t getRegSize(MCPhysReg Reg) const; + + /// For aliased registers, return an alias of \p Reg that has the width of + /// \p Size bytes + virtual MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const { + llvm_unreachable("not implemented"); + return 0; + } + + /// For X86, return whether this register is an upper 8-bit register, such as + /// AH, BH, etc. + virtual bool isUpper8BitReg(MCPhysReg Reg) const { + llvm_unreachable("not implemented"); + return false; + } + + /// For X86, return whether this instruction has special constraints that + /// prevents it from encoding registers that require a REX prefix. + virtual bool cannotUseREX(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Modifies the set \p Regs by adding registers \p Inst may rewrite. Caller + /// is responsible for passing a valid BitVector with the size equivalent to + /// the number of registers in the target. + /// Since this function is called many times during clobber analysis, it + /// expects the caller to manage BitVector creation to avoid extra overhead. + virtual void getClobberedRegs(const MCInst &Inst, BitVector &Regs) const; + + /// Set of all registers touched by this instruction, including implicit uses + /// and defs. + virtual void getTouchedRegs(const MCInst &Inst, BitVector &Regs) const; + + /// Set of all registers being written to by this instruction -- includes + /// aliases but only if they are strictly smaller than the actual reg + virtual void getWrittenRegs(const MCInst &Inst, BitVector &Regs) const; + + /// Set of all registers being read by this instruction -- includes aliases + /// but only if they are strictly smaller than the actual reg + virtual void getUsedRegs(const MCInst &Inst, BitVector &Regs) const; + + /// Replace displacement in compound memory operand with given \p Label. + bool replaceMemOperandDisp(MCInst &Inst, const MCSymbol *Label, + MCContext *Ctx) const { + return replaceMemOperandDisp( + Inst, MCOperand::createExpr(MCSymbolRefExpr::create(Label, *Ctx))); + } + + /// Returns how many bits we have in this instruction to encode a PC-rel + /// imm. + virtual int getPCRelEncodingSize(MCInst &Inst) const { + llvm_unreachable("not implemented"); + return 0; + } + + /// Replace instruction opcode to be a tail call instead of jump. + virtual bool convertJmpToTailCall(MCInst &Inst, MCContext *Ctx) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Perform any additional actions to transform a (conditional) tail call + /// into a (conditional) jump. Assume the target was already replaced with + /// a local one, so the default is to do nothing more. + virtual bool convertTailCallToJmp(MCInst &Inst) const { + return true; + } + + /// Replace instruction opcode to be a regural call instead of tail call. + virtual bool convertTailCallToCall(MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Modify a direct call instruction \p Inst with an indirect call taking + /// a destination from a memory location pointed by \p TargetLocation symbol. + virtual bool convertCallToIndirectCall(MCInst &Inst, + const MCSymbol *TargetLocation, + MCContext *Ctx) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Replace instruction with a shorter version that could be relaxed later + /// if needed. + virtual bool shortenInstruction(MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Lower a tail call instruction \p Inst if required by target. + virtual bool lowerTailCall(MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Receives a list of MCInst of the basic block to analyze and interpret the + /// terminators of this basic block. TBB must be initialized with the original + /// fall-through for this BB. + virtual bool analyzeBranch(InstructionIterator Begin, + InstructionIterator End, + const MCSymbol *&TBB, + const MCSymbol *&FBB, + MCInst *&CondBranch, + MCInst *&UncondBranch) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Analyze \p Instruction to try and determine what type of indirect branch + /// it is. It is assumed that \p Instruction passes isIndirectBranch(). + /// \p BB is an array of instructions immediately preceding \p Instruction. + /// If \p Instruction can be successfully analyzed, the output parameters + /// will be set to the different components of the branch. \p MemLocInstr + /// is the instruction that loads up the indirect function pointer. It may + /// or may not be same as \p Instruction. + virtual IndirectBranchType analyzeIndirectBranch( + MCInst &Instruction, + InstructionIterator Begin, + InstructionIterator End, + const unsigned PtrSize, + MCInst *&MemLocInstr, + unsigned &BaseRegNum, + unsigned &IndexRegNum, + int64_t &DispValue, + const MCExpr *&DispExpr, + MCInst *&PCRelBaseOut + ) const { + llvm_unreachable("not implemented"); + return IndirectBranchType::UNKNOWN; + } + + virtual bool + analyzeVirtualMethodCall(InstructionIterator Begin, + InstructionIterator End, + std::vector &MethodFetchInsns, + unsigned &VtableRegNum, + unsigned &BaseRegNum, + uint64_t &MethodOffset) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual void createLongJmp(std::vector &Seq, const MCSymbol *Target, + MCContext *Ctx) const { + assert(0 && "not implemented"); + } + + virtual void createShortJmp(std::vector &Seq, const MCSymbol *Target, + MCContext *Ctx) const { + assert(0 && "not implemented"); + } + + virtual int getShortJmpEncodingSize() const { + assert(0 && "not implemented"); + return 0; + } + + virtual int getUncondBranchEncodingSize() const { + assert(0 && "not implemented"); + return 0; + } + + /// Create a no-op instruction. + virtual bool createNoop(MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Create a return instruction. + virtual bool createReturn(MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Creates a new unconditional branch instruction in Inst and set its operand + /// to TBB. + /// + /// Returns true on success. + virtual bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Creates a new tail call instruction in Inst and sets its operand to + /// Target. + /// + /// Returns true on success. + virtual bool createTailCall(MCInst &Inst, const MCSymbol *Target, + MCContext *Ctx) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Creates a trap instruction in Inst. + /// + /// Returns true on success. + virtual bool createTrap(MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Creates an instruction to bump the stack pointer just like a call. + virtual bool createStackPointerIncrement(MCInst &Inst, int Size = 8, + bool NoFlagsClobber = false) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Creates an instruction to move the stack pointer just like a ret. + virtual bool createStackPointerDecrement(MCInst &Inst, int Size = 8, + bool NoFlagsClobber = false) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Create a store instruction using \p StackReg as the base register + /// and \p Offset as the displacement. + virtual bool createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, + int Offset, const MCPhysReg &SrcReg, + int Size) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int Scale, + const MCPhysReg &IndexReg, int Offset, + const MCPhysReg &DstReg, int Size) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Create a fragment of code (sequence of instructions) that load a 32-bit + /// address from memory, zero-extends it to 64 and jump to it (indirect jump). + virtual bool + createIJmp32Frag(SmallVectorImpl &Insts, const MCOperand &BaseReg, + const MCOperand &Scale, const MCOperand &IndexReg, + const MCOperand &Offset, const MCOperand &TmpReg) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Create a load instruction using \p StackReg as the base register + /// and \p Offset as the displacement. + virtual bool createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg, + int Offset, const MCPhysReg &DstReg, + int Size) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Creates a call frame pseudo instruction. A single operand identifies which + /// MCCFIInstruction this MCInst is referring to. + /// + /// Returns true on success. + virtual bool createCFI(MCInst &Inst, int64_t Offset) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Returns true if instruction is a call frame pseudo instruction. + virtual bool isCFI(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Reverses the branch condition in Inst and update its taken target to TBB. + /// + /// Returns true on success. + virtual bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Return canonical branch opcode for a reversible branch opcode. For every + /// opposite branch opcode pair Op <-> OpR this function returns one of the + /// opcodes which is considered a canonical. + virtual unsigned getCanonicalBranchOpcode(unsigned BranchOpcode) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Sets the taken target of the branch instruction to Target. + /// + /// Returns true on success. + virtual bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx) const { + llvm_unreachable("not implemented"); + return false; + } + + virtual bool createEHLabel(MCInst &Inst, const MCSymbol *Label, + MCContext *Ctx) const { + llvm_unreachable("not implemented"); + return false; + } + + /// Store an annotation value on an MCInst. This assumes the annotation + /// is not already present. + template > + const ValueType &addAnnotation(MCContext *Ctx, + MCInst &Inst, + StringRef Name, + const ValueType &Val, + const PrintType &Print = PrintType()) const { + assert(!hasAnnotation(Inst, Name)); + auto PooledName = AnnotationNames.intern(Name); + AnnotationNameRefs.insert(PooledName); + auto A = createSimpleAnnotation(*Ctx, *PooledName, Val, Print); + if (!std::is_trivial::value) { + AnnotationPool.insert(A); + } + Inst.addOperand(MCOperand::createAnnotation(A)); + using AnnotationType = const MCSimpleAnnotation *; + auto &Op = Inst.getOperand(Inst.getNumOperands() - 1); + return static_cast(Op.getAnnotation())->getValue(); + } + + /// Remove Annotation associated with Name. + /// Return true if the annotation were removed, false if the + /// annotation were not present. + bool removeAnnotation(MCInst &Inst, StringRef Name) const; + + /// Remove all annotations from Inst. + void removeAllAnnotations(MCInst &Inst) const; + + /// Rename annotation from Before to After. Return false if + /// the annotation does not exist. + bool renameAnnotation(MCInst &Inst, StringRef Before, StringRef After) const; + + /// Check if the specified annotation exists on this instruction. + bool hasAnnotation(const MCInst &Inst, StringRef Name) const; + + /// Get an annotation. Assumes that the annotation exists. + /// Use hasAnnotation() if the annotation may not exist. + const MCAnnotation *getAnnotation(const MCInst &Inst, StringRef Name) const; + + /// Get an annotation as a specific value. Assumes that the annotation exist. + /// Use hasAnnotation() if the annotation may not exist. + template + const ValueType &getAnnotationAs(const MCInst &Inst, StringRef Name) const { + const auto *Annotation = getAnnotation(Inst, Name); + assert(Annotation->isa(Name)); + using AnnotationType = const MCSimpleAnnotation *; + return static_cast(Annotation)->getValue(); + } + + /// Get an annotation as a specific value. If the annotation does not exist, + /// return the \p DefaultValue. + template const ValueType & + getAnnotationWithDefault(const MCInst &Inst, StringRef Name, + const ValueType &DefaultValue = ValueType()) const { + if (!hasAnnotation(Inst, Name)) + return DefaultValue; + return getAnnotationAs(Inst, Name); + } + + /// Get an annotation as a specific value, but if the annotation does not + /// exist, return errc::result_out_of_range. + template + ErrorOr tryGetAnnotationAs(const MCInst &Inst, + StringRef Name) const { + if (!hasAnnotation(Inst, Name)) + return make_error_code(std::errc::result_out_of_range); + return getAnnotationAs(Inst, Name); + } + + template + ErrorOr tryGetAnnotationAs(MCInst &Inst, + StringRef Name) const { + if (!hasAnnotation(Inst, Name)) + return make_error_code(std::errc::result_out_of_range); + return const_cast(getAnnotationAs(Inst, Name)); + } + + /// Get an annotation as a specific value, but if the annotation does not + /// exist, create a new annotation with the default constructor for that type. + /// Return a non-const ref so caller can freely modify its contents + /// afterwards. + template > ValueType& + getOrCreateAnnotationAs(MCContext *Ctx, + MCInst &Inst, + StringRef Name, + const PrintType &Print = PrintType()) const { + auto Val = tryGetAnnotationAs((const MCInst &)Inst, Name); + if (!Val) { + Val = addAnnotation(Ctx, Inst, Name, ValueType(), Print); + } + return const_cast(*Val); + } + + /// Apply a function to each annotation attached to the given instruction. + template + void forEachAnnotation(const MCInst &Inst, Fn &&Func) const { + for (unsigned I = 0; I < Inst.getNumOperands(); ++I) { + const auto& Op = Inst.getOperand(I); + if (Op.isAnnotation()) { + Func(Op.getAnnotation()); + } + } + } + + /// This method takes an indirect call instruction and splits it up into an + /// equivalent set of instructions that use direct calls for target + /// symbols/addresses that are contained in the Targets vector. This is done + /// by guarding each direct call with a compare instruction to verify that + /// the target is correct. + /// If the VtableAddrs vector is not empty, the call will have the extra + /// load of the method pointer from the vtable eliminated. When non-empty + /// the VtableAddrs vector must be the same size as Targets and include the + /// address of a vtable for each corresponding method call in Targets. The + /// MethodFetchInsns vector holds instructions that are used to load the + /// correct method for the cold call case. + /// + /// The return value is a vector of code snippets (essentially basic blocks). + /// There is a symbol associated with each snippet except for the first. + /// If the original call is not a tail call, the last snippet will have an + /// empty vector of instructions. The label is meant to indicate the basic + /// block where all previous snippets are joined, i.e. the instructions that + /// would immediate follow the original call. + using ICPdata = std::vector>>; + virtual ICPdata indirectCallPromotion( + const MCInst &CallInst, + const std::vector>& Targets, + const std::vector &VtableAddrs, + const std::vector &MethodFetchInsns, + const bool MinimizeCodeSize, + MCContext *Ctx + ) const { + llvm_unreachable("not implemented"); + return ICPdata(); + } + + virtual ICPdata jumpTablePromotion( + const MCInst &IJmpInst, + const std::vector>& Targets, + const std::vector &TargetFetchInsns, + MCContext *Ctx + ) const { + llvm_unreachable("not implemented"); + return ICPdata(); + } + +}; + +} // namespace bolt +} // namespace llvm + +#endif diff --git a/bolt/Passes/AllocCombiner.cpp b/bolt/Passes/AllocCombiner.cpp index 0a1208a7fba5..724dfef9e50e 100644 --- a/bolt/Passes/AllocCombiner.cpp +++ b/bolt/Passes/AllocCombiner.cpp @@ -18,24 +18,24 @@ namespace { bool getStackAdjustmentSize(const BinaryContext &BC, const MCInst &Inst, int64_t &Adjustment) { - return BC.MIA->evaluateSimple(Inst, Adjustment, - std::make_pair(BC.MIA->getStackPointer(), 0LL), + return BC.MIB->evaluateSimple(Inst, Adjustment, + std::make_pair(BC.MIB->getStackPointer(), 0LL), std::make_pair(0, 0LL)); } bool isIndifferentToSP(const MCInst &Inst, const BinaryContext &BC) { - if (BC.MIA->isCFI(Inst)) + if (BC.MIB->isCFI(Inst)) return true; const auto II = BC.MII->get(Inst.getOpcode()); - if (BC.MIA->isTerminator(Inst) || - II.hasImplicitDefOfPhysReg(BC.MIA->getStackPointer(), BC.MRI.get()) || - II.hasImplicitUseOfPhysReg(BC.MIA->getStackPointer())) + if (BC.MIB->isTerminator(Inst) || + II.hasImplicitDefOfPhysReg(BC.MIB->getStackPointer(), BC.MRI.get()) || + II.hasImplicitUseOfPhysReg(BC.MIB->getStackPointer())) return false; for (int I = 0, E = Inst.getNumOperands(); I != E; ++I) { const auto &Operand = Inst.getOperand(I); - if (Operand.isReg() && Operand.getReg() == BC.MIA->getStackPointer()) { + if (Operand.isReg() && Operand.getReg() == BC.MIB->getStackPointer()) { return false; } } @@ -68,8 +68,8 @@ void AllocCombinerPass::combineAdjustments(BinaryContext &BC, continue; // Skip updating Prev int64_t Adjustment{0LL}; - if (!Prev || !BC.MIA->isStackAdjustment(Inst) || - !BC.MIA->isStackAdjustment(*Prev) || + if (!Prev || !BC.MIB->isStackAdjustment(Inst) || + !BC.MIB->isStackAdjustment(*Prev) || !getStackAdjustmentSize(BC, *Prev, Adjustment)) { Prev = &Inst; continue; @@ -82,10 +82,10 @@ void AllocCombinerPass::combineAdjustments(BinaryContext &BC, dbgs() << "Adjustment: " << Adjustment << "\n"; }); - if (BC.MIA->isSUB(Inst)) + if (BC.MIB->isSUB(Inst)) Adjustment = -Adjustment; - BC.MIA->addToImm(Inst, Adjustment, BC.Ctx.get()); + BC.MIB->addToImm(Inst, Adjustment, BC.Ctx.get()); DEBUG({ dbgs() << "After adjustment:\n"; diff --git a/bolt/Passes/BinaryFunctionCallGraph.cpp b/bolt/Passes/BinaryFunctionCallGraph.cpp index 5af734bec450..f15409af030e 100644 --- a/bolt/Passes/BinaryFunctionCallGraph.cpp +++ b/bolt/Passes/BinaryFunctionCallGraph.cpp @@ -171,12 +171,12 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC, // If there is no profiling data the count will be COUNT_NO_PROFILE. auto getCallInfo = [&](const BinaryBasicBlock *BB, const MCInst &Inst) { std::vector> Counts; - const auto *DstSym = BC.MIA->getTargetSymbol(Inst); + const auto *DstSym = BC.MIB->getTargetSymbol(Inst); // If this is an indirect call use perf data directly. - if (!DstSym && BC.MIA->hasAnnotation(Inst, "CallProfile")) { + if (!DstSym && BC.MIB->hasAnnotation(Inst, "CallProfile")) { const auto &ICSP = - BC.MIA->getAnnotationAs(Inst, "CallProfile"); + BC.MIB->getAnnotationAs(Inst, "CallProfile"); for (const auto &CSI : ICSP) { if (!CSI.IsFunction) continue; @@ -241,7 +241,7 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC, for (auto &Inst : *BB) { // Find call instructions and extract target symbols from each one. - if (BC.MIA->isCall(Inst)) { + if (BC.MIB->isCall(Inst)) { const auto CallInfo = getCallInfo(BB, Inst); if (!CallInfo.empty()) { diff --git a/bolt/Passes/BinaryPasses.cpp b/bolt/Passes/BinaryPasses.cpp index e540a5446e54..a7780ca3d14c 100644 --- a/bolt/Passes/BinaryPasses.cpp +++ b/bolt/Passes/BinaryPasses.cpp @@ -250,9 +250,9 @@ void OptimizeBodylessFunctions::analyze( const auto *FirstInstr = BF.front().getFirstNonPseudoInstr(); if (!FirstInstr) return; - if (!BC.MIA->isTailCall(*FirstInstr)) + if (!BC.MIB->isTailCall(*FirstInstr)) return; - const auto *TargetSymbol = BC.MIA->getTargetSymbol(*FirstInstr); + const auto *TargetSymbol = BC.MIB->getTargetSymbol(*FirstInstr); if (!TargetSymbol) return; const auto *Function = BC.getFunctionForSymbol(TargetSymbol); @@ -266,9 +266,9 @@ void OptimizeBodylessFunctions::optimizeCalls(BinaryFunction &BF, BinaryContext &BC) { for (auto *BB : BF.layout()) { for (auto &Inst : *BB) { - if (!BC.MIA->isCall(Inst)) + if (!BC.MIB->isCall(Inst)) continue; - const auto *OriginalTarget = BC.MIA->getTargetSymbol(Inst); + const auto *OriginalTarget = BC.MIB->getTargetSymbol(Inst); if (!OriginalTarget) continue; const auto *Target = OriginalTarget; @@ -288,7 +288,7 @@ void OptimizeBodylessFunctions::optimizeCalls(BinaryFunction &BF, << ": replacing call to " << OriginalTarget->getName() << " by call to " << Target->getName() << " while folding " << CallSites << " call sites\n"); - BC.MIA->replaceBranchTarget(Inst, Target, BC.Ctx.get()); + BC.MIB->replaceBranchTarget(Inst, Target, BC.Ctx.get()); NumOptimizedCallSites += CallSites; if (BB->hasProfile()) { @@ -528,7 +528,7 @@ void ReorderBasicBlocks::splitFunction(BinaryFunction &BF) const { // that the block never throws, it is safe to move the block to // decrease the size of the function. for (auto &Instr : *BB) { - if (BF.getBinaryContext().MIA->isInvoke(Instr)) { + if (BF.getBinaryContext().MIB->isInvoke(Instr)) { BB->setCanOutline(false); break; } @@ -624,7 +624,7 @@ void StripAnnotations::runOnFunctions( for (auto &BB : Function) { for (auto &Inst : BB) { - BC.MIA->removeAllAnnotations(Inst); + BC.MIB->removeAllAnnotations(Inst); } } } @@ -677,11 +677,11 @@ uint64_t fixDoubleJumps(BinaryContext &BC, assert((CondBranch || (!CondBranch && Pred->succ_size() == 1)) && "Predecessor block has inconsistent number of successors"); if (CondBranch && - BC.MIA->getTargetSymbol(*CondBranch) == BB.getLabel()) { - BC.MIA->replaceBranchTarget(*CondBranch, Succ->getLabel(), Ctx); + BC.MIB->getTargetSymbol(*CondBranch) == BB.getLabel()) { + BC.MIB->replaceBranchTarget(*CondBranch, Succ->getLabel(), Ctx); } else if (UncondBranch && - BC.MIA->getTargetSymbol(*UncondBranch) == BB.getLabel()) { - BC.MIA->replaceBranchTarget(*UncondBranch, Succ->getLabel(), Ctx); + BC.MIB->getTargetSymbol(*UncondBranch) == BB.getLabel()) { + BC.MIB->replaceBranchTarget(*UncondBranch, Succ->getLabel(), Ctx); } else if (!UncondBranch) { assert(Function.getBasicBlockAfter(Pred, false) != Succ && "Don't add an explicit jump to a fallthrough block."); @@ -691,8 +691,8 @@ uint64_t fixDoubleJumps(BinaryContext &BC, // Succ will be null in the tail call case. In this case we // need to explicitly add a tail call instruction. auto *Branch = Pred->getLastNonPseudoInstr(); - if (Branch && BC.MIA->isUnconditionalBranch(*Branch)) { - assert(BC.MIA->getTargetSymbol(*Branch) == BB.getLabel()); + if (Branch && BC.MIB->isUnconditionalBranch(*Branch)) { + assert(BC.MIB->getTargetSymbol(*Branch) == BB.getLabel()); Pred->removeSuccessor(&BB); Pred->eraseInstruction(Branch); Pred->addTailCallInstruction(SuccSym); @@ -714,16 +714,16 @@ uint64_t fixDoubleJumps(BinaryContext &BC, continue; auto *Inst = BB.getFirstNonPseudoInstr(); - const bool IsTailCall = BC.MIA->isTailCall(*Inst); + const bool IsTailCall = BC.MIB->isTailCall(*Inst); - if (!BC.MIA->isUnconditionalBranch(*Inst) && !IsTailCall) + if (!BC.MIB->isUnconditionalBranch(*Inst) && !IsTailCall) continue; // If we operate after SCTC make sure it's not a conditional tail call. - if (IsTailCall && BC.MIA->isConditionalBranch(*Inst)) + if (IsTailCall && BC.MIB->isConditionalBranch(*Inst)) continue; - const auto *SuccSym = BC.MIA->getTargetSymbol(*Inst); + const auto *SuccSym = BC.MIB->getTargetSymbol(*Inst); auto *Succ = BB.getSuccessor(); if (((!Succ || &BB == Succ) && !IsTailCall) || (IsTailCall && !SuccSym)) @@ -799,7 +799,7 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC, BF.updateLayoutIndices(); BF.markUnreachable(); - auto &MIA = BC.MIA; + auto &MIB = BC.MIB; uint64_t NumLocalCTCCandidates = 0; uint64_t NumLocalCTCs = 0; uint64_t LocalCTCTakenCount = 0; @@ -820,10 +820,10 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC, continue; auto *Instr = BB->getFirstNonPseudoInstr(); - if (!MIA->isTailCall(*Instr) || BC.MIA->isConditionalBranch(*Instr)) + if (!MIB->isTailCall(*Instr) || BC.MIB->isConditionalBranch(*Instr)) continue; - auto *CalleeSymbol = MIA->getTargetSymbol(*Instr); + auto *CalleeSymbol = MIB->getTargetSymbol(*Instr); if (!CalleeSymbol) continue; @@ -876,7 +876,7 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC, bool BranchForStats; if (CondSucc != BB) { // Patch the new target address into the conditional branch. - MIA->reverseBranchCondition(*CondBranch, CalleeSymbol, BC.Ctx.get()); + MIB->reverseBranchCondition(*CondBranch, CalleeSymbol, BC.Ctx.get()); // Since we reversed the condition on the branch we need to change // the target for the unconditional branch or add a unconditional // branch to the old target. This has to be done manually since @@ -885,7 +885,7 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC, BranchForStats = false; } else { // Change destination of the conditional branch. - MIA->replaceBranchTarget(*CondBranch, CalleeSymbol, BC.Ctx.get()); + MIB->replaceBranchTarget(*CondBranch, CalleeSymbol, BC.Ctx.get()); BranchForStats = true; } const auto Count = PredBB->getBranchInfo(BranchForStats).Count; @@ -893,11 +893,11 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC, Count == BinaryBasicBlock::COUNT_NO_PROFILE ? 0 : Count; // Annotate it, so "isCall" returns true for this jcc - MIA->setConditionalTailCall(*CondBranch); + MIB->setConditionalTailCall(*CondBranch); // Add info abount the conditional tail call frequency, otherwise this // info will be lost when we delete the associated BranchInfo entry - BC.MIA->removeAnnotation(*CondBranch, "CTCTakenCount"); - BC.MIA->addAnnotation(BC.Ctx.get(), *CondBranch, "CTCTakenCount", + BC.MIB->removeAnnotation(*CondBranch, "CTCTakenCount"); + BC.MIB->addAnnotation(BC.Ctx.get(), *CondBranch, "CTCTakenCount", CTCTakenFreq); // Remove the unused successor which may be eliminated later @@ -948,12 +948,12 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC, if (HasFallthrough) PredBB->eraseInstruction(UncondBranch); else - MIA->replaceBranchTarget(*UncondBranch, + MIB->replaceBranchTarget(*UncondBranch, CondSucc->getLabel(), BC.Ctx.get()); } else if (!HasFallthrough) { MCInst Branch; - MIA->createUncondBranch(Branch, CondSucc->getLabel(), BC.Ctx.get()); + MIB->createUncondBranch(Branch, CondSucc->getLabel(), BC.Ctx.get()); PredBB->addInstruction(Branch); } } @@ -1018,7 +1018,7 @@ uint64_t Peepholes::shortenInstructions(BinaryContext &BC, if (opts::Verbosity > 1) { DebugInst = Inst; } - if (BC.MIA->shortenInstruction(Inst)) { + if (BC.MIB->shortenInstruction(Inst)) { if (opts::Verbosity > 1) { outs() << "BOLT-INFO: peephole, shortening:\n" << "BOLT-INFO: "; @@ -1037,9 +1037,9 @@ void Peepholes::addTailcallTraps(BinaryContext &BC, BinaryFunction &Function) { for (auto &BB : Function) { auto *Inst = BB.getLastNonPseudoInstr(); - if (Inst && BC.MIA->isTailCall(*Inst) && BC.MIA->isIndirectBranch(*Inst)) { + if (Inst && BC.MIB->isTailCall(*Inst) && BC.MIB->isIndirectBranch(*Inst)) { MCInst Trap; - if (BC.MIA->createTrap(Trap)) { + if (BC.MIB->createTrap(Trap)) { BB.addInstruction(Trap); ++TailCallTraps; } @@ -1112,7 +1112,7 @@ void Peepholes::runOnFunctions(BinaryContext &BC, bool SimplifyRODataLoads::simplifyRODataLoads( BinaryContext &BC, BinaryFunction &BF) { - auto &MIA = BC.MIA; + auto &MIB = BC.MIB; uint64_t NumLocalLoadsSimplified = 0; uint64_t NumDynamicLocalLoadsSimplified = 0; @@ -1131,9 +1131,9 @@ bool SimplifyRODataLoads::simplifyRODataLoads( // Try to statically evaluate the target memory address; uint64_t TargetAddress; - if (MIA->hasPCRelOperand(Inst)) { + if (MIB->hasPCRelOperand(Inst)) { // Try to find the symbol that corresponds to the PC-relative operand. - auto DispOpI = MIA->getMemOperandDisp(Inst); + auto DispOpI = MIB->getMemOperandDisp(Inst); assert(DispOpI != Inst.end() && "expected PC-relative displacement"); assert(DispOpI->isExpr() && "found PC-relative with non-symbolic displacement"); @@ -1143,7 +1143,7 @@ bool SimplifyRODataLoads::simplifyRODataLoads( uint64_t DisplOffset; std::tie(DisplSymbol, DisplOffset) = - BC.MIA->getTargetSymbolInfo(DispOpI->getExpr()); + BC.MIB->getTargetSymbolInfo(DispOpI->getExpr()); if (!DisplSymbol) continue; @@ -1154,7 +1154,7 @@ bool SimplifyRODataLoads::simplifyRODataLoads( if (!BD) continue; TargetAddress = BD->getAddress() + DisplOffset; - } else if (!MIA->evaluateMemOperandTarget(Inst, TargetAddress)) { + } else if (!MIB->evaluateMemOperandTarget(Inst, TargetAddress)) { continue; } @@ -1174,7 +1174,7 @@ bool SimplifyRODataLoads::simplifyRODataLoads( if (BB->hasProfile()) NumDynamicLocalLoadsFound += BB->getExecutionCount(); - if (MIA->replaceMemOperandWithImm(Inst, ConstantData, Offset)) { + if (MIB->replaceMemOperandWithImm(Inst, ConstantData, Offset)) { ++NumLocalLoadsSimplified; if (BB->hasProfile()) NumDynamicLocalLoadsSimplified += BB->getExecutionCount(); @@ -1547,7 +1547,7 @@ void InstructionLowering::runOnFunctions( for (auto &BFI : BFs) { for (auto &BB : BFI.second) { for (auto &Instruction : BB) { - BC.MIA->lowerTailCall(Instruction); + BC.MIB->lowerTailCall(Instruction); } } } @@ -1563,8 +1563,8 @@ void StripRepRet::runOnFunctions( for (auto &BB : BFI.second) { auto LastInstRIter = BB.getLastNonPseudo(); if (LastInstRIter == BB.rend() || - !BC.MIA->isReturn(*LastInstRIter) || - !BC.MIA->deleteREPPrefix(*LastInstRIter)) + !BC.MIB->isReturn(*LastInstRIter) || + !BC.MIB->deleteREPPrefix(*LastInstRIter)) continue; NumPrefixesRemoved += BB.getKnownExecutionCount(); diff --git a/bolt/Passes/DataflowAnalysis.cpp b/bolt/Passes/DataflowAnalysis.cpp index 5b093d9fdbcb..57f6db9bc49c 100644 --- a/bolt/Passes/DataflowAnalysis.cpp +++ b/bolt/Passes/DataflowAnalysis.cpp @@ -19,8 +19,8 @@ void doForAllPreds(const BinaryContext &BC, const BinaryBasicBlock &BB, return; for (auto Thrower : BB.throwers()) { for (auto &Inst : *Thrower) { - if (!BC.MIA->isInvoke(Inst) || - BC.MIA->getEHInfo(Inst).first != BB.getLabel()) + if (!BC.MIB->isInvoke(Inst) || + BC.MIB->getEHInfo(Inst).first != BB.getLabel()) continue; Task(ProgramPoint(&Inst)); } diff --git a/bolt/Passes/DataflowAnalysis.h b/bolt/Passes/DataflowAnalysis.h index 63d28c88f986..addf8665adfc 100644 --- a/bolt/Passes/DataflowAnalysis.h +++ b/bolt/Passes/DataflowAnalysis.h @@ -233,7 +233,7 @@ protected: } StateTy &getOrCreateStateAt(MCInst &Point) { - return BC.MIA->getOrCreateAnnotationAs( + return BC.MIB->getOrCreateAnnotationAs( BC.Ctx.get(), Point, derived().getAnnotationName(), StatePrinterTy(BC)); } @@ -275,7 +275,7 @@ public: /// Track the state at the end (start) of each MCInst in this function if /// the direction of the dataflow is forward (backward). ErrorOr getStateAt(const MCInst &Point) const { - return BC.MIA->tryGetAnnotationAs( + return BC.MIB->tryGetAnnotationAs( Point, const_derived().getAnnotationName()); } @@ -304,7 +304,7 @@ public: void cleanAnnotations() { for (auto &BB : Func) { for (auto &Inst : BB) { - BC.MIA->removeAnnotation(Inst, derived().getAnnotationName()); + BC.MIB->removeAnnotation(Inst, derived().getAnnotationName()); } } } @@ -358,7 +358,7 @@ public: StateTy StateAtEntry = getOrCreateStateAt(*BB); if (BB->isLandingPad()) { doForAllSuccsOrPreds(*BB, [&](ProgramPoint P) { - if (P.isInst() && BC.MIA->isInvoke(*P.getInst())) + if (P.isInst() && BC.MIB->isInvoke(*P.getInst())) derived().doConfluenceWithLP(StateAtEntry, *getStateAt(P), *P.getInst()); else @@ -388,7 +388,7 @@ public: auto doNext = [&] (MCInst &Inst, const BinaryBasicBlock &BB) { StateTy CurState = derived().computeNext(Inst, *PrevState); - if (Backward && BC.MIA->isInvoke(Inst)) { + if (Backward && BC.MIB->isInvoke(Inst)) { auto *LBB = Func.getLandingPadBBFor(BB, Inst); if (LBB) { auto First = LBB->begin(); diff --git a/bolt/Passes/DominatorAnalysis.h b/bolt/Passes/DominatorAnalysis.h index f807c577eb52..ffd045ef3026 100644 --- a/bolt/Passes/DominatorAnalysis.h +++ b/bolt/Passes/DominatorAnalysis.h @@ -142,7 +142,7 @@ private: BitVector computeNext(const MCInst &Point, const BitVector &Cur) { BitVector Next = Cur; // Gen - if (!this->BC.MIA->isCFI(Point)) { + if (!this->BC.MIB->isCFI(Point)) { Next.set(this->ExprToIdx[&Point]); } return Next; diff --git a/bolt/Passes/FrameAnalysis.cpp b/bolt/Passes/FrameAnalysis.cpp index 2cb72c9bcbce..bcb602d68b00 100644 --- a/bolt/Passes/FrameAnalysis.cpp +++ b/bolt/Passes/FrameAnalysis.cpp @@ -107,7 +107,7 @@ class FrameAccessAnalysis { MCPhysReg Reg{0}; int64_t StackOffset{0}; bool IsIndexed{false}; - if (!BC.MIA->isStackAccess(Inst, FIE.IsLoad, FIE.IsStore, FIE.IsStoreFromReg, + if (!BC.MIB->isStackAccess(Inst, FIE.IsLoad, FIE.IsStore, FIE.IsStoreFromReg, Reg, SrcImm, FIE.StackPtrReg, StackOffset, FIE.Size, FIE.IsSimple, IsIndexed)) { return true; @@ -126,11 +126,11 @@ class FrameAccessAnalysis { if (FIE.IsLoad || FIE.IsStoreFromReg) FIE.RegOrImm = Reg; - if (FIE.StackPtrReg == BC.MIA->getStackPointer() && SPOffset != SPT.EMPTY && + if (FIE.StackPtrReg == BC.MIB->getStackPointer() && SPOffset != SPT.EMPTY && SPOffset != SPT.SUPERPOSITION) { DEBUG(dbgs() << "Adding access via SP while CFA reg is another one\n"); FIE.StackOffset = SPOffset + StackOffset; - } else if (FIE.StackPtrReg == BC.MIA->getFramePointer() && + } else if (FIE.StackPtrReg == BC.MIB->getFramePointer() && FPOffset != SPT.EMPTY && FPOffset != SPT.SUPERPOSITION) { DEBUG(dbgs() << "Adding access via FP while CFA reg is another one\n"); FIE.StackOffset = FPOffset + StackOffset; @@ -171,7 +171,7 @@ public: Prev = &Inst; // Use CFI information to keep track of which register is being used to // access the frame - if (BC.MIA->isCFI(Inst)) { + if (BC.MIB->isCFI(Inst)) { const auto *CFI = BF.getCFIFor(Inst); switch (CFI->getOperation()) { case MCCFIInstruction::OpDefCfa: @@ -206,7 +206,7 @@ public: return true; } - if (BC.MIA->escapesVariable(Inst, SPT.HasFramePointer)) { + if (BC.MIB->escapesVariable(Inst, SPT.HasFramePointer)) { DEBUG(dbgs() << "Leaked stack address, giving up on this function.\n"); DEBUG(dbgs() << "Blame insn: "); DEBUG(Inst.dump()); @@ -228,10 +228,10 @@ void FrameAnalysis::addArgAccessesFor(MCInst &Inst, ArgAccesses &&AA) { } if (AA.AssumeEverything) { // Index 0 in ArgAccessesVector represents an "assumeeverything" entry - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, "ArgAccessEntry", 0U); + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, "ArgAccessEntry", 0U); return; } - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, "ArgAccessEntry", + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, "ArgAccessEntry", (unsigned)ArgAccessesVector.size()); ArgAccessesVector.emplace_back(std::move(AA)); } @@ -250,13 +250,13 @@ void FrameAnalysis::addArgInStackAccessFor(MCInst &Inst, } void FrameAnalysis::addFIEFor(MCInst &Inst, const FrameIndexEntry &FIE) { - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, "FrameAccessEntry", + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, "FrameAccessEntry", (unsigned)FIEVector.size()); FIEVector.emplace_back(FIE); } ErrorOr FrameAnalysis::getArgAccessesFor(const MCInst &Inst) { - if (auto Idx = BC.MIA->tryGetAnnotationAs(Inst, "ArgAccessEntry")) { + if (auto Idx = BC.MIB->tryGetAnnotationAs(Inst, "ArgAccessEntry")) { assert(ArgAccessesVector.size() > *Idx && "Out of bounds"); return ArgAccessesVector[*Idx]; } @@ -265,7 +265,7 @@ ErrorOr FrameAnalysis::getArgAccessesFor(const MCInst &Inst) { ErrorOr FrameAnalysis::getArgAccessesFor(const MCInst &Inst) const { - if (auto Idx = BC.MIA->tryGetAnnotationAs(Inst, "ArgAccessEntry")) { + if (auto Idx = BC.MIB->tryGetAnnotationAs(Inst, "ArgAccessEntry")) { assert(ArgAccessesVector.size() > *Idx && "Out of bounds"); return ArgAccessesVector[*Idx]; } @@ -275,7 +275,7 @@ FrameAnalysis::getArgAccessesFor(const MCInst &Inst) const { ErrorOr FrameAnalysis::getFIEFor(const MCInst &Inst) const { if (auto Idx = - BC.MIA->tryGetAnnotationAs(Inst, "FrameAccessEntry")) { + BC.MIB->tryGetAnnotationAs(Inst, "FrameAccessEntry")) { assert(FIEVector.size() > *Idx && "Out of bounds"); return FIEVector[*Idx]; } @@ -309,11 +309,11 @@ void FrameAnalysis::traverseCG(BinaryFunctionCallGraph &CG) { bool FrameAnalysis::updateArgsTouchedFor(const BinaryFunction &BF, MCInst &Inst, int CurOffset) { - if (!BC.MIA->isCall(Inst)) + if (!BC.MIB->isCall(Inst)) return false; std::set Res; - const auto *TargetSymbol = BC.MIA->getTargetSymbol(Inst); + const auto *TargetSymbol = BC.MIB->getTargetSymbol(Inst); // If indirect call, we conservatively assume it accesses all stack positions if (TargetSymbol == nullptr) { addArgAccessesFor(Inst, ArgAccesses(/*AssumeEverything=*/true)); @@ -339,7 +339,7 @@ bool FrameAnalysis::updateArgsTouchedFor(const BinaryFunction &BF, MCInst &Inst, auto Iter = ArgsTouchedMap.find(Function); bool Changed = false; - if (BC.MIA->isTailCall(Inst) && Iter != ArgsTouchedMap.end()) { + if (BC.MIB->isTailCall(Inst) && Iter != ArgsTouchedMap.end()) { // Ignore checking CurOffset because we can't always reliably determine the // offset specially after an epilogue, where tailcalls happen. It should be // -8. @@ -442,7 +442,7 @@ bool FrameAnalysis::computeArgsAccessed(BinaryFunction &BF) { for (auto &BB : BF) { for (auto &Inst : BB) { - if (BC.MIA->requiresAlignedAddress(Inst)) { + if (BC.MIB->requiresAlignedAddress(Inst)) { FunctionsRequireAlignment.insert(&BF); return true; } @@ -488,8 +488,8 @@ void FrameAnalysis::cleanAnnotations() { for (auto &I : BFs) { for (auto &BB : I.second) { for (auto &Inst : BB) { - BC.MIA->removeAnnotation(Inst, "ArgAccessEntry"); - BC.MIA->removeAnnotation(Inst, "FrameAccessEntry"); + BC.MIB->removeAnnotation(Inst, "ArgAccessEntry"); + BC.MIB->removeAnnotation(Inst, "FrameAccessEntry"); } } } diff --git a/bolt/Passes/FrameOptimizer.cpp b/bolt/Passes/FrameOptimizer.cpp index 94cbb09bca8d..f014c16e6dcf 100644 --- a/bolt/Passes/FrameOptimizer.cpp +++ b/bolt/Passes/FrameOptimizer.cpp @@ -104,7 +104,7 @@ void FrameOptimizerPass::removeUnnecessaryLoads(const RegAnalysis &RA, if (FIEX->StackOffset != FIEY->StackOffset || FIEX->Size != FIEY->Size) continue; // TODO: Change push/pops to stack adjustment instruction - if (BC.MIA->isPop(Inst)) + if (BC.MIB->isPop(Inst)) continue; ++NumRedundantLoads; @@ -116,14 +116,14 @@ void FrameOptimizerPass::removeUnnecessaryLoads(const RegAnalysis &RA, DEBUG(dbgs() << "@BB: " << BB.getName() << "\n"); // Replace load if (FIEY->IsStoreFromReg) { - if (!BC.MIA->replaceMemOperandWithReg(Inst, FIEY->RegOrImm)) { + if (!BC.MIB->replaceMemOperandWithReg(Inst, FIEY->RegOrImm)) { DEBUG(dbgs() << "FAILED to change operand to a reg\n"); break; } ++NumLoadsChangedToReg; - BC.MIA->removeAnnotation(Inst, "FrameAccessEntry"); + BC.MIB->removeAnnotation(Inst, "FrameAccessEntry"); DEBUG(dbgs() << "Changed operand to a reg\n"); - if (BC.MIA->isRedundantMove(Inst)) { + if (BC.MIB->isRedundantMove(Inst)) { ++NumLoadsDeleted; DEBUG(dbgs() << "Created a redundant move\n"); // Delete it! @@ -133,11 +133,11 @@ void FrameOptimizerPass::removeUnnecessaryLoads(const RegAnalysis &RA, char Buf[8] = {0, 0, 0, 0, 0, 0, 0, 0}; support::ulittle64_t::ref(Buf + 0) = FIEY->RegOrImm; DEBUG(dbgs() << "Changing operand to an imm... "); - if (!BC.MIA->replaceMemOperandWithImm(Inst, StringRef(Buf, 8), 0)) { + if (!BC.MIB->replaceMemOperandWithImm(Inst, StringRef(Buf, 8), 0)) { DEBUG(dbgs() << "FAILED\n"); } else { ++NumLoadsChangedToImm; - BC.MIA->removeAnnotation(Inst, "FrameAccessEntry"); + BC.MIB->removeAnnotation(Inst, "FrameAccessEntry"); DEBUG(dbgs() << "Ok\n"); } } @@ -197,7 +197,7 @@ void FrameOptimizerPass::removeUnusedStores(const FrameAnalysis &FA, continue; } // TODO: Change push/pops to stack adjustment instruction - if (BC.MIA->isPush(Inst)) + if (BC.MIB->isPush(Inst)) continue; ++NumRedundantStores; diff --git a/bolt/Passes/IndirectCallPromotion.cpp b/bolt/Passes/IndirectCallPromotion.cpp index 4ed710ca89ab..c14eabd83d8c 100644 --- a/bolt/Passes/IndirectCallPromotion.cpp +++ b/bolt/Passes/IndirectCallPromotion.cpp @@ -169,7 +169,7 @@ IndirectCallPromotion::getCallTargets( if (!opts::ICPJumpTablesByTarget && JT->Type == JumpTable::JTT_PIC) return Targets; const Location From(BF.getSymbol()); - const auto Range = JT->getEntriesForAddress(BC.MIA->getJumpTable(Inst)); + const auto Range = JT->getEntriesForAddress(BC.MIB->getJumpTable(Inst)); assert(JT->Counts.empty() || JT->Counts.size() >= Range.second); JumpTable::JumpInfo DefaultJI; const auto *JI = JT->Counts.empty() ? &DefaultJI : &JT->Counts[Range.first]; @@ -233,7 +233,7 @@ IndirectCallPromotion::getCallTargets( return Targets; } auto ICSP = - BC.MIA->tryGetAnnotationAs(Inst, "CallProfile"); + BC.MIB->tryGetAnnotationAs(Inst, "CallProfile"); if (ICSP) { for (const auto &CSP : ICSP.get()) { Callsite Site(BF, CSP); @@ -308,7 +308,7 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets( int64_t DispValue; const MCExpr *DispExpr; MutableArrayRef Insts(&BB->front(), &CallInst); - const auto Type = BC.MIA->analyzeIndirectBranch( + const auto Type = BC.MIB->analyzeIndirectBranch( CallInst, Insts.begin(), Insts.end(), BC.AsmInfo->getCodePointerSize(), MemLocInstr, BaseReg, IndexReg, DispValue, DispExpr, PCRelBaseOut); @@ -341,7 +341,7 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets( ++TotalIndexBasedCandidates; // Try to get value profiling data for the method load instruction. - auto DataOffset = BC.MIA->tryGetAnnotationAs(*MemLocInstr, + auto DataOffset = BC.MIB->tryGetAnnotationAs(*MemLocInstr, "MemDataOffset"); if (!DataOffset) { @@ -451,7 +451,7 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets( } }); - BC.MIA->getOrCreateAnnotationAs(BC.Ctx.get(), + BC.MIB->getOrCreateAnnotationAs(BC.Ctx.get(), CallInst, "JTIndexReg") = IndexReg; @@ -561,7 +561,7 @@ IndirectCallPromotion::maybeGetVtableAddrs( return MethodInfoType(); MutableArrayRef Insts(&BB->front(), &Inst + 1); - if (!BC.MIA->analyzeVirtualMethodCall(Insts.begin(), + if (!BC.MIB->analyzeVirtualMethodCall(Insts.begin(), Insts.end(), MethodFetchInsns, VtableReg, @@ -588,7 +588,7 @@ IndirectCallPromotion::maybeGetVtableAddrs( ); // Try to get value profiling data for the method load instruction. - auto DataOffset = BC.MIA->tryGetAnnotationAs(*MethodFetchInsns.back(), + auto DataOffset = BC.MIB->tryGetAnnotationAs(*MethodFetchInsns.back(), "MemDataOffset"); if (!DataOffset) { @@ -667,12 +667,12 @@ IndirectCallPromotion::rewriteCall( BinaryFunction &Function, BinaryBasicBlock *IndCallBlock, const MCInst &CallInst, - MCInstrAnalysis::ICPdata &&ICPcode, + MCPlusBuilder::ICPdata &&ICPcode, const std::vector &MethodFetchInsns ) const { // Create new basic blocks with correct code in each one first. std::vector> NewBBs; - const bool IsTailCallOrJT = (BC.MIA->isTailCall(CallInst) || + const bool IsTailCallOrJT = (BC.MIB->isTailCall(CallInst) || Function.getJumpTable(CallInst)); // Move instructions from the tail of the original call block @@ -709,8 +709,8 @@ IndirectCallPromotion::rewriteCall( assert(Sym); auto TBB = Function.createBasicBlock(0, Sym); for (auto &Inst : Insts) { // sanitize new instructions. - if (BC.MIA->isCall(Inst)) - BC.MIA->removeAnnotation(Inst, "CallProfile"); + if (BC.MIB->isCall(Inst)) + BC.MIB->removeAnnotation(Inst, "CallProfile"); } TBB->addInstructions(Insts.begin(), Insts.end()); NewBBs.emplace_back(std::move(TBB)); @@ -942,7 +942,7 @@ IndirectCallPromotion::canPromoteCallsite(const BinaryBasicBlock *BB, if (opts::ICPTopCallsites > 0) { auto &BC = BB->getFunction()->getBinaryContext(); - if (BC.MIA->hasAnnotation(Inst, "DoICP")) { + if (BC.MIB->hasAnnotation(Inst, "DoICP")) { computeStats(TrialN); return TrialN; } @@ -1060,7 +1060,7 @@ IndirectCallPromotion::printCallsiteInfo(const BinaryBasicBlock *BB, const size_t N, uint64_t NumCalls) const { auto &BC = BB->getFunction()->getBinaryContext(); - const bool IsTailCall = BC.MIA->isTailCall(Inst); + const bool IsTailCall = BC.MIB->isTailCall(Inst); const bool IsJumpTable = BB->getFunction()->getJumpTable(Inst); const auto InstIdx = &Inst - &(*BB->begin()); @@ -1132,7 +1132,7 @@ void IndirectCallPromotion::runOnFunctions( bool PrintBB = false; for (auto &Inst : BB) { if (auto Mem = - BC.MIA->tryGetAnnotationAs(Inst, "MemDataOffset")) { + BC.MIB->tryGetAnnotationAs(Inst, "MemDataOffset")) { for (auto &MI : MemData->getMemInfoRange(Mem.get())) { if (MI.Addr.IsSymbol) { PrintBB = true; @@ -1185,9 +1185,9 @@ void IndirectCallPromotion::runOnFunctions( for (auto &Inst : BB) { const bool IsJumpTable = Function.getJumpTable(Inst); const bool HasIndirectCallProfile = - BC.MIA->hasAnnotation(Inst, "CallProfile"); - const bool IsDirectCall = (BC.MIA->isCall(Inst) && - BC.MIA->getTargetSymbol(Inst, 0)); + BC.MIB->hasAnnotation(Inst, "CallProfile"); + const bool IsDirectCall = (BC.MIB->isCall(Inst) && + BC.MIB->getTargetSymbol(Inst, 0)); if (!IsDirectCall && ((HasIndirectCallProfile && !IsJumpTable && OptimizeCalls) || @@ -1225,7 +1225,7 @@ void IndirectCallPromotion::runOnFunctions( // Mark sites to optimize with "DoICP" annotation. for (size_t I = 0; I < Num; ++I) { auto *Inst = IndirectCalls[I].second; - BC.MIA->addAnnotation(BC.Ctx.get(), *Inst, "DoICP", true); + BC.MIB->addAnnotation(BC.Ctx.get(), *Inst, "DoICP", true); } } @@ -1263,12 +1263,12 @@ void IndirectCallPromotion::runOnFunctions( for (unsigned Idx = 0; Idx < BB->size(); ++Idx) { auto &Inst = BB->getInstructionAtIndex(Idx); const auto InstIdx = &Inst - &(*BB->begin()); - const bool IsTailCall = BC.MIA->isTailCall(Inst); + const bool IsTailCall = BC.MIB->isTailCall(Inst); const bool HasIndirectCallProfile = - BC.MIA->hasAnnotation(Inst, "CallProfile"); + BC.MIB->hasAnnotation(Inst, "CallProfile"); const bool IsJumpTable = Function.getJumpTable(Inst); - if (BC.MIA->isCall(Inst)) { + if (BC.MIB->isCall(Inst)) { TotalCalls += BB->getKnownExecutionCount(); } @@ -1277,10 +1277,10 @@ void IndirectCallPromotion::runOnFunctions( continue; // Ignore direct calls. - if (BC.MIA->isCall(Inst) && BC.MIA->getTargetSymbol(Inst, 0)) + if (BC.MIB->isCall(Inst) && BC.MIB->getTargetSymbol(Inst, 0)) continue; - assert((BC.MIA->isCall(Inst) || BC.MIA->isIndirectBranch(Inst)) + assert((BC.MIB->isCall(Inst) || BC.MIB->isIndirectBranch(Inst)) && "expected a call or an indirect jump instruction"); if (IsJumpTable) @@ -1304,7 +1304,7 @@ void IndirectCallPromotion::runOnFunctions( // promoting because we will clobber FLAGS. if (IsJumpTable) { auto State = Info.getLivenessAnalysis().getStateBefore(Inst); - if (!State || (State && (*State)[BC.MIA->getFlagsReg()])) { + if (!State || (State && (*State)[BC.MIB->getFlagsReg()])) { if (opts::Verbosity >= 1) { outs() << "BOLT-INFO: ICP failed in " << Function << " @ " << InstIdx << " in " << BB->getName() @@ -1372,11 +1372,11 @@ void IndirectCallPromotion::runOnFunctions( // Generate new promoted call code for this callsite. auto ICPcode = (IsJumpTable && !opts::ICPJumpTablesByTarget) - ? BC.MIA->jumpTablePromotion(Inst, + ? BC.MIB->jumpTablePromotion(Inst, SymTargets, MethodInfo.second, BC.Ctx.get()) - : BC.MIA->indirectCallPromotion( + : BC.MIB->indirectCallPromotion( Inst, SymTargets, MethodInfo.first, MethodInfo.second, opts::ICPOldCodeSequence, BC.Ctx.get()); diff --git a/bolt/Passes/IndirectCallPromotion.h b/bolt/Passes/IndirectCallPromotion.h index 2d8d8a72ad26..95c671199926 100644 --- a/bolt/Passes/IndirectCallPromotion.h +++ b/bolt/Passes/IndirectCallPromotion.h @@ -218,7 +218,7 @@ class IndirectCallPromotion : public BinaryFunctionPass { BinaryFunction &Function, BinaryBasicBlock *IndCallBlock, const MCInst &CallInst, - MCInstrAnalysis::ICPdata &&ICPcode, + MCPlusBuilder::ICPdata &&ICPcode, const std::vector &MethodFetchInsns) const; BinaryBasicBlock *fixCFG(BinaryContext &BC, diff --git a/bolt/Passes/Inliner.cpp b/bolt/Passes/Inliner.cpp index dc7e62e34b6c..b9ccce2e9193 100644 --- a/bolt/Passes/Inliner.cpp +++ b/bolt/Passes/Inliner.cpp @@ -54,8 +54,8 @@ void InlineSmallFunctions::findInliningCandidates( if (BB.size() > 0 && BB.getNumNonPseudos() <= kMaxInstructions && BB.lp_empty() && - BC.MIA->isReturn(LastInstruction) && - !BC.MIA->isTailCall(LastInstruction)) { + BC.MIB->isReturn(LastInstruction) && + !BC.MIB->isTailCall(LastInstruction)) { InliningCandidates.insert(&Function); } } @@ -90,7 +90,7 @@ void InlineSmallFunctions::findInliningCandidatesAggressive( bool FoundCFI = false; for (const auto BB : Function.layout()) { for (const auto &Inst : *BB) { - if (BC.MIA->isEHLabel(Inst) || BC.MIA->isCFI(Inst)) { + if (BC.MIB->isEHLabel(Inst) || BC.MIB->isCFI(Inst)) { FoundCFI = true; break; } @@ -122,8 +122,8 @@ void InlineSmallFunctions::inlineCall( BinaryBasicBlock &BB, MCInst *CallInst, const BinaryBasicBlock &InlinedFunctionBB) { - assert(BC.MIA->isCall(*CallInst) && "Can only inline a call."); - assert(BC.MIA->isReturn(*InlinedFunctionBB.rbegin()) && + assert(BC.MIB->isCall(*CallInst) && "Can only inline a call."); + assert(BC.MIB->isReturn(*InlinedFunctionBB.rbegin()) && "Inlined function should end with a return."); std::vector InlinedInstance; @@ -133,16 +133,16 @@ void InlineSmallFunctions::inlineCall( // Move stack like 'call' would if needed. if (ShouldAdjustStack) { MCInst StackInc; - BC.MIA->createStackPointerIncrement(StackInc); + BC.MIB->createStackPointerIncrement(StackInc); InlinedInstance.push_back(StackInc); } for (auto Instruction : InlinedFunctionBB) { - if (BC.MIA->isReturn(Instruction)) { + if (BC.MIB->isReturn(Instruction)) { break; } - if (!BC.MIA->isEHLabel(Instruction) && - !BC.MIA->isCFI(Instruction)) { + if (!BC.MIB->isEHLabel(Instruction) && + !BC.MIB->isCFI(Instruction)) { InlinedInstance.push_back(Instruction); } } @@ -150,7 +150,7 @@ void InlineSmallFunctions::inlineCall( // Move stack pointer like 'ret' would. if (ShouldAdjustStack) { MCInst StackDec; - BC.MIA->createStackPointerDecrement(StackDec); + BC.MIB->createStackPointerDecrement(StackDec); InlinedInstance.push_back(StackDec); } @@ -166,7 +166,7 @@ InlineSmallFunctions::inlineCall( const BinaryFunction &InlinedFunction) { // Get the instruction to be replaced with inlined code. MCInst &CallInst = CallerBB->getInstructionAtIndex(CallInstIndex); - assert(BC.MIA->isCall(CallInst) && "Can only inline a call."); + assert(BC.MIB->isCall(CallInst) && "Can only inline a call."); // Point in the function after the inlined code. BinaryBasicBlock *AfterInlinedBB = nullptr; @@ -174,7 +174,7 @@ InlineSmallFunctions::inlineCall( // In case of a tail call we should not remove any ret instructions from the // inlined instance. - bool IsTailCall = BC.MIA->isTailCall(CallInst); + bool IsTailCall = BC.MIB->isTailCall(CallInst); // The first block of the function to be inlined can be merged with the caller // basic block. This cannot happen if there are jumps to the first block. @@ -223,30 +223,30 @@ InlineSmallFunctions::inlineCall( // Copy instructions into the inlined instance. for (auto Instruction : *InlinedFunctionBB) { if (!IsTailCall && - BC.MIA->isReturn(Instruction) && - !BC.MIA->isTailCall(Instruction)) { + BC.MIB->isReturn(Instruction) && + !BC.MIB->isTailCall(Instruction)) { // Skip returns when the caller does a normal call as opposed to a tail // call. IsExitingBlock = true; continue; } if (!IsTailCall && - BC.MIA->isTailCall(Instruction)) { + BC.MIB->isTailCall(Instruction)) { // Convert tail calls to normal calls when the caller does a normal // call. - if (!BC.MIA->convertTailCallToCall(Instruction)) + if (!BC.MIB->convertTailCallToCall(Instruction)) assert(false && "unexpected tail call opcode found"); IsExitingBlock = true; } - if (BC.MIA->isBranch(Instruction) && - !BC.MIA->isIndirectBranch(Instruction)) { + if (BC.MIB->isBranch(Instruction) && + !BC.MIB->isIndirectBranch(Instruction)) { // Convert the branch targets in the branch instructions that will be // added to the inlined instance. const MCSymbol *OldTargetLabel = nullptr; const MCSymbol *OldFTLabel = nullptr; MCInst *CondBranch = nullptr; MCInst *UncondBranch = nullptr; - const bool Result = BC.MIA->analyzeBranch(&Instruction, + const bool Result = BC.MIB->analyzeBranch(&Instruction, &Instruction + 1, OldTargetLabel, OldFTLabel, CondBranch, @@ -263,12 +263,12 @@ InlineSmallFunctions::inlineCall( } } assert(NewTargetLabel); - BC.MIA->replaceBranchTarget(Instruction, NewTargetLabel, BC.Ctx.get()); + BC.MIB->replaceBranchTarget(Instruction, NewTargetLabel, BC.Ctx.get()); } // TODO; Currently we simply ignore CFI instructions but we need to // address them for correctness. - if (!BC.MIA->isEHLabel(Instruction) && - !BC.MIA->isCFI(Instruction)) { + if (!BC.MIB->isEHLabel(Instruction) && + !BC.MIB->isCFI(Instruction)) { InlinedInstanceBB->addInstruction(std::move(Instruction)); } } @@ -433,7 +433,7 @@ bool InlineSmallFunctions::inlineCallsInFunction( for (auto BB : Blocks) { for (auto InstIt = BB->begin(), End = BB->end(); InstIt != End; ++InstIt) { auto &Inst = *InstIt; - if (BC.MIA->isCall(Inst)) { + if (BC.MIB->isCall(Inst)) { TotalDynamicCalls += BB->getExecutionCount(); } } @@ -447,11 +447,11 @@ bool InlineSmallFunctions::inlineCallsInFunction( for (auto InstIt = BB->begin(), End = BB->end(); InstIt != End; ) { auto &Inst = *InstIt; - if (BC.MIA->isCall(Inst) && - !BC.MIA->isTailCall(Inst) && + if (BC.MIB->isCall(Inst) && + !BC.MIB->isTailCall(Inst) && Inst.getNumPrimeOperands() == 1 && Inst.getOperand(0).isExpr()) { - const auto *TargetSymbol = BC.MIA->getTargetSymbol(Inst); + const auto *TargetSymbol = BC.MIB->getTargetSymbol(Inst); assert(TargetSymbol && "target symbol expected for direct call"); const auto *TargetFunction = BC.getFunctionForSymbol(TargetSymbol); if (TargetFunction) { @@ -499,7 +499,7 @@ bool InlineSmallFunctions::inlineCallsInFunctionAggressive( for (auto BB : Blocks) { for (auto InstIt = BB->begin(), End = BB->end(); InstIt != End; ++InstIt) { auto &Inst = *InstIt; - if (BC.MIA->isCall(Inst)) { + if (BC.MIB->isCall(Inst)) { TotalDynamicCalls += BB->getExecutionCount(); } } @@ -514,11 +514,11 @@ bool InlineSmallFunctions::inlineCallsInFunctionAggressive( unsigned InstIndex = 0; for (auto InstIt = BB->begin(); InstIt != BB->end(); ) { auto &Inst = *InstIt; - if (BC.MIA->isCall(Inst) && + if (BC.MIB->isCall(Inst) && Inst.getNumPrimeOperands() == 1 && Inst.getOperand(0).isExpr()) { - assert(!BC.MIA->isInvoke(Inst)); - const auto *TargetSymbol = BC.MIA->getTargetSymbol(Inst); + assert(!BC.MIB->isInvoke(Inst)); + const auto *TargetSymbol = BC.MIB->getTargetSymbol(Inst); assert(TargetSymbol && "target symbol expected for direct call"); const auto *TargetFunction = BC.getFunctionForSymbol(TargetSymbol); if (TargetFunction) { diff --git a/bolt/Passes/JTFootprintReduction.cpp b/bolt/Passes/JTFootprintReduction.cpp index b8ef6e65e920..06c685f08314 100644 --- a/bolt/Passes/JTFootprintReduction.cpp +++ b/bolt/Passes/JTFootprintReduction.cpp @@ -60,11 +60,11 @@ void JTFootprintReduction::checkOpportunities(BinaryContext &BC, uint64_t Scale; // Try a standard indirect jump matcher - auto IndJmpMatcher = BC.MIA->matchIndJmp( - BC.MIA->matchAnyOperand(), BC.MIA->matchImm(Scale), - BC.MIA->matchReg(), BC.MIA->matchAnyOperand()); + auto IndJmpMatcher = BC.MIB->matchIndJmp( + BC.MIB->matchAnyOperand(), BC.MIB->matchImm(Scale), + BC.MIB->matchReg(), BC.MIB->matchAnyOperand()); if (!opts::JTFootprintOnlyPIC && - IndJmpMatcher->match(*BC.MRI, *BC.MIA, + IndJmpMatcher->match(*BC.MRI, *BC.MIB, MutableArrayRef(&*BB.begin(), &Inst + 1), -1) && Scale == 8) { @@ -86,19 +86,19 @@ void JTFootprintReduction::checkOpportunities(BinaryContext &BC, MCPhysReg BaseReg1; MCPhysReg BaseReg2; uint64_t Offset; - auto PICIndJmpMatcher = BC.MIA->matchIndJmp(BC.MIA->matchAdd( - BC.MIA->matchReg(BaseReg1), - BC.MIA->matchLoad(BC.MIA->matchReg(BaseReg2), BC.MIA->matchImm(Scale), - BC.MIA->matchReg(), BC.MIA->matchImm(Offset)))); - auto PICBaseAddrMatcher = BC.MIA->matchIndJmp( - BC.MIA->matchAdd(BC.MIA->matchLoadAddr(BC.MIA->matchSymbol()), - BC.MIA->matchAnyOperand())); + auto PICIndJmpMatcher = BC.MIB->matchIndJmp(BC.MIB->matchAdd( + BC.MIB->matchReg(BaseReg1), + BC.MIB->matchLoad(BC.MIB->matchReg(BaseReg2), BC.MIB->matchImm(Scale), + BC.MIB->matchReg(), BC.MIB->matchImm(Offset)))); + auto PICBaseAddrMatcher = BC.MIB->matchIndJmp( + BC.MIB->matchAdd(BC.MIB->matchLoadAddr(BC.MIB->matchSymbol()), + BC.MIB->matchAnyOperand())); if (!PICIndJmpMatcher->match( - *BC.MRI, *BC.MIA, + *BC.MRI, *BC.MIB, MutableArrayRef(&*BB.begin(), &Inst + 1), -1) || Scale != 4 || BaseReg1 != BaseReg2 || Offset != 0 || !PICBaseAddrMatcher->match( - *BC.MRI, *BC.MIA, + *BC.MRI, *BC.MIB, MutableArrayRef(&*BB.begin(), &Inst + 1), -1)) { BlacklistedJTs.insert(JumpTable); ++IndJmpsDenied; @@ -133,10 +133,10 @@ bool JTFootprintReduction::tryOptimizeNonPIC( uint64_t Scale; MCPhysReg Index; MCOperand Offset; - auto IndJmpMatcher = BC.MIA->matchIndJmp( - BC.MIA->matchAnyOperand(Base), BC.MIA->matchImm(Scale), - BC.MIA->matchReg(Index), BC.MIA->matchAnyOperand(Offset)); - if (!IndJmpMatcher->match(*BC.MRI, *BC.MIA, + auto IndJmpMatcher = BC.MIB->matchIndJmp( + BC.MIB->matchAnyOperand(Base), BC.MIB->matchImm(Scale), + BC.MIB->matchReg(Index), BC.MIB->matchAnyOperand(Offset)); + if (!IndJmpMatcher->match(*BC.MRI, *BC.MIB, MutableArrayRef(&*BB.begin(), &Inst + 1), -1)) { return false; @@ -145,7 +145,7 @@ bool JTFootprintReduction::tryOptimizeNonPIC( assert(Scale == 8 && "Wrong scale"); Scale = 4; - IndJmpMatcher->annotate(*BC.MIA, *BC.Ctx.get(), "DeleteMe"); + IndJmpMatcher->annotate(*BC.MIB, *BC.Ctx.get(), "DeleteMe"); auto &LA = Info.getLivenessAnalysis(); MCPhysReg Reg = LA.scavengeRegAfter(&Inst); @@ -153,9 +153,9 @@ bool JTFootprintReduction::tryOptimizeNonPIC( auto RegOp = MCOperand::createReg(Reg); SmallVector NewFrag; - BC.MIA->createIJmp32Frag(NewFrag, Base, MCOperand::createImm(Scale), + BC.MIB->createIJmp32Frag(NewFrag, Base, MCOperand::createImm(Scale), MCOperand::createReg(Index), Offset, RegOp); - BC.MIA->setJumpTable(BC.Ctx.get(), NewFrag.back(), JTAddr, Index); + BC.MIB->setJumpTable(BC.Ctx.get(), NewFrag.back(), JTAddr, Index); JumpTable->OutputEntrySize = 4; @@ -171,11 +171,11 @@ bool JTFootprintReduction::tryOptimizePIC( MCPhysReg Index; MCOperand Offset; MCOperand JumpTableRef; - auto PICIndJmpMatcher = BC.MIA->matchIndJmp(BC.MIA->matchAdd( - BC.MIA->matchLoadAddr(BC.MIA->matchAnyOperand(JumpTableRef)), - BC.MIA->matchLoad(BC.MIA->matchReg(BaseReg), BC.MIA->matchImm(Scale), - BC.MIA->matchReg(Index), BC.MIA->matchAnyOperand()))); - if (!PICIndJmpMatcher->match(*BC.MRI, *BC.MIA, + auto PICIndJmpMatcher = BC.MIB->matchIndJmp(BC.MIB->matchAdd( + BC.MIB->matchLoadAddr(BC.MIB->matchAnyOperand(JumpTableRef)), + BC.MIB->matchLoad(BC.MIB->matchReg(BaseReg), BC.MIB->matchImm(Scale), + BC.MIB->matchReg(Index), BC.MIB->matchAnyOperand()))); + if (!PICIndJmpMatcher->match(*BC.MRI, *BC.MIB, MutableArrayRef(&*BB.begin(), &Inst + 1), -1)) { return false; @@ -183,15 +183,15 @@ bool JTFootprintReduction::tryOptimizePIC( assert(Scale == 4 && "Wrong scale"); - PICIndJmpMatcher->annotate(*BC.MIA, *BC.Ctx.get(), "DeleteMe"); + PICIndJmpMatcher->annotate(*BC.MIB, *BC.Ctx.get(), "DeleteMe"); auto RegOp = MCOperand::createReg(BaseReg); SmallVector NewFrag; - BC.MIA->createIJmp32Frag(NewFrag, MCOperand::createReg(0), + BC.MIB->createIJmp32Frag(NewFrag, MCOperand::createReg(0), MCOperand::createImm(Scale), MCOperand::createReg(Index), JumpTableRef, RegOp); - BC.MIA->setJumpTable(BC.Ctx.get(), NewFrag.back(), JTAddr, Index); + BC.MIB->setJumpTable(BC.Ctx.get(), NewFrag.back(), JTAddr, Index); JumpTable->OutputEntrySize = 4; // DePICify @@ -209,7 +209,7 @@ void JTFootprintReduction::optimizeFunction(BinaryContext &BC, continue; MCInst &IndJmp = *BB.getLastNonPseudo(); - uint64_t JTAddr = BC.MIA->getJumpTable(IndJmp); + uint64_t JTAddr = BC.MIB->getJumpTable(IndJmp); if (!JTAddr) continue; @@ -232,7 +232,7 @@ void JTFootprintReduction::optimizeFunction(BinaryContext &BC, for (auto &BB : Function) { for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) { - if (BC.MIA->hasAnnotation(*I, "DeleteMe")) + if (BC.MIB->hasAnnotation(*I, "DeleteMe")) BB.eraseInstruction(&*I); } } diff --git a/bolt/Passes/LivenessAnalysis.h b/bolt/Passes/LivenessAnalysis.h index fa8347e44d5b..f658512382df 100644 --- a/bolt/Passes/LivenessAnalysis.h +++ b/bolt/Passes/LivenessAnalysis.h @@ -44,7 +44,7 @@ public: bool isAlive(ProgramPoint PP, MCPhysReg Reg) const { BitVector BV = (*this->getStateAt(PP)); - const BitVector &RegAliases = BC.MIA->getAliases(Reg); + const BitVector &RegAliases = BC.MIB->getAliases(Reg); BV &= RegAliases; return BV.any(); } @@ -61,10 +61,10 @@ public: BitVector BV = *this->getStateAt(P); BV.flip(); BitVector GPRegs(NumRegs, false); - this->BC.MIA->getGPRegs(GPRegs, /*IncludeAlias=*/false); + this->BC.MIB->getGPRegs(GPRegs, /*IncludeAlias=*/false); // Ignore the register used for frame pointer even if it is not alive (it // may be used by CFI which is not represented in our dataflow). - auto FP = BC.MIA->getAliases(BC.MIA->getFramePointer()); + auto FP = BC.MIB->getAliases(BC.MIB->getFramePointer()); FP.flip(); BV &= GPRegs; BV &= FP; @@ -85,11 +85,11 @@ protected: if (BB.succ_size() == 0) { BitVector State(NumRegs, false); if (opts::AssumeABI) { - BC.MIA->getDefaultLiveOut(State); - BC.MIA->getCalleeSavedRegs(State); + BC.MIB->getDefaultLiveOut(State); + BC.MIB->getCalleeSavedRegs(State); } else { State.set(); - State.reset(BC.MIA->getFlagsReg()); + State.reset(BC.MIB->getFlagsReg()); } return State; } @@ -106,11 +106,11 @@ protected: BitVector computeNext(const MCInst &Point, const BitVector &Cur) { BitVector Next = Cur; - bool IsCall = this->BC.MIA->isCall(Point); + bool IsCall = this->BC.MIB->isCall(Point); // Kill auto Written = BitVector(NumRegs, false); if (!IsCall) { - this->BC.MIA->getWrittenRegs(Point, Written); + this->BC.MIB->getWrittenRegs(Point, Written); } else { RA.getInstClobberList(Point, Written); // When clobber list is conservative, it is clobbering all/most registers, @@ -119,12 +119,12 @@ protected: // because we don't really know what's going on. if (RA.isConservative(Written)) { Written.reset(); - BC.MIA->getDefaultLiveOut(Written); + BC.MIB->getDefaultLiveOut(Written); // If ABI is respected, everything except CSRs should be dead after a // call if (opts::AssumeABI) { auto CSR = BitVector(NumRegs, false); - BC.MIA->getCalleeSavedRegs(CSR); + BC.MIB->getCalleeSavedRegs(CSR); CSR.flip(); Written |= CSR; } @@ -133,36 +133,36 @@ protected: Written.flip(); Next &= Written; // Gen - if (!this->BC.MIA->isCFI(Point)) { - if (BC.MIA->isCleanRegXOR(Point)) + if (!this->BC.MIB->isCFI(Point)) { + if (BC.MIB->isCleanRegXOR(Point)) return Next; auto Used = BitVector(NumRegs, false); if (IsCall) { RA.getInstUsedRegsList(Point, Used, /*GetClobbers*/true); if (RA.isConservative(Used)) { - Used = BC.MIA->getRegsUsedAsParams(); - BC.MIA->getDefaultLiveOut(Used); + Used = BC.MIB->getRegsUsedAsParams(); + BC.MIB->getDefaultLiveOut(Used); } } const auto InstInfo = BC.MII->get(Point.getOpcode()); for (unsigned I = 0, E = Point.getNumOperands(); I != E; ++I) { if (!Point.getOperand(I).isReg() || I < InstInfo.getNumDefs()) continue; - Used |= BC.MIA->getAliases(Point.getOperand(I).getReg(), + Used |= BC.MIB->getAliases(Point.getOperand(I).getReg(), /*OnlySmaller=*/false); } for (auto I = InstInfo.getImplicitUses(), E = InstInfo.getImplicitUses() + InstInfo.getNumImplicitUses(); I != E; ++I) { - Used |= BC.MIA->getAliases(*I, false); + Used |= BC.MIB->getAliases(*I, false); } if (IsCall && - (!BC.MIA->isTailCall(Point) || !BC.MIA->isConditionalBranch(Point))) { + (!BC.MIB->isTailCall(Point) || !BC.MIB->isConditionalBranch(Point))) { // Never gen FLAGS from a non-conditional call... this is overly // conservative - Used.reset(BC.MIA->getFlagsReg()); + Used.reset(BC.MIB->getFlagsReg()); } Next |= Used; } diff --git a/bolt/Passes/LongJmp.cpp b/bolt/Passes/LongJmp.cpp index 5eecd5363d8d..45f851265387 100644 --- a/bolt/Passes/LongJmp.cpp +++ b/bolt/Passes/LongJmp.cpp @@ -34,7 +34,7 @@ createNewStub(const BinaryContext &BC, BinaryFunction &Func, auto *StubSym = BC.Ctx->createTempSymbol("Stub", true); auto StubBB = Func.createBasicBlock(0, StubSym); std::vector Seq; - BC.MIA->createLongJmp(Seq, TgtSym, BC.Ctx.get()); + BC.MIB->createLongJmp(Seq, TgtSym, BC.Ctx.get()); StubBB->addInstructions(Seq.begin(), Seq.end()); StubBB->setExecutionCount(0); return std::make_pair(std::move(StubBB), StubSym); @@ -43,7 +43,7 @@ createNewStub(const BinaryContext &BC, BinaryFunction &Func, void shrinkStubToShortJmp(const BinaryContext &BC, BinaryBasicBlock &StubBB, const MCSymbol *Tgt) { std::vector Seq; - BC.MIA->createShortJmp(Seq, Tgt, BC.Ctx.get()); + BC.MIB->createShortJmp(Seq, Tgt, BC.Ctx.get()); StubBB.clear(); StubBB.addInstructions(Seq.begin(), Seq.end()); } @@ -51,9 +51,9 @@ void shrinkStubToShortJmp(const BinaryContext &BC, BinaryBasicBlock &StubBB, void shrinkStubToSingleInst(const BinaryContext &BC, BinaryBasicBlock &StubBB, const MCSymbol *Tgt, bool TgtIsFunc) { MCInst Inst; - BC.MIA->createUncondBranch(Inst, Tgt, BC.Ctx.get()); + BC.MIB->createUncondBranch(Inst, Tgt, BC.Ctx.get()); if (TgtIsFunc) - BC.MIA->convertJmpToTailCall(Inst, BC.Ctx.get()); + BC.MIB->convertJmpToTailCall(Inst, BC.Ctx.get()); StubBB.clear(); StubBB.addInstruction(Inst); } @@ -77,7 +77,7 @@ LongJmpPass::replaceTargetWithStub(const BinaryContext &BC, BinaryFunction &Func, BinaryBasicBlock &BB, MCInst &Inst) { std::unique_ptr NewBB; - auto TgtSym = BC.MIA->getTargetSymbol(Inst); + auto TgtSym = BC.MIB->getTargetSymbol(Inst); assert (TgtSym && "getTargetSymbol failed"); BinaryBasicBlock::BinaryBranchInfo BI{0, 0}; @@ -87,14 +87,14 @@ LongJmpPass::replaceTargetWithStub(const BinaryContext &BC, if (TgtBB && TgtBB->isCold() == BB.isCold()) { // Suppose we have half the available space to account for increase in the // function size due to extra blocks being inserted (conservative estimate) - auto BitsAvail = BC.MIA->getPCRelEncodingSize(Inst) - 2; + auto BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 2; uint64_t Mask = ~((1ULL << BitsAvail) - 1); if (!(Func.getMaxSize() & Mask)) return nullptr; // This is a special case for fixBranches, which is usually free to swap // targets when a block has two successors. The other successor may not // fit in this instruction as well. - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, "DoNotChangeTarget", true); + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, "DoNotChangeTarget", true); } BinaryBasicBlock *StubBB = @@ -128,7 +128,7 @@ LongJmpPass::replaceTargetWithStub(const BinaryContext &BC, StubBB->setEntryPoint(true); } } - BC.MIA->replaceBranchTarget(Inst, StubSymbol, BC.Ctx.get()); + BC.MIB->replaceBranchTarget(Inst, StubSymbol, BC.Ctx.get()); ++StubRefCount[StubBB]; StubBits[StubBB] = BC.AsmInfo->getCodePointerSize() * 8; @@ -145,8 +145,8 @@ LongJmpPass::replaceTargetWithStub(const BinaryContext &BC, namespace { bool shouldInsertStub(const BinaryContext &BC, const MCInst &Inst) { - return (BC.MIA->isBranch(Inst) || BC.MIA->isCall(Inst)) && - !BC.MIA->isIndirectBranch(Inst) && !BC.MIA->isIndirectCall(Inst); + return (BC.MIB->isBranch(Inst) || BC.MIB->isCall(Inst)) && + !BC.MIB->isIndirectBranch(Inst) && !BC.MIB->isIndirectCall(Inst); } } @@ -166,8 +166,8 @@ void LongJmpPass::insertStubs(const BinaryContext &BC, BinaryFunction &Func) { // Insert stubs close to the patched BB if call, but far away from the // hot path if a branch, since this branch target is the cold region. BinaryBasicBlock *InsertionPoint = &BB; - if (!BC.MIA->isCall(Inst) && Frontier && !BB.isCold()) { - auto BitsAvail = BC.MIA->getPCRelEncodingSize(Inst) - 2; + if (!BC.MIB->isCall(Inst) && Frontier && !BB.isCold()) { + auto BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 2; uint64_t Mask = ~((1ULL << BitsAvail) - 1); if (!(Func.getMaxSize() & Mask)) InsertionPoint = Frontier; @@ -314,7 +314,7 @@ void LongJmpPass::removeStubRef(const BinaryContext &BC, BinaryBasicBlock *StubBB, const MCSymbol *Target, BinaryBasicBlock *TgtBB) { - BC.MIA->replaceBranchTarget(Inst, Target, BC.Ctx.get()); + BC.MIB->replaceBranchTarget(Inst, Target, BC.Ctx.get()); --StubRefCount[StubBB]; assert(StubRefCount[StubBB] >= 0 && "Ref count is lost"); @@ -336,7 +336,7 @@ void LongJmpPass::removeStubRef(const BinaryContext &BC, bool LongJmpPass::usesStub(const BinaryContext &BC, const BinaryFunction &Func, const MCInst &Inst) const { - auto TgtSym = BC.MIA->getTargetSymbol(Inst); + auto TgtSym = BC.MIB->getTargetSymbol(Inst); auto *TgtBB = Func.getBasicBlockForLabel(TgtSym); auto Iter = Stubs.find(&Func); if (Iter != Stubs.end()) @@ -383,33 +383,33 @@ bool LongJmpPass::removeOrShrinkStubs(const BinaryContext &BC, // Compute DoNotChangeTarget annotation, when fixBranches cannot swap // targets - if (BC.MIA->isConditionalBranch(Inst) && BB.succ_size() == 2) { + if (BC.MIB->isConditionalBranch(Inst) && BB.succ_size() == 2) { auto *SuccBB = BB.getConditionalSuccessor(false); bool IsStub = false; auto Iter = Stubs.find(&Func); if (Iter != Stubs.end()) IsStub = Iter->second.count(SuccBB); auto *RealTargetSym = - IsStub ? BC.MIA->getTargetSymbol(*SuccBB->begin()) : nullptr; + IsStub ? BC.MIB->getTargetSymbol(*SuccBB->begin()) : nullptr; if (IsStub) SuccBB = Func.getBasicBlockForLabel(RealTargetSym); uint64_t Offset = getSymbolAddress(BC, RealTargetSym, SuccBB); - auto BitsAvail = BC.MIA->getPCRelEncodingSize(Inst) - 1; + auto BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1; uint64_t Mask = ~((1ULL << BitsAvail) - 1); if ((Offset & Mask) && - !BC.MIA->hasAnnotation(Inst, "DoNotChangeTarget")) { - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, "DoNotChangeTarget", true); + !BC.MIB->hasAnnotation(Inst, "DoNotChangeTarget")) { + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, "DoNotChangeTarget", true); } else if ((!(Offset & Mask)) && - BC.MIA->hasAnnotation(Inst, "DoNotChangeTarget")) { - BC.MIA->removeAnnotation(Inst, "DoNotChangeTarget"); + BC.MIB->hasAnnotation(Inst, "DoNotChangeTarget")) { + BC.MIB->removeAnnotation(Inst, "DoNotChangeTarget"); } } - auto StubSym = BC.MIA->getTargetSymbol(Inst); + auto StubSym = BC.MIB->getTargetSymbol(Inst); auto *StubBB = Func.getBasicBlockForLabel(StubSym); - auto *RealTargetSym = BC.MIA->getTargetSymbol(*StubBB->begin()); + auto *RealTargetSym = BC.MIB->getTargetSymbol(*StubBB->begin()); auto *TgtBB = Func.getBasicBlockForLabel(RealTargetSym); - auto BitsAvail = BC.MIA->getPCRelEncodingSize(Inst) - 1; + auto BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1; uint64_t Mask = ~((1ULL << BitsAvail) - 1); uint64_t Offset = getSymbolAddress(BC, RealTargetSym, TgtBB); if (DotAddress > Offset) @@ -425,8 +425,8 @@ bool LongJmpPass::removeOrShrinkStubs(const BinaryContext &BC, } } - auto RangeShortJmp = BC.MIA->getShortJmpEncodingSize(); - auto RangeSingleInstr = BC.MIA->getUncondBranchEncodingSize(); + auto RangeShortJmp = BC.MIB->getShortJmpEncodingSize(); + auto RangeSingleInstr = BC.MIB->getUncondBranchEncodingSize(); uint64_t ShortJmpMask = ~((1ULL << RangeShortJmp) - 1); uint64_t SingleInstrMask = ~((1ULL << (RangeSingleInstr - 1)) - 1); // Shrink stubs from 64 to 32 or 28 bit whenever possible @@ -440,7 +440,7 @@ bool LongJmpPass::removeOrShrinkStubs(const BinaryContext &BC, continue; // Attempt to tight to short jmp - auto *RealTargetSym = BC.MIA->getTargetSymbol(*BB.begin()); + auto *RealTargetSym = BC.MIB->getTargetSymbol(*BB.begin()); auto *TgtBB = Func.getBasicBlockForLabel(RealTargetSym); uint64_t DotAddress = BBAddresses[&BB]; uint64_t TgtAddress = getSymbolAddress(BC, RealTargetSym, TgtBB); diff --git a/bolt/Passes/PLTCall.cpp b/bolt/Passes/PLTCall.cpp index 219ecc10f363..8d8351261ca1 100644 --- a/bolt/Passes/PLTCall.cpp +++ b/bolt/Passes/PLTCall.cpp @@ -65,15 +65,15 @@ void PLTCall::runOnFunctions( continue; for (auto &Instr : *BB) { - if (!BC.MIA->isCall(Instr)) + if (!BC.MIB->isCall(Instr)) continue; - const auto *CallSymbol = BC.MIA->getTargetSymbol(Instr); + const auto *CallSymbol = BC.MIB->getTargetSymbol(Instr); if (!CallSymbol) continue; const auto *CalleeBF = BC.getFunctionForSymbol(CallSymbol); if (!CalleeBF || !CalleeBF->isPLTFunction()) continue; - BC.MIA->convertCallToIndirectCall(Instr, + BC.MIB->convertCallToIndirectCall(Instr, CalleeBF->getPLTSymbol(), BC.Ctx.get()); ++NumCallsOptimized; diff --git a/bolt/Passes/ReachingDefOrUse.h b/bolt/Passes/ReachingDefOrUse.h index 01ed496106fb..67701948de2d 100644 --- a/bolt/Passes/ReachingDefOrUse.h +++ b/bolt/Passes/ReachingDefOrUse.h @@ -45,7 +45,7 @@ public: if (Def) { RA.getInstClobberList(**I, BV); } else { - this->BC.MIA->getTouchedRegs(**I, BV); + this->BC.MIB->getTouchedRegs(**I, BV); } if (BV[Reg]) return true; @@ -102,7 +102,7 @@ protected: if (Def) RA.getInstClobberList(*Y, YClobbers); else - this->BC.MIA->getTouchedRegs(*Y, YClobbers); + this->BC.MIB->getTouchedRegs(*Y, YClobbers); // X kills Y if it clobbers Y completely -- this is a conservative approach. // In practice, we may produce use-def links that may not exist. XClobbers &= YClobbers; @@ -119,7 +119,7 @@ protected: } } // Gen - if (!this->BC.MIA->isCFI(Point)) { + if (!this->BC.MIB->isCFI(Point)) { Next.set(this->ExprToIdx[&Point]); } return Next; diff --git a/bolt/Passes/ReachingInsns.h b/bolt/Passes/ReachingInsns.h index 3706380042b5..6437af0e8953 100644 --- a/bolt/Passes/ReachingInsns.h +++ b/bolt/Passes/ReachingInsns.h @@ -79,7 +79,7 @@ protected: BitVector computeNext(const MCInst &Point, const BitVector &Cur) { BitVector Next = Cur; // Gen - if (!this->BC.MIA->isCFI(Point)) { + if (!this->BC.MIB->isCFI(Point)) { Next.set(this->ExprToIdx[&Point]); } return Next; diff --git a/bolt/Passes/RegAnalysis.cpp b/bolt/Passes/RegAnalysis.cpp index 8207ec9718e8..994da13838fc 100644 --- a/bolt/Passes/RegAnalysis.cpp +++ b/bolt/Passes/RegAnalysis.cpp @@ -93,7 +93,7 @@ void RegAnalysis::beConservative(BitVector &Result) const { Result.set(); } else { BitVector BV(BC.MRI->getNumRegs(), false); - BC.MIA->getCalleeSavedRegs(BV); + BC.MIB->getCalleeSavedRegs(BV); BV.flip(); Result |= BV; } @@ -104,7 +104,7 @@ bool RegAnalysis::isConservative(BitVector &Vec) const { return Vec.all(); } else { BitVector BV(BC.MRI->getNumRegs(), false); - BC.MIA->getCalleeSavedRegs(BV); + BC.MIB->getCalleeSavedRegs(BV); BV |= Vec; return BV.all(); } @@ -112,15 +112,15 @@ bool RegAnalysis::isConservative(BitVector &Vec) const { void RegAnalysis::getInstUsedRegsList(const MCInst &Inst, BitVector &RegSet, bool GetClobbers) const { - if (!BC.MIA->isCall(Inst)) { + if (!BC.MIB->isCall(Inst)) { if (GetClobbers) - BC.MIA->getClobberedRegs(Inst, RegSet); + BC.MIB->getClobberedRegs(Inst, RegSet); else - BC.MIA->getUsedRegs(Inst, RegSet); + BC.MIB->getUsedRegs(Inst, RegSet); return; } - const auto *TargetSymbol = BC.MIA->getTargetSymbol(Inst); + const auto *TargetSymbol = BC.MIB->getTargetSymbol(Inst); // If indirect call, we know nothing if (TargetSymbol == nullptr) { beConservative(RegSet); diff --git a/bolt/Passes/RegReAssign.cpp b/bolt/Passes/RegReAssign.cpp index 848147481132..cd01b669f918 100644 --- a/bolt/Passes/RegReAssign.cpp +++ b/bolt/Passes/RegReAssign.cpp @@ -38,8 +38,8 @@ namespace bolt { void RegReAssign::swap(BinaryContext &BC, BinaryFunction &Function, MCPhysReg A, MCPhysReg B) { - const BitVector &AliasA = BC.MIA->getAliases(A, false); - const BitVector &AliasB = BC.MIA->getAliases(B, false); + const BitVector &AliasA = BC.MIB->getAliases(A, false); + const BitVector &AliasB = BC.MIB->getAliases(B, false); // Regular instructions for (auto &BB : Function) { @@ -51,14 +51,14 @@ void RegReAssign::swap(BinaryContext &BC, BinaryFunction &Function, MCPhysReg A, auto Reg = Operand.getReg(); if (AliasA.test(Reg)) { - Operand.setReg(BC.MIA->getAliasSized(B, BC.MIA->getRegSize(Reg))); + Operand.setReg(BC.MIB->getAliasSized(B, BC.MIB->getRegSize(Reg))); --StaticBytesSaved; DynBytesSaved -= BB.getKnownExecutionCount(); continue; } if (!AliasB.test(Reg)) continue; - Operand.setReg(BC.MIA->getAliasSized(A, BC.MIA->getRegSize(Reg))); + Operand.setReg(BC.MIB->getAliasSized(A, BC.MIB->getRegSize(Reg))); ++StaticBytesSaved; DynBytesSaved += BB.getKnownExecutionCount(); } @@ -69,7 +69,7 @@ void RegReAssign::swap(BinaryContext &BC, BinaryFunction &Function, MCPhysReg A, DenseSet Changed; for (auto &BB : Function) { for (auto &Inst : BB) { - if (!BC.MIA->isCFI(Inst)) + if (!BC.MIB->isCFI(Inst)) continue; auto *CFI = Function.getCFIFor(Inst); if (Changed.count(CFI)) @@ -82,10 +82,10 @@ void RegReAssign::swap(BinaryContext &BC, BinaryFunction &Function, MCPhysReg A, const MCPhysReg Reg2 = BC.MRI->getLLVMRegNum(CFIReg2, /*isEH=*/false); if (AliasA.test(Reg2)) { CFI->setRegister2(BC.MRI->getDwarfRegNum( - BC.MIA->getAliasSized(B, BC.MIA->getRegSize(Reg2)), false)); + BC.MIB->getAliasSized(B, BC.MIB->getRegSize(Reg2)), false)); } else if (AliasB.test(Reg2)) { CFI->setRegister2(BC.MRI->getDwarfRegNum( - BC.MIA->getAliasSized(A, BC.MIA->getRegSize(Reg2)), false)); + BC.MIB->getAliasSized(A, BC.MIB->getRegSize(Reg2)), false)); } } // Fall-through @@ -102,10 +102,10 @@ void RegReAssign::swap(BinaryContext &BC, BinaryFunction &Function, MCPhysReg A, const MCPhysReg Reg = BC.MRI->getLLVMRegNum(CFIReg, /*isEH=*/false); if (AliasA.test(Reg)) { CFI->setRegister(BC.MRI->getDwarfRegNum( - BC.MIA->getAliasSized(B, BC.MIA->getRegSize(Reg)), false)); + BC.MIB->getAliasSized(B, BC.MIB->getRegSize(Reg)), false)); } else if (AliasB.test(Reg)) { CFI->setRegister(BC.MRI->getDwarfRegNum( - BC.MIA->getAliasSized(A, BC.MIA->getRegSize(Reg)), false)); + BC.MIB->getAliasSized(A, BC.MIB->getRegSize(Reg)), false)); } break; } @@ -122,14 +122,14 @@ void RegReAssign::rankRegisters(BinaryContext &BC, BinaryFunction &Function) { for (auto &BB : Function) { for (auto &Inst : BB) { - const bool CannotUseREX = BC.MIA->cannotUseREX(Inst); + const bool CannotUseREX = BC.MIB->cannotUseREX(Inst); const auto &Desc = BC.MII->get(Inst.getOpcode()); // Disallow substituitions involving regs in implicit uses lists const auto *ImplicitUses = Desc.getImplicitUses(); while (ImplicitUses && *ImplicitUses) { const size_t RegEC = - BC.MIA->getAliases(*ImplicitUses, false).find_first(); + BC.MIB->getAliases(*ImplicitUses, false).find_first(); RegScore[RegEC] = std::numeric_limits::min(); ++ImplicitUses; @@ -139,7 +139,7 @@ void RegReAssign::rankRegisters(BinaryContext &BC, BinaryFunction &Function) { const auto *ImplicitDefs = Desc.getImplicitDefs(); while (ImplicitDefs && *ImplicitDefs) { const size_t RegEC = - BC.MIA->getAliases(*ImplicitDefs, false).find_first(); + BC.MIB->getAliases(*ImplicitDefs, false).find_first(); RegScore[RegEC] = std::numeric_limits::min(); ++ImplicitDefs; @@ -154,7 +154,7 @@ void RegReAssign::rankRegisters(BinaryContext &BC, BinaryFunction &Function) { continue; auto Reg = Operand.getReg(); - size_t RegEC = BC.MIA->getAliases(Reg, false).find_first(); + size_t RegEC = BC.MIB->getAliases(Reg, false).find_first(); if (RegEC == 0) continue; @@ -166,7 +166,7 @@ void RegReAssign::rankRegisters(BinaryContext &BC, BinaryFunction &Function) { } // Unsupported substitution, cannot swap BH with R* regs, bail - if (BC.MIA->isUpper8BitReg(Reg) && ClassicCSR.test(Reg)) { + if (BC.MIB->isUpper8BitReg(Reg) && ClassicCSR.test(Reg)) { RegScore[RegEC] = std::numeric_limits::min(); continue; @@ -238,9 +238,9 @@ void RegReAssign::aggressivePassOverFunction(BinaryContext &BC, ProgramPoint::getFirstPointAt(BB)); } // Mark frame pointer alive because of CFI - AliveAtStart |= BC.MIA->getAliases(BC.MIA->getFramePointer(), false); + AliveAtStart |= BC.MIB->getAliases(BC.MIB->getFramePointer(), false); // Never touch return registers - BC.MIA->getDefaultLiveOut(AliveAtStart); + BC.MIB->getDefaultLiveOut(AliveAtStart); // Try swapping more profitable options first auto Begin = RankedRegs.begin(); @@ -266,7 +266,7 @@ void RegReAssign::aggressivePassOverFunction(BinaryContext &BC, } BitVector AnyAliasAlive = AliveAtStart; - AnyAliasAlive &= BC.MIA->getAliases(ClassicReg); + AnyAliasAlive &= BC.MIB->getAliases(ClassicReg); if (AnyAliasAlive.any()) { DEBUG(dbgs() << " Bailed on " << BC.MRI->getName(ClassicReg) << " with " << BC.MRI->getName(ExtReg) @@ -275,7 +275,7 @@ void RegReAssign::aggressivePassOverFunction(BinaryContext &BC, continue; } AnyAliasAlive = AliveAtStart; - AnyAliasAlive &= BC.MIA->getAliases(ExtReg); + AnyAliasAlive &= BC.MIB->getAliases(ExtReg); if (AnyAliasAlive.any()) { DEBUG(dbgs() << " Bailed on " << BC.MRI->getName(ClassicReg) << " with " << BC.MRI->getName(ExtReg) @@ -342,7 +342,7 @@ void RegReAssign::setupAggressivePass(BinaryContext &BC, RA.reset(new RegAnalysis(BC, BFs, *CG)); GPRegs = BitVector(BC.MRI->getNumRegs(), false); - BC.MIA->getGPRegs(GPRegs); + BC.MIB->getGPRegs(GPRegs); } void RegReAssign::setupConservativePass( @@ -353,14 +353,14 @@ void RegReAssign::setupConservativePass( ClassicCSR = BitVector(BC.MRI->getNumRegs(), false); ExtendedCSR = BitVector(BC.MRI->getNumRegs(), false); // Never consider the frame pointer - BC.MIA->getClassicGPRegs(ClassicRegs); + BC.MIB->getClassicGPRegs(ClassicRegs); ClassicRegs.flip(); - ClassicRegs |= BC.MIA->getAliases(BC.MIA->getFramePointer(), false); + ClassicRegs |= BC.MIB->getAliases(BC.MIB->getFramePointer(), false); ClassicRegs.flip(); - BC.MIA->getCalleeSavedRegs(CalleeSaved); + BC.MIB->getCalleeSavedRegs(CalleeSaved); ClassicCSR |= ClassicRegs; ClassicCSR &= CalleeSaved; - BC.MIA->getClassicGPRegs(ClassicRegs); + BC.MIB->getClassicGPRegs(ClassicRegs); ExtendedCSR |= ClassicRegs; ExtendedCSR.flip(); ExtendedCSR &= CalleeSaved; diff --git a/bolt/Passes/ShrinkWrapping.cpp b/bolt/Passes/ShrinkWrapping.cpp index e0f5dabb44f5..a26a61db1a4e 100644 --- a/bolt/Passes/ShrinkWrapping.cpp +++ b/bolt/Passes/ShrinkWrapping.cpp @@ -101,7 +101,7 @@ void CalleeSavedAnalysis::analyzeSaves() { CalleeSaved.set(FIE->RegOrImm); SaveFIEByReg[FIE->RegOrImm] = &*FIE; SavingCost[FIE->RegOrImm] += InsnToBB[&Inst]->getKnownExecutionCount(); - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, getSaveTag(), FIE->RegOrImm); + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, getSaveTag(), FIE->RegOrImm); OffsetsByReg[FIE->RegOrImm] = FIE->StackOffset; DEBUG(dbgs() << "Logging new candidate for Callee-Saved Reg: " << FIE->RegOrImm << "\n"); @@ -152,7 +152,7 @@ void CalleeSavedAnalysis::analyzeRestores() { << "\n"); if (LoadFIEByReg[FIE->RegOrImm] == nullptr) LoadFIEByReg[FIE->RegOrImm] = &*FIE; - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, getRestoreTag(), + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, getRestoreTag(), FIE->RegOrImm); HasRestores.set(FIE->RegOrImm); } @@ -186,8 +186,8 @@ std::vector CalleeSavedAnalysis::getRestoresByReg(uint16_t Reg) { CalleeSavedAnalysis::~CalleeSavedAnalysis() { for (auto &BB : BF) { for (auto &Inst : BB) { - BC.MIA->removeAnnotation(Inst, getSaveTag()); - BC.MIA->removeAnnotation(Inst, getRestoreTag()); + BC.MIB->removeAnnotation(Inst, getSaveTag()); + BC.MIB->removeAnnotation(Inst, getRestoreTag()); } } } @@ -229,7 +229,7 @@ bool StackLayoutModifier::blacklistAllInConflictWith(int64_t Offset, void StackLayoutModifier::checkFramePointerInitialization(MCInst &Point) { auto &SPT = Info.getStackPointerTracking(); if (!BC.MII->get(Point.getOpcode()) - .hasDefOfPhysReg(Point, BC.MIA->getFramePointer(), *BC.MRI)) + .hasDefOfPhysReg(Point, BC.MIB->getFramePointer(), *BC.MRI)) return; int SPVal, FPVal; @@ -237,18 +237,18 @@ void StackLayoutModifier::checkFramePointerInitialization(MCInst &Point) { std::pair FP; if (FPVal != SPT.EMPTY && FPVal != SPT.SUPERPOSITION) - FP = std::make_pair(BC.MIA->getFramePointer(), FPVal); + FP = std::make_pair(BC.MIB->getFramePointer(), FPVal); else FP = std::make_pair(0, 0); std::pair SP; if (SPVal != SPT.EMPTY && SPVal != SPT.SUPERPOSITION) - SP = std::make_pair(BC.MIA->getStackPointer(), SPVal); + SP = std::make_pair(BC.MIB->getStackPointer(), SPVal); else SP = std::make_pair(0, 0); int64_t Output; - if (!BC.MIA->evaluateSimple(Point, Output, SP, FP)) + if (!BC.MIB->evaluateSimple(Point, Output, SP, FP)) return; // Not your regular frame pointer initialization... bail @@ -259,7 +259,7 @@ void StackLayoutModifier::checkFramePointerInitialization(MCInst &Point) { void StackLayoutModifier::checkStackPointerRestore(MCInst &Point) { auto &SPT = Info.getStackPointerTracking(); if (!BC.MII->get(Point.getOpcode()) - .hasDefOfPhysReg(Point, BC.MIA->getStackPointer(), *BC.MRI)) + .hasDefOfPhysReg(Point, BC.MIB->getStackPointer(), *BC.MRI)) return; // Check if the definition of SP comes from FP -- in this case, this // value may need to be updated depending on our stack layout changes @@ -270,7 +270,7 @@ void StackLayoutModifier::checkStackPointerRestore(MCInst &Point) { auto &Operand = Point.getOperand(I); if (!Operand.isReg()) continue; - if (Operand.getReg() == BC.MIA->getFramePointer()) { + if (Operand.getReg() == BC.MIB->getFramePointer()) { UsesFP = true; break; } @@ -284,18 +284,18 @@ void StackLayoutModifier::checkStackPointerRestore(MCInst &Point) { std::pair FP; if (FPVal != SPT.EMPTY && FPVal != SPT.SUPERPOSITION) - FP = std::make_pair(BC.MIA->getFramePointer(), FPVal); + FP = std::make_pair(BC.MIB->getFramePointer(), FPVal); else FP = std::make_pair(0, 0); std::pair SP; if (SPVal != SPT.EMPTY && SPVal != SPT.SUPERPOSITION) - SP = std::make_pair(BC.MIA->getStackPointer(), SPVal); + SP = std::make_pair(BC.MIB->getStackPointer(), SPVal); else SP = std::make_pair(0, 0); int64_t Output; - if (!BC.MIA->evaluateSimple(Point, Output, SP, FP)) + if (!BC.MIB->evaluateSimple(Point, Output, SP, FP)) return; // If the value is the same of FP, no need to adjust it @@ -310,7 +310,7 @@ void StackLayoutModifier::checkStackPointerRestore(MCInst &Point) { // We are restoring SP to an old value based on FP. Mark it as a stack // access to be fixed later. - BC.MIA->addAnnotation(BC.Ctx.get(), Point, getSlotTagName(), Output); + BC.MIB->addAnnotation(BC.Ctx.get(), Point, getSlotTagName(), Output); } void StackLayoutModifier::classifyStackAccesses() { @@ -353,7 +353,7 @@ void StackLayoutModifier::classifyStackAccesses() { // We are free to go. Add it as available stack slot which we know how // to move it. AvailableRegions[FIEX->StackOffset] = FIEX->Size; - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, getSlotTagName(), + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, getSlotTagName(), FIEX->StackOffset); RegionToRegMap[FIEX->StackOffset].insert(FIEX->RegOrImm); RegToRegionMap[FIEX->RegOrImm].insert(FIEX->StackOffset); @@ -370,8 +370,8 @@ void StackLayoutModifier::classifyCFIs() { auto recordAccess = [&](MCInst *Inst, int64_t Offset) { const uint16_t Reg = BC.MRI->getLLVMRegNum(CfaReg, /*isEH=*/false); - if (Reg == BC.MIA->getStackPointer() || Reg == BC.MIA->getFramePointer()) { - BC.MIA->addAnnotation(BC.Ctx.get(), *Inst, getSlotTagName(), Offset); + if (Reg == BC.MIB->getStackPointer() || Reg == BC.MIB->getFramePointer()) { + BC.MIB->addAnnotation(BC.Ctx.get(), *Inst, getSlotTagName(), Offset); DEBUG(dbgs() << "Recording CFI " << Offset << "\n"); } else { IsSimple = false; @@ -381,7 +381,7 @@ void StackLayoutModifier::classifyCFIs() { for (auto &BB : BF.layout()) { for (auto &Inst : *BB) { - if (!BC.MIA->isCFI(Inst)) + if (!BC.MIB->isCFI(Inst)) continue; auto *CFI = BF.getCFIFor(Inst); switch (CFI->getOperation()) { @@ -398,12 +398,12 @@ void StackLayoutModifier::classifyCFIs() { break; case MCCFIInstruction::OpOffset: recordAccess(&Inst, CFI->getOffset()); - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, getOffsetCFIRegTagName(), + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, getOffsetCFIRegTagName(), BC.MRI->getLLVMRegNum(CFI->getRegister(), /*isEH=*/false)); break; case MCCFIInstruction::OpSameValue: - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, getOffsetCFIRegTagName(), + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, getOffsetCFIRegTagName(), BC.MRI->getLLVMRegNum(CFI->getRegister(), /*isEH=*/false)); break; @@ -431,13 +431,13 @@ void StackLayoutModifier::classifyCFIs() { void StackLayoutModifier::scheduleChange( MCInst &Inst, StackLayoutModifier::WorklistItem Item) { - auto &WList = BC.MIA->getOrCreateAnnotationAs>( + auto &WList = BC.MIB->getOrCreateAnnotationAs>( BC.Ctx.get(), Inst, getTodoTagName()); WList.push_back(Item); } bool StackLayoutModifier::canCollapseRegion(MCInst *DeletedPush) { - if (!IsSimple || !BC.MIA->isPush(*DeletedPush)) + if (!IsSimple || !BC.MIB->isPush(*DeletedPush)) return false; auto FIE = FA.getFIEFor(*DeletedPush); @@ -482,10 +482,10 @@ bool StackLayoutModifier::collapseRegion(MCInst *Alloc, int64_t RegionAddr, for (auto &BB : BF) { for (auto &Inst : BB) { - if (!BC.MIA->hasAnnotation(Inst, getSlotTagName())) + if (!BC.MIB->hasAnnotation(Inst, getSlotTagName())) continue; auto Slot = - BC.MIA->getAnnotationAs( + BC.MIB->getAnnotationAs( Inst, getSlotTagName()); if (!AvailableRegions.count(Slot)) continue; @@ -493,7 +493,7 @@ bool StackLayoutModifier::collapseRegion(MCInst *Alloc, int64_t RegionAddr, if (!(*SAA.getStateBefore(Inst))[SAA.ExprToIdx[Alloc]]) continue; - if (BC.MIA->isCFI(Inst)) { + if (BC.MIB->isCFI(Inst)) { if (Slot > RegionAddr) continue; scheduleChange(Inst, WorklistItem(WorklistItem::AdjustCFI, RegionSz)); @@ -510,18 +510,18 @@ bool StackLayoutModifier::collapseRegion(MCInst *Alloc, int64_t RegionAddr, } if (Slot == RegionAddr) { - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, "AccessesDeletedPos", 0U); + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, "AccessesDeletedPos", 0U); continue; } - if (BC.MIA->isPush(Inst) || BC.MIA->isPop(Inst)) { + if (BC.MIB->isPush(Inst) || BC.MIB->isPop(Inst)) { continue; } - if (FIE->StackPtrReg == BC.MIA->getStackPointer() && Slot < RegionAddr) + if (FIE->StackPtrReg == BC.MIB->getStackPointer() && Slot < RegionAddr) continue; - if (FIE->StackPtrReg == BC.MIA->getFramePointer() && Slot > RegionAddr) + if (FIE->StackPtrReg == BC.MIB->getFramePointer() && Slot > RegionAddr) continue; scheduleChange( @@ -536,9 +536,9 @@ bool StackLayoutModifier::collapseRegion(MCInst *Alloc, int64_t RegionAddr, void StackLayoutModifier::setOffsetForCollapsedAccesses(int64_t NewOffset) { for (auto &BB : BF) { for (auto &Inst : BB) { - if (!BC.MIA->hasAnnotation(Inst, "AccessesDeletedPos")) + if (!BC.MIB->hasAnnotation(Inst, "AccessesDeletedPos")) continue; - BC.MIA->removeAnnotation(Inst, "AccessesDeletedPos"); + BC.MIB->removeAnnotation(Inst, "AccessesDeletedPos"); scheduleChange( Inst, WorklistItem(WorklistItem::AdjustLoadStoreOffset, NewOffset)); } @@ -583,10 +583,10 @@ bool StackLayoutModifier::insertRegion(ProgramPoint P, int64_t RegionSz) { for (auto &BB : BF) { for (auto &Inst : BB) { - if (!BC.MIA->hasAnnotation(Inst, getSlotTagName())) + if (!BC.MIB->hasAnnotation(Inst, getSlotTagName())) continue; auto Slot = - BC.MIA->getAnnotationAs( + BC.MIB->getAnnotationAs( Inst, getSlotTagName()); if (!AvailableRegions.count(Slot)) continue; @@ -594,7 +594,7 @@ bool StackLayoutModifier::insertRegion(ProgramPoint P, int64_t RegionSz) { if (!(DA.doesADominateB(P, Inst))) continue; - if (BC.MIA->isCFI(Inst)) { + if (BC.MIB->isCFI(Inst)) { if (Slot >= RegionAddr) continue; scheduleChange(Inst, WorklistItem(WorklistItem::AdjustCFI, -RegionSz)); @@ -609,11 +609,11 @@ bool StackLayoutModifier::insertRegion(ProgramPoint P, int64_t RegionSz) { continue; } - if (FIE->StackPtrReg == BC.MIA->getStackPointer() && Slot < RegionAddr) + if (FIE->StackPtrReg == BC.MIB->getStackPointer() && Slot < RegionAddr) continue; - if (FIE->StackPtrReg == BC.MIA->getFramePointer() && Slot >= RegionAddr) + if (FIE->StackPtrReg == BC.MIB->getFramePointer() && Slot >= RegionAddr) continue; - if (BC.MIA->isPush(Inst) || BC.MIA->isPop(Inst)) + if (BC.MIB->isPush(Inst) || BC.MIB->isPop(Inst)) continue; scheduleChange( Inst, WorklistItem(WorklistItem::AdjustLoadStoreOffset, -RegionSz)); @@ -629,13 +629,13 @@ void StackLayoutModifier::performChanges() { for (auto &BB : BF) { for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) { auto &Inst = *I; - if (BC.MIA->hasAnnotation(Inst, "AccessesDeletedPos")) { - assert(BC.MIA->isPop(Inst) || BC.MIA->isPush(Inst)); - BC.MIA->removeAnnotation(Inst, "AccessesDeletedPos"); + if (BC.MIB->hasAnnotation(Inst, "AccessesDeletedPos")) { + assert(BC.MIB->isPop(Inst) || BC.MIB->isPush(Inst)); + BC.MIB->removeAnnotation(Inst, "AccessesDeletedPos"); } - if (!BC.MIA->hasAnnotation(Inst, getTodoTagName())) + if (!BC.MIB->hasAnnotation(Inst, getTodoTagName())) continue; - auto &WList = BC.MIA->getAnnotationAs>( + auto &WList = BC.MIB->getAnnotationAs>( Inst, getTodoTagName()); int64_t Adjustment = 0; WorklistItem::ActionType AdjustmentType = WorklistItem::None; @@ -653,7 +653,7 @@ void StackLayoutModifier::performChanges() { if (!Adjustment) continue; if (AdjustmentType != WorklistItem::AdjustLoadStoreOffset) { - assert(BC.MIA->isCFI(Inst)); + assert(BC.MIB->isCFI(Inst)); uint32_t CFINum = Inst.getOperand(0).getImm(); if (ModifiedCFIIndices.count(CFINum)) continue; @@ -675,23 +675,23 @@ void StackLayoutModifier::performChanges() { bool IsStoreFromReg{false}; uint8_t Size{0}; bool Success{false}; - Success = BC.MIA->isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, + Success = BC.MIB->isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm, StackPtrReg, StackOffset, Size, IsSimple, IsIndexed); if (!Success) { // SP update based on FP value - Success = BC.MIA->addToImm(Inst, Adjustment, &*BC.Ctx); + Success = BC.MIB->addToImm(Inst, Adjustment, &*BC.Ctx); assert(Success); continue; } assert(Success && IsSimple && !IsIndexed && (!IsStore || IsStoreFromReg)); - if (StackPtrReg != BC.MIA->getFramePointer()) + if (StackPtrReg != BC.MIB->getFramePointer()) Adjustment = -Adjustment; if (IsLoad) - Success = BC.MIA->createRestoreFromStack( + Success = BC.MIB->createRestoreFromStack( Inst, StackPtrReg, StackOffset + Adjustment, Reg, Size); else if (IsStore) - Success = BC.MIA->createSaveToStack( + Success = BC.MIB->createSaveToStack( Inst, StackPtrReg, StackOffset + Adjustment, Reg, Size); DEBUG({ dbgs() << "Adjusted instruction: "; @@ -720,13 +720,13 @@ void ShrinkWrapping::classifyCSRUses() { BitVector(DA.NumInstrs, false)); const BitVector &FPAliases = - BC.MIA->getAliases(BC.MIA->getFramePointer()); + BC.MIB->getAliases(BC.MIB->getFramePointer()); for (auto &BB : BF) { for (auto &Inst : BB) { - if (BC.MIA->isCFI(Inst)) + if (BC.MIB->isCFI(Inst)) continue; auto BV = BitVector(BC.MRI->getNumRegs(), false); - BC.MIA->getTouchedRegs(Inst, BV); + BC.MIB->getTouchedRegs(Inst, BV); BV &= CSA.CalleeSaved; for (int I = BV.find_first(); I != -1; I = BV.find_next(I)) { if (I == 0) @@ -734,7 +734,7 @@ void ShrinkWrapping::classifyCSRUses() { if (CSA.getSavedReg(Inst) != I && CSA.getRestoredReg(Inst) != I) UsesByReg[I].set(DA.ExprToIdx[&Inst]); } - if (!SPT.HasFramePointer || !BC.MIA->isCall(Inst)) + if (!SPT.HasFramePointer || !BC.MIB->isCall(Inst)) continue; BV = CSA.CalleeSaved; BV &= FPAliases; @@ -746,7 +746,7 @@ void ShrinkWrapping::classifyCSRUses() { } void ShrinkWrapping::pruneUnwantedCSRs() { - BitVector ParamRegs = BC.MIA->getRegsUsedAsParams(); + BitVector ParamRegs = BC.MIB->getRegsUsedAsParams(); for (unsigned I = 0, E = BC.MRI->getNumRegs(); I != E; ++I) { if (!CSA.CalleeSaved[I]) continue; @@ -936,7 +936,7 @@ void ShrinkWrapping::splitFrontierCritEdges( // and not BBs). if (NewBB->empty()) { MCInst NewInst; - BC.MIA->createNoop(NewInst); + BC.MIB->createNoop(NewInst); NewBB->addInstruction(std::move(NewInst)); scheduleChange(&*NewBB->begin(), WorklistItem(WorklistItem::Erase, 0)); } @@ -963,7 +963,7 @@ ShrinkWrapping::doRestorePlacement(MCInst *BestPosSave, unsigned CSR, Frontier = DA.getDominanceFrontierFor(*BestPosSave); for (auto &PP : Frontier) { bool HasCritEdges{false}; - if (PP.isInst() && BC.MIA->isTerminator(*PP.getInst()) && + if (PP.isInst() && BC.MIB->isTerminator(*PP.getInst()) && doesInstUsesCSR(*PP.getInst(), CSR)) { CannotPlace = true; } @@ -973,7 +973,7 @@ ShrinkWrapping::doRestorePlacement(MCInst *BestPosSave, unsigned CSR, auto &Dests = CritEdgesTo.back(); // Check for invoke instructions at the dominance frontier, which indicates // the landing pad is not dominated. - if (PP.isInst() && BC.MIA->isInvoke(*PP.getInst())) { + if (PP.isInst() && BC.MIB->isInvoke(*PP.getInst())) { DEBUG(dbgs() << "Bailing on restore placement to avoid LP splitting\n"); Frontier.clear(); return Frontier; @@ -1100,7 +1100,7 @@ void ShrinkWrapping::scheduleOldSaveRestoresRemoval(unsigned CSR, std::vector CFIs; for (auto I = BB->rbegin(), E = BB->rend(); I != E; ++I) { auto &Inst = *I; - if (BC.MIA->isCFI(Inst)) { + if (BC.MIB->isCFI(Inst)) { // Delete all offset CFIs related to this CSR if (SLM.getOffsetCFIReg(Inst) == CSR) { HasDeletedOffsetCFIs[CSR] = true; @@ -1154,11 +1154,11 @@ void ShrinkWrapping::scheduleOldSaveRestoresRemoval(unsigned CSR, } bool ShrinkWrapping::doesInstUsesCSR(const MCInst &Inst, uint16_t CSR) { - if (BC.MIA->isCFI(Inst) || CSA.getSavedReg(Inst) == CSR || + if (BC.MIB->isCFI(Inst) || CSA.getSavedReg(Inst) == CSR || CSA.getRestoredReg(Inst) == CSR) return false; BitVector BV = BitVector(BC.MRI->getNumRegs(), false); - BC.MIA->getTouchedRegs(Inst, BV); + BC.MIB->getTouchedRegs(Inst, BV); return BV[CSR]; } @@ -1274,11 +1274,11 @@ void ShrinkWrapping::moveSaveRestores() { } for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) { auto &Inst = *I; - auto TodoList = BC.MIA->tryGetAnnotationAs>( + auto TodoList = BC.MIB->tryGetAnnotationAs>( Inst, getAnnotationName()); if (!TodoList) continue; - bool isCFI = BC.MIA->isCFI(Inst); + bool isCFI = BC.MIB->isCFI(Inst); for (auto &Item : *TodoList) { if (Item.Action == WorklistItem::InsertPushOrPop) Item.Action = WorklistItem::InsertLoadOrStore; @@ -1388,13 +1388,13 @@ protected: std::pair &Res) { for (const auto &Item : TodoItems) { if (Item.Action == ShrinkWrapping::WorklistItem::Erase && - BC.MIA->isPush(Point)) { - Res.first += BC.MIA->getPushSize(Point); + BC.MIB->isPush(Point)) { + Res.first += BC.MIB->getPushSize(Point); continue; } if (Item.Action == ShrinkWrapping::WorklistItem::Erase && - BC.MIA->isPop(Point)) { - Res.first -= BC.MIA->getPopSize(Point); + BC.MIB->isPop(Point)) { + Res.first -= BC.MIB->getPopSize(Point); continue; } if (Item.Action == ShrinkWrapping::WorklistItem::InsertPushOrPop && @@ -1419,7 +1419,7 @@ protected: Res.first == StackPointerTracking::EMPTY) return Res; auto TodoItems = - BC.MIA->tryGetAnnotationAs>( + BC.MIB->tryGetAnnotationAs>( Point, ShrinkWrapping::getAnnotationName()); if (TodoItems) compNextAux(Point, *TodoItems, Res); @@ -1469,7 +1469,7 @@ void ShrinkWrapping::insertUpdatedCFI(unsigned CSR, int SPValPush, bool IsSimple{false}; bool IsStoreFromReg{false}; uint8_t Size{0}; - if (!BC.MIA->isStackAccess(*InstIter, IsLoad, IsStore, IsStoreFromReg, + if (!BC.MIB->isStackAccess(*InstIter, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm, StackPtrReg, StackOffset, Size, IsSimple, IsIndexed)) continue; @@ -1535,11 +1535,11 @@ void ShrinkWrapping::insertUpdatedCFI(unsigned CSR, int SPValPush, void ShrinkWrapping::rebuildCFIForSP() { for (auto &BB : BF) { for (auto &Inst : BB) { - if (!BC.MIA->isCFI(Inst)) + if (!BC.MIB->isCFI(Inst)) continue; auto *CFI = BF.getCFIFor(Inst); if (CFI->getOperation() == MCCFIInstruction::OpDefCfaOffset) - BC.MIA->addAnnotation(BC.Ctx.get(), Inst, "DeleteMe", 0U); + BC.MIB->addAnnotation(BC.Ctx.get(), Inst, "DeleteMe", 0U); } } @@ -1580,7 +1580,7 @@ void ShrinkWrapping::rebuildCFIForSP() { for (auto &BB : BF) for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) - if (BC.MIA->hasAnnotation(*I, "DeleteMe")) + if (BC.MIB->hasAnnotation(*I, "DeleteMe")) BB.eraseInstruction(&*I); } @@ -1591,14 +1591,14 @@ MCInst ShrinkWrapping::createStackAccess(int SPVal, int FPVal, if (SPVal != StackPointerTracking::SUPERPOSITION && SPVal != StackPointerTracking::EMPTY) { if (FIE.IsLoad) { - if (!BC.MIA->createRestoreFromStack(NewInst, BC.MIA->getStackPointer(), + if (!BC.MIB->createRestoreFromStack(NewInst, BC.MIB->getStackPointer(), FIE.StackOffset - SPVal, FIE.RegOrImm, FIE.Size)) { errs() << "createRestoreFromStack: not supported on this platform\n"; abort(); } } else { - if (!BC.MIA->createSaveToStack(NewInst, BC.MIA->getStackPointer(), + if (!BC.MIB->createSaveToStack(NewInst, BC.MIB->getStackPointer(), FIE.StackOffset - SPVal, FIE.RegOrImm, FIE.Size)) { errs() << "createSaveToStack: not supported on this platform\n"; @@ -1606,21 +1606,21 @@ MCInst ShrinkWrapping::createStackAccess(int SPVal, int FPVal, } } if (CreatePushOrPop) - BC.MIA->changeToPushOrPop(NewInst); + BC.MIB->changeToPushOrPop(NewInst); return NewInst; } assert(FPVal != StackPointerTracking::SUPERPOSITION && FPVal != StackPointerTracking::EMPTY); if (FIE.IsLoad) { - if (!BC.MIA->createRestoreFromStack(NewInst, BC.MIA->getFramePointer(), + if (!BC.MIB->createRestoreFromStack(NewInst, BC.MIB->getFramePointer(), FIE.StackOffset - FPVal, FIE.RegOrImm, FIE.Size)) { errs() << "createRestoreFromStack: not supported on this platform\n"; abort(); } } else { - if (!BC.MIA->createSaveToStack(NewInst, BC.MIA->getFramePointer(), + if (!BC.MIB->createSaveToStack(NewInst, BC.MIB->getFramePointer(), FIE.StackOffset - FPVal, FIE.RegOrImm, FIE.Size)) { errs() << "createSaveToStack: not supported on this platform\n"; @@ -1802,7 +1802,7 @@ bool ShrinkWrapping::processInsertions() { // Process insertions before some inst. for (auto I = BB.begin(); I != BB.end(); ++I) { auto &Inst = *I; - auto TodoList = BC.MIA->tryGetAnnotationAs>( + auto TodoList = BC.MIB->tryGetAnnotationAs>( Inst, getAnnotationName()); if (!TodoList) continue; @@ -1835,7 +1835,7 @@ void ShrinkWrapping::processDeletions() { for (auto &BB : BF) { for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) { auto &Inst = *I; - auto TodoList = BC.MIA->tryGetAnnotationAs>( + auto TodoList = BC.MIB->tryGetAnnotationAs>( Inst, getAnnotationName()); if (!TodoList) continue; @@ -1847,13 +1847,13 @@ void ShrinkWrapping::processDeletions() { if (Item.Action == WorklistItem::ChangeToAdjustment) { // Is flag reg alive across this func? - bool DontClobberFlags = LA.isAlive(&Inst, BC.MIA->getFlagsReg()); - if (auto Sz = BC.MIA->getPushSize(Inst)) { - BC.MIA->createStackPointerIncrement(Inst, Sz, DontClobberFlags); + bool DontClobberFlags = LA.isAlive(&Inst, BC.MIB->getFlagsReg()); + if (auto Sz = BC.MIB->getPushSize(Inst)) { + BC.MIB->createStackPointerIncrement(Inst, Sz, DontClobberFlags); continue; } - if (auto Sz = BC.MIA->getPopSize(Inst)) { - BC.MIA->createStackPointerDecrement(Inst, Sz, DontClobberFlags); + if (auto Sz = BC.MIB->getPopSize(Inst)) { + BC.MIB->createStackPointerDecrement(Inst, Sz, DontClobberFlags); continue; } } diff --git a/bolt/Passes/ShrinkWrapping.h b/bolt/Passes/ShrinkWrapping.h index f22b188c5d97..fb6fe1dc75d5 100644 --- a/bolt/Passes/ShrinkWrapping.h +++ b/bolt/Passes/ShrinkWrapping.h @@ -74,7 +74,7 @@ public: /// Retrieves the value of the callee-saved register that is saved by this /// instruction or 0 if this is not a CSR save instruction. uint16_t getSavedReg(const MCInst &Inst) { - auto Val = BC.MIA->tryGetAnnotationAs( + auto Val = BC.MIB->tryGetAnnotationAs( Inst, getSaveTag()); if (Val) return *Val; @@ -84,7 +84,7 @@ public: /// Retrieves the value of the callee-saved register that is restored by this /// instruction or 0 if this is not a CSR restore instruction. uint16_t getRestoredReg(const MCInst &Inst) { - auto Val = BC.MIA->tryGetAnnotationAs( + auto Val = BC.MIB->tryGetAnnotationAs( Inst, getRestoreTag()); if (Val) return *Val; @@ -191,9 +191,9 @@ public: ~StackLayoutModifier() { for (auto &BB : BF) { for (auto &Inst : BB) { - BC.MIA->removeAnnotation(Inst, getTodoTagName()); - BC.MIA->removeAnnotation(Inst, getSlotTagName()); - BC.MIA->removeAnnotation(Inst, getOffsetCFIRegTagName()); + BC.MIB->removeAnnotation(Inst, getTodoTagName()); + BC.MIB->removeAnnotation(Inst, getSlotTagName()); + BC.MIB->removeAnnotation(Inst, getOffsetCFIRegTagName()); } } } @@ -202,7 +202,7 @@ public: /// instruction or 0 if this is not a CSR restore instruction. uint16_t getOffsetCFIReg(const MCInst &Inst) { auto Val = - BC.MIA->tryGetAnnotationAs(Inst, getOffsetCFIRegTagName()); + BC.MIB->tryGetAnnotationAs(Inst, getOffsetCFIRegTagName()); if (Val) return *Val; return 0; @@ -326,7 +326,7 @@ private: template void scheduleChange(ProgramPoint PP, T&& ...Item) { if (PP.isInst()) { - auto &WList = BC.MIA->getOrCreateAnnotationAs>( + auto &WList = BC.MIB->getOrCreateAnnotationAs>( BC.Ctx.get(), *PP.getInst(), getAnnotationName()); WList.emplace_back(std::forward(Item)...); return; @@ -343,7 +343,7 @@ private: assert (BB->succ_size() == 1); BB = *BB->succ_begin(); } - auto &WList = BC.MIA->getOrCreateAnnotationAs>( + auto &WList = BC.MIB->getOrCreateAnnotationAs>( BC.Ctx.get(), *BB->begin(), getAnnotationName()); WList.emplace_back(std::forward(Item)...); } @@ -470,7 +470,7 @@ public: ~ShrinkWrapping() { for (auto &BB : BF) { for (auto &Inst : BB) { - BC.MIA->removeAnnotation(Inst, getAnnotationName()); + BC.MIB->removeAnnotation(Inst, getAnnotationName()); } } } diff --git a/bolt/Passes/StackAllocationAnalysis.cpp b/bolt/Passes/StackAllocationAnalysis.cpp index 89f2d2a1c254..e39fcaeb3b37 100644 --- a/bolt/Passes/StackAllocationAnalysis.cpp +++ b/bolt/Passes/StackAllocationAnalysis.cpp @@ -24,11 +24,11 @@ void StackAllocationAnalysis::preflight() { for (auto &BB : this->Func) { for (auto &Inst : BB) { MCPhysReg From, To; - if (!BC.MIA->isPush(Inst) && (!BC.MIA->isRegToRegMove(Inst, From, To) || - To != BC.MIA->getStackPointer() || - From != BC.MIA->getFramePointer()) && + if (!BC.MIB->isPush(Inst) && (!BC.MIB->isRegToRegMove(Inst, From, To) || + To != BC.MIB->getStackPointer() || + From != BC.MIB->getFramePointer()) && !BC.MII->get(Inst.getOpcode()) - .hasDefOfPhysReg(Inst, BC.MIA->getStackPointer(), *BC.MRI)) + .hasDefOfPhysReg(Inst, BC.MIB->getStackPointer(), *BC.MRI)) continue; this->Expressions.push_back(&Inst); this->ExprToIdx[&Inst] = this->NumInstrs++; @@ -94,13 +94,13 @@ void StackAllocationAnalysis::doConfluenceWithLP(BitVector &StateOut, BitVector StackAllocationAnalysis::computeNext(const MCInst &Point, const BitVector &Cur) { - const auto &MIA = BC.MIA; + const auto &MIB = BC.MIB; BitVector Next = Cur; - if (int Sz = MIA->getPopSize(Point)) { + if (int Sz = MIB->getPopSize(Point)) { Next = doKill(Point, Next, Sz); return Next; } - if (MIA->isPush(Point)) { + if (MIB->isPush(Point)) { Next.set(this->ExprToIdx[&Point]); return Next; } @@ -108,9 +108,9 @@ BitVector StackAllocationAnalysis::computeNext(const MCInst &Point, MCPhysReg From, To; int64_t SPOffset, FPOffset; std::tie(SPOffset, FPOffset) = *SPT.getStateBefore(Point); - if (MIA->isRegToRegMove(Point, From, To) && To == MIA->getStackPointer() && - From == MIA->getFramePointer()) { - if (MIA->isLeave(Point)) + if (MIB->isRegToRegMove(Point, From, To) && To == MIB->getStackPointer() && + From == MIB->getFramePointer()) { + if (MIB->isLeave(Point)) FPOffset += 8; if (SPOffset < FPOffset) { Next = doKill(Point, Next, FPOffset - SPOffset); @@ -122,19 +122,19 @@ BitVector StackAllocationAnalysis::computeNext(const MCInst &Point, } } if (BC.MII->get(Point.getOpcode()) - .hasDefOfPhysReg(Point, MIA->getStackPointer(), *BC.MRI)) { + .hasDefOfPhysReg(Point, MIB->getStackPointer(), *BC.MRI)) { std::pair SP; if (SPOffset != SPT.EMPTY && SPOffset != SPT.SUPERPOSITION) - SP = std::make_pair(MIA->getStackPointer(), SPOffset); + SP = std::make_pair(MIB->getStackPointer(), SPOffset); else SP = std::make_pair(0, 0); std::pair FP; if (FPOffset != SPT.EMPTY && FPOffset != SPT.SUPERPOSITION) - FP = std::make_pair(MIA->getFramePointer(), FPOffset); + FP = std::make_pair(MIB->getFramePointer(), FPOffset); else FP = std::make_pair(0, 0); int64_t Output; - if (!MIA->evaluateSimple(Point, Output, SP, FP)) + if (!MIB->evaluateSimple(Point, Output, SP, FP)) return Next; if (SPOffset < Output) { diff --git a/bolt/Passes/StackPointerTracking.h b/bolt/Passes/StackPointerTracking.h index 3438a07a3225..712f279a2854 100644 --- a/bolt/Passes/StackPointerTracking.h +++ b/bolt/Passes/StackPointerTracking.h @@ -82,16 +82,16 @@ protected: } int computeNextSP(const MCInst &Point, int SPVal, int FPVal) { - const auto &MIA = this->BC.MIA; + const auto &MIB = this->BC.MIB; - if (int Sz = MIA->getPushSize(Point)) { + if (int Sz = MIB->getPushSize(Point)) { if (SPVal == EMPTY || SPVal == SUPERPOSITION) return SPVal; return SPVal - Sz; } - if (int Sz = MIA->getPopSize(Point)) { + if (int Sz = MIB->getPopSize(Point)) { if (SPVal == EMPTY || SPVal == SUPERPOSITION) return SPVal; @@ -99,31 +99,31 @@ protected: } MCPhysReg From, To; - if (MIA->isRegToRegMove(Point, From, To) && To == MIA->getStackPointer() && - From == MIA->getFramePointer()) { + if (MIB->isRegToRegMove(Point, From, To) && To == MIB->getStackPointer() && + From == MIB->getFramePointer()) { if (FPVal == EMPTY || FPVal == SUPERPOSITION) return FPVal; - if (MIA->isLeave(Point)) + if (MIB->isLeave(Point)) return FPVal + 8; else return FPVal; } if (this->BC.MII->get(Point.getOpcode()) - .hasDefOfPhysReg(Point, MIA->getStackPointer(), *this->BC.MRI)) { + .hasDefOfPhysReg(Point, MIB->getStackPointer(), *this->BC.MRI)) { std::pair SP; if (SPVal != EMPTY && SPVal != SUPERPOSITION) - SP = std::make_pair(MIA->getStackPointer(), SPVal); + SP = std::make_pair(MIB->getStackPointer(), SPVal); else SP = std::make_pair(0, 0); std::pair FP; if (FPVal != EMPTY && FPVal != SUPERPOSITION) - FP = std::make_pair(MIA->getFramePointer(), FPVal); + FP = std::make_pair(MIB->getFramePointer(), FPVal); else FP = std::make_pair(0, 0); int64_t Output; - if (!MIA->evaluateSimple(Point, Output, SP, FP)) { + if (!MIB->evaluateSimple(Point, Output, SP, FP)) { if (SPVal == EMPTY && FPVal == EMPTY) return SPVal; return SUPERPOSITION; @@ -136,36 +136,36 @@ protected: } int computeNextFP(const MCInst &Point, int SPVal, int FPVal) { - const auto &MIA = this->BC.MIA; + const auto &MIB = this->BC.MIB; MCPhysReg From, To; - if (MIA->isRegToRegMove(Point, From, To) && To == MIA->getFramePointer() && - From == MIA->getStackPointer()) { + if (MIB->isRegToRegMove(Point, From, To) && To == MIB->getFramePointer() && + From == MIB->getStackPointer()) { HasFramePointer = true; return SPVal; } if (this->BC.MII->get(Point.getOpcode()) - .hasDefOfPhysReg(Point, MIA->getFramePointer(), *this->BC.MRI)) { + .hasDefOfPhysReg(Point, MIB->getFramePointer(), *this->BC.MRI)) { std::pair FP; if (FPVal != EMPTY && FPVal != SUPERPOSITION) - FP = std::make_pair(MIA->getFramePointer(), FPVal); + FP = std::make_pair(MIB->getFramePointer(), FPVal); else FP = std::make_pair(0, 0); std::pair SP; if (SPVal != EMPTY && SPVal != SUPERPOSITION) - SP = std::make_pair(MIA->getStackPointer(), SPVal); + SP = std::make_pair(MIB->getStackPointer(), SPVal); else SP = std::make_pair(0, 0); int64_t Output; - if (!MIA->evaluateSimple(Point, Output, SP, FP)) { + if (!MIB->evaluateSimple(Point, Output, SP, FP)) { if (SPVal == EMPTY && FPVal == EMPTY) return FPVal; return SUPERPOSITION; } if (!HasFramePointer) { - if (MIA->escapesVariable(Point, false)) { + if (MIB->escapesVariable(Point, false)) { HasFramePointer = true; } } diff --git a/bolt/Passes/StokeInfo.cpp b/bolt/Passes/StokeInfo.cpp index 4890bdfbeaf2..edfd65f895a5 100644 --- a/bolt/Passes/StokeInfo.cpp +++ b/bolt/Passes/StokeInfo.cpp @@ -48,14 +48,14 @@ void StokeInfo::checkInstr(const BinaryContext &BC, const BinaryFunction &BF, continue; } // skip function with exception handling yet - if (BC.MIA->isEHLabel(It) || BC.MIA->isInvoke(It) || BC.MIA->hasEHInfo(It)) { + if (BC.MIB->isEHLabel(It) || BC.MIB->isInvoke(It) || BC.MIB->hasEHInfo(It)) { FuncInfo.Omitted = true; return; } // check if this function contains call instruction - if (BC.MIA->isCall(It)) { + if (BC.MIB->isCall(It)) { FuncInfo.HasCall = true; - const auto *TargetSymbol = BC.MIA->getTargetSymbol(It); + const auto *TargetSymbol = BC.MIB->getTargetSymbol(It); // if it is an indirect call, skip if (TargetSymbol == nullptr) { FuncInfo.Omitted = true; @@ -64,12 +64,12 @@ void StokeInfo::checkInstr(const BinaryContext &BC, const BinaryFunction &BF, } // check if this function modify stack or heap // TODO: more accurate analysis - auto IsPush = BC.MIA->isPush(It); - auto IsRipAddr = BC.MIA->hasPCRelOperand(It); + auto IsPush = BC.MIB->isPush(It); + auto IsRipAddr = BC.MIB->hasPCRelOperand(It); if (IsPush) { FuncInfo.StackOut = true; } - if (BC.MIA->isStore(It) && !IsPush && !IsRipAddr) { + if (BC.MIB->isStore(It) && !IsPush && !IsRipAddr) { FuncInfo.HeapOut = true; } if (IsRipAddr) { @@ -165,8 +165,8 @@ void StokeInfo::runOnFunctions( DefaultDefInMask.resize(NumRegs, false); DefaultLiveOutMask.resize(NumRegs, false); - BC.MIA->getDefaultDefIn(DefaultDefInMask); - BC.MIA->getDefaultLiveOut(DefaultLiveOutMask); + BC.MIB->getDefaultDefIn(DefaultDefInMask); + BC.MIB->getDefaultLiveOut(DefaultLiveOutMask); getRegNameFromBitVec(BC, DefaultDefInMask); getRegNameFromBitVec(BC, DefaultLiveOutMask); diff --git a/bolt/ProfileReader.cpp b/bolt/ProfileReader.cpp index ffac6b614bf3..9daa5fc72a6c 100644 --- a/bolt/ProfileReader.cpp +++ b/bolt/ProfileReader.cpp @@ -111,7 +111,7 @@ ProfileReader::parseFunctionProfile(BinaryFunction &BF, ++MismatchedCalls; continue; } - if (!BC.MIA->isCall(*Instr) && !BC.MIA->isIndirectBranch(*Instr)) { + if (!BC.MIB->isCall(*Instr) && !BC.MIB->isIndirectBranch(*Instr)) { if (opts::Verbosity >= 2) errs() << "BOLT-WARNING: expected call at offset " << YamlCSI.Offset << " in block " << BB.getName() << '\n'; @@ -120,22 +120,22 @@ ProfileReader::parseFunctionProfile(BinaryFunction &BF, } auto setAnnotation = [&](StringRef Name, uint64_t Count) { - if (BC.MIA->hasAnnotation(*Instr, Name)) { + if (BC.MIB->hasAnnotation(*Instr, Name)) { if (opts::Verbosity >= 1) errs() << "BOLT-WARNING: ignoring duplicate " << Name << " info for offset 0x" << Twine::utohexstr(YamlCSI.Offset) << " in function " << BF << '\n'; return; } - BC.MIA->addAnnotation(BC.Ctx.get(), *Instr, Name, Count); + BC.MIB->addAnnotation(BC.Ctx.get(), *Instr, Name, Count); }; - if (BC.MIA->isIndirectCall(*Instr) || BC.MIA->isIndirectBranch(*Instr)) { + if (BC.MIB->isIndirectCall(*Instr) || BC.MIB->isIndirectBranch(*Instr)) { IndirectCallSiteProfile &CSP = - BC.MIA->getOrCreateAnnotationAs(BC.Ctx.get(), + BC.MIB->getOrCreateAnnotationAs(BC.Ctx.get(), *Instr, "CallProfile"); CSP.emplace_back(IsFunction, Name, YamlCSI.Count, YamlCSI.Mispreds); - } else if (BC.MIA->getConditionalTailCall(*Instr)) { + } else if (BC.MIB->getConditionalTailCall(*Instr)) { setAnnotation("CTCTakenCount", YamlCSI.Count); setAnnotation("CTCMispredCount", YamlCSI.Mispreds); } else { diff --git a/bolt/ProfileWriter.cpp b/bolt/ProfileWriter.cpp index d26269824640..4a83ad196baa 100644 --- a/bolt/ProfileWriter.cpp +++ b/bolt/ProfileWriter.cpp @@ -59,18 +59,18 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) { YamlBB.ExecCount = BB->getKnownExecutionCount(); for (const auto &Instr : *BB) { - if (!BC.MIA->isCall(Instr) && !BC.MIA->isIndirectBranch(Instr)) + if (!BC.MIB->isCall(Instr) && !BC.MIB->isIndirectBranch(Instr)) continue; yaml::bolt::CallSiteInfo CSI; - auto Offset = BC.MIA->tryGetAnnotationAs(Instr, "Offset"); + auto Offset = BC.MIB->tryGetAnnotationAs(Instr, "Offset"); if (!Offset || Offset.get() < BB->getInputOffset()) continue; CSI.Offset = Offset.get() - BB->getInputOffset(); - if (BC.MIA->isIndirectCall(Instr) || BC.MIA->isIndirectBranch(Instr)) { + if (BC.MIB->isIndirectCall(Instr) || BC.MIB->isIndirectBranch(Instr)) { auto ICSP = - BC.MIA->tryGetAnnotationAs(Instr, + BC.MIB->tryGetAnnotationAs(Instr, "CallProfile"); if (!ICSP) continue; @@ -92,25 +92,25 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) { YamlBB.CallSites.push_back(CSI); } } else { // direct call or a tail call - const auto *CalleeSymbol = BC.MIA->getTargetSymbol(Instr); + const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Instr); const auto Callee = BC.getFunctionForSymbol(CalleeSymbol); if (Callee) { CSI.DestId = Callee->getFunctionNumber();; CSI.EntryDiscriminator = Callee->getEntryForSymbol(CalleeSymbol); } - if (BC.MIA->getConditionalTailCall(Instr)) { + if (BC.MIB->getConditionalTailCall(Instr)) { auto CTCCount = - BC.MIA->tryGetAnnotationAs(Instr, "CTCTakenCount"); + BC.MIB->tryGetAnnotationAs(Instr, "CTCTakenCount"); if (CTCCount) { CSI.Count = *CTCCount; auto CTCMispreds = - BC.MIA->tryGetAnnotationAs(Instr, "CTCMispredCount"); + BC.MIB->tryGetAnnotationAs(Instr, "CTCMispredCount"); if (CTCMispreds) CSI.Mispreds = *CTCMispreds; } } else { - auto Count = BC.MIA->tryGetAnnotationAs(Instr, "Count"); + auto Count = BC.MIB->tryGetAnnotationAs(Instr, "Count"); if (Count) CSI.Count = *Count; } diff --git a/bolt/RewriteInstance.cpp b/bolt/RewriteInstance.cpp index 58a4ab104d9a..8f4ed4c62fbb 100644 --- a/bolt/RewriteInstance.cpp +++ b/bolt/RewriteInstance.cpp @@ -18,6 +18,7 @@ #include "DataAggregator.h" #include "DataReader.h" #include "Exceptions.h" +#include "MCPlusBuilder.h" #include "ProfileReader.h" #include "ProfileWriter.h" #include "RewriteInstance.h" @@ -442,6 +443,28 @@ size_t padFunction(const BinaryFunction &Function) { } // namespace opts +extern MCPlusBuilder * createX86MCPlusBuilder(const MCInstrAnalysis *, + const MCInstrInfo *, + const MCRegisterInfo *); +extern MCPlusBuilder * createAArch64MCPlusBuilder(const MCInstrAnalysis *, + const MCInstrInfo *, + const MCRegisterInfo *); +namespace { + +MCPlusBuilder *createMCPlusBuilder(const Triple::ArchType Arch, + const MCInstrAnalysis *Analysis, const MCInstrInfo *Info, + const MCRegisterInfo *RegInfo) { + if (Arch == Triple::x86_64) { + return createX86MCPlusBuilder(Analysis, Info, RegInfo); + } else if (Arch == Triple::aarch64) { + return createAArch64MCPlusBuilder(Analysis, Info, RegInfo); + } else { + llvm_unreachable("architecture unsupport by MCPlusBuilder"); + } +} + +} + constexpr const char *RewriteInstance::SectionsToOverwrite[]; const std::string RewriteInstance::OrgSecPrefix = ".bolt.org"; @@ -645,13 +668,22 @@ createBinaryContext(ELFObjectFileBase *File, DataReader &DR, } std::unique_ptr MIA( - TheTarget->createMCInstrAnalysis(MII.get(), MRI.get())); + TheTarget->createMCInstrAnalysis(MII.get())); if (!MIA) { errs() << "BOLT-ERROR: failed to create instruction analysis for target" << TripleName << "\n"; return nullptr; } + + std::unique_ptr MIB( + createMCPlusBuilder(Arch, MIA.get(), MII.get(), MRI.get())); + if (!MIB) { + errs() << "BOLT-ERROR: failed to create instruction builder for target" + << TripleName << "\n"; + return nullptr; + } + int AsmPrinterVariant = AsmInfo->getAssemblerDialect(); std::unique_ptr InstructionPrinter( TheTarget->createMCInstPrinter(Triple(TripleName), AsmPrinterVariant, @@ -684,6 +716,7 @@ createBinaryContext(ELFObjectFileBase *File, DataReader &DR, std::move(STI), std::move(InstructionPrinter), std::move(MIA), + std::move(MIB), std::move(MRI), std::move(DisAsm), DR); @@ -1465,11 +1498,11 @@ void RewriteInstance::disassemblePLT() { exit(1); } - if (!BC->MIA->isIndirectBranch(Instruction)) + if (!BC->MIB->isIndirectBranch(Instruction)) continue; uint64_t TargetAddress; - if (!BC->MIA->evaluateMemOperandTarget(Instruction, + if (!BC->MIB->evaluateMemOperandTarget(Instruction, TargetAddress, InstrAddr, InstrSize)) { diff --git a/bolt/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/Target/AArch64/AArch64MCPlusBuilder.cpp new file mode 100644 index 000000000000..f3745b7a03d1 --- /dev/null +++ b/bolt/Target/AArch64/AArch64MCPlusBuilder.cpp @@ -0,0 +1,1005 @@ +//===-- AArch64MCPlusBuilder.cpp - --------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file provides AArch64-specific MC+ builder. +// +//===----------------------------------------------------------------------===// + +#include "MCPlusBuilder.h" +#include "llvm/MC/MCAsmBackend.h" +#include "llvm/MC/MCCodeEmitter.h" +#include "llvm/MC/MCInstrAnalysis.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/Object/ELF.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/TargetRegistry.h" +#include "InstPrinter/AArch64InstPrinter.h" +#include "MCTargetDesc/AArch64AddressingModes.h" +#include "MCTargetDesc/AArch64MCTargetDesc.h" +#include "MCTargetDesc/AArch64ELFStreamer.h" +#include "MCTargetDesc/AArch64MCAsmInfo.h" +#include "MCTargetDesc/AArch64MCExpr.h" +#include "MCTargetDesc/AArch64WinCOFFStreamer.h" +#include "Utils/AArch64BaseInfo.h" + +#define DEBUG_TYPE "bolt-aarch64" + +using namespace llvm; +using namespace bolt; + +namespace { + +class AArch64MCPlusBuilder : public MCPlusBuilder { +public: + AArch64MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info, + const MCRegisterInfo *RegInfo) + : MCPlusBuilder(Analysis, Info, RegInfo) {} + + bool hasEVEXEncoding(const MCInst &) const override { + return false; + } + + bool shortenInstruction(MCInst &) const override { + return false; + } + + bool isADRP(const MCInst &Inst) const override { + return Inst.getOpcode() == AArch64::ADRP; + } + + bool isADR(const MCInst &Inst) const { + return (Inst.getOpcode() == AArch64::ADR || + Inst.getOpcode() == AArch64::ADRP); + } + + bool isTB(const MCInst &Inst) const { + return (Inst.getOpcode() == AArch64::TBNZW || + Inst.getOpcode() == AArch64::TBNZX || + Inst.getOpcode() == AArch64::TBZW || + Inst.getOpcode() == AArch64::TBZX); + } + + bool isCB(const MCInst &Inst) const { + return (Inst.getOpcode() == AArch64::CBNZW || + Inst.getOpcode() == AArch64::CBNZX || + Inst.getOpcode() == AArch64::CBZW || + Inst.getOpcode() == AArch64::CBZX); + } + + bool isMOVW(const MCInst &Inst) const { + return (Inst.getOpcode() == AArch64::MOVKWi || + Inst.getOpcode() == AArch64::MOVKXi || + Inst.getOpcode() == AArch64::MOVNWi || + Inst.getOpcode() == AArch64::MOVNXi || + Inst.getOpcode() == AArch64::MOVZXi || + Inst.getOpcode() == AArch64::MOVZWi); + } + + bool isADD(const MCInst &Inst) const { + return (Inst.getOpcode() == AArch64::ADDSWri || + Inst.getOpcode() == AArch64::ADDSWrr || + Inst.getOpcode() == AArch64::ADDSWrs || + Inst.getOpcode() == AArch64::ADDSWrx || + Inst.getOpcode() == AArch64::ADDSXri || + Inst.getOpcode() == AArch64::ADDSXrr || + Inst.getOpcode() == AArch64::ADDSXrs || + Inst.getOpcode() == AArch64::ADDSXrx || + Inst.getOpcode() == AArch64::ADDSXrx64 || + Inst.getOpcode() == AArch64::ADDWri || + Inst.getOpcode() == AArch64::ADDWrr || + Inst.getOpcode() == AArch64::ADDWrs || + Inst.getOpcode() == AArch64::ADDWrx || + Inst.getOpcode() == AArch64::ADDXri || + Inst.getOpcode() == AArch64::ADDXrr || + Inst.getOpcode() == AArch64::ADDXrs || + Inst.getOpcode() == AArch64::ADDXrx || + Inst.getOpcode() == AArch64::ADDXrx64); + } + + bool isLDRB(const MCInst &Inst) const { + return (Inst.getOpcode() == AArch64::LDRBBpost || + Inst.getOpcode() == AArch64::LDRBBpre || + Inst.getOpcode() == AArch64::LDRBBroW || + Inst.getOpcode() == AArch64::LDRBBroX || + Inst.getOpcode() == AArch64::LDRBBui || + Inst.getOpcode() == AArch64::LDRSBWpost || + Inst.getOpcode() == AArch64::LDRSBWpre || + Inst.getOpcode() == AArch64::LDRSBWroW || + Inst.getOpcode() == AArch64::LDRSBWroX || + Inst.getOpcode() == AArch64::LDRSBWui || + Inst.getOpcode() == AArch64::LDRSBXpost || + Inst.getOpcode() == AArch64::LDRSBXpre || + Inst.getOpcode() == AArch64::LDRSBXroW || + Inst.getOpcode() == AArch64::LDRSBXroX || + Inst.getOpcode() == AArch64::LDRSBXui); + } + + bool isLDRH(const MCInst &Inst) const { + return (Inst.getOpcode() == AArch64::LDRHHpost || + Inst.getOpcode() == AArch64::LDRHHpre || + Inst.getOpcode() == AArch64::LDRHHroW || + Inst.getOpcode() == AArch64::LDRHHroX || + Inst.getOpcode() == AArch64::LDRHHui || + Inst.getOpcode() == AArch64::LDRSHWpost || + Inst.getOpcode() == AArch64::LDRSHWpre || + Inst.getOpcode() == AArch64::LDRSHWroW || + Inst.getOpcode() == AArch64::LDRSHWroX || + Inst.getOpcode() == AArch64::LDRSHWui || + Inst.getOpcode() == AArch64::LDRSHXpost || + Inst.getOpcode() == AArch64::LDRSHXpre || + Inst.getOpcode() == AArch64::LDRSHXroW || + Inst.getOpcode() == AArch64::LDRSHXroX || + Inst.getOpcode() == AArch64::LDRSHXui); + } + + bool isLDRW(const MCInst &Inst) const { + return (Inst.getOpcode() == AArch64::LDRWpost || + Inst.getOpcode() == AArch64::LDRWpre || + Inst.getOpcode() == AArch64::LDRWroW || + Inst.getOpcode() == AArch64::LDRWroX || + Inst.getOpcode() == AArch64::LDRWui); + } + + bool isLDRX(const MCInst &Inst) const { + return (Inst.getOpcode() == AArch64::LDRXpost || + Inst.getOpcode() == AArch64::LDRXpre || + Inst.getOpcode() == AArch64::LDRXroW || + Inst.getOpcode() == AArch64::LDRXroX || + Inst.getOpcode() == AArch64::LDRXui); + } + + bool isLoad(const MCInst &Inst) const override { + return isLDRB(Inst) || isLDRH(Inst) || isLDRW(Inst) || isLDRX(Inst); + } + + bool isEHLabel(const MCInst &Inst) const override { + return Inst.getOpcode() == AArch64::EH_LABEL; + } + + bool createEHLabel(MCInst &Inst, const MCSymbol *Label, + MCContext *Ctx) const override { + Inst.setOpcode(AArch64::EH_LABEL); + Inst.clear(); + Inst.addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(Label, MCSymbolRefExpr::VK_None, *Ctx))); + return true; + } + + bool isLoadFromStack(const MCInst &Inst) const { + if (!isLoad(Inst)) + return false; + const auto InstInfo = Info->get(Inst.getOpcode()); + auto NumDefs = InstInfo.getNumDefs(); + for (unsigned I = NumDefs, E = Inst.getNumOperands(); I < E; ++I) { + auto &Operand = Inst.getOperand(I); + if (!Operand.isReg()) + continue; + auto Reg = Operand.getReg(); + if (Reg == AArch64::SP || Reg == AArch64::WSP || + Reg == AArch64::FP || Reg == AArch64::W29) + return true; + } + return false; + } + + bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From, + MCPhysReg &To) const override { + if (Inst.getOpcode() != AArch64::ORRXrs) + return false; + if (Inst.getOperand(1).getReg() != AArch64::XZR) + return false; + if (Inst.getOperand(3).getImm() != 0) + return false; + From = Inst.getOperand(2).getReg(); + To = Inst.getOperand(0).getReg(); + return true; + } + + bool isIndirectCall(const MCInst &Inst) const override { + return Inst.getOpcode() == AArch64::BLR; + } + + bool hasPCRelOperand(const MCInst &Inst) const override { + if (isADR(Inst)) + return true; + + // Look for literal addressing mode (see C1-143 ARM DDI 0487B.a) + const auto MCII = Info->get(Inst.getOpcode()); + for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) { + if (MCII.OpInfo[I].OperandType == MCOI::OPERAND_PCREL) + return true; + } + return false; + } + + bool evaluateADR(const MCInst &Inst, int64_t &Imm, + const MCExpr **DispExpr) const { + assert(isADR(Inst) && "Not an ADR instruction"); + + auto &Label = Inst.getOperand(1); + if (!Label.isImm()) { + assert (Label.isExpr() && "Unexpected ADR operand"); + assert (DispExpr && "DispExpr must be set"); + *DispExpr = Label.getExpr(); + return false; + } + + if (Inst.getOpcode() == AArch64::ADR) { + Imm = Label.getImm(); + return true; + } + Imm = Label.getImm() << 12; + return true; + } + + bool evaluateAArch64MemoryOperand(const MCInst &Inst, + int64_t &DispImm, + const MCExpr **DispExpr = nullptr) + const { + if (isADR(Inst)) + return evaluateADR(Inst, DispImm, DispExpr); + + // Literal addressing mode + const auto MCII = Info->get(Inst.getOpcode()); + for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) { + if (MCII.OpInfo[I].OperandType != MCOI::OPERAND_PCREL) + continue; + + if (!Inst.getOperand(I).isImm()) { + assert(Inst.getOperand(I).isExpr() && "Unexpected PCREL operand"); + assert(DispExpr && "DispExpr must be set"); + *DispExpr = Inst.getOperand(I).getExpr(); + return true; + } + + DispImm = Inst.getOperand(I).getImm() << 2; + return true; + } + return false; + } + + bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target, + uint64_t Address, + uint64_t Size) const override { + int64_t DispValue; + const MCExpr* DispExpr{nullptr}; + if (!evaluateAArch64MemoryOperand(Inst, DispValue, &DispExpr)) + return false; + + // Make sure it's a well-formed addressing we can statically evaluate. + if (DispExpr) + return false; + + Target = DispValue; + if (Inst.getOpcode() == AArch64::ADRP) + Target += Address & ~0xFFFULL; + else + Target += Address; + return true; + } + + bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override { + MCInst::iterator OI = Inst.begin(); + if (isADR(Inst)) { + assert(Inst.getNumPrimeOperands() >= 2 && "Unexpected number of operands"); + ++OI; + } else { + const auto MCII = Info->get(Inst.getOpcode()); + for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) { + if (MCII.OpInfo[I].OperandType == MCOI::OPERAND_PCREL) { + break; + } + ++OI; + } + assert(OI != Inst.end() && "Literal operand not found"); + } + OI = Inst.erase(OI); + Inst.insert(OI, Operand); + return true; + } + + const MCExpr *getTargetExprFor(MCInst &Inst, const MCExpr *Expr, + MCContext &Ctx, + uint64_t RelType) const override { + if (RelType == ELF::R_AARCH64_ADR_GOT_PAGE || + RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21) { + // Never emit a GOT reloc, we handled this in + // RewriteInstance::readRelocations(). + return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx); + } else if (Inst.getOpcode() == AArch64::ADRP) { + return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx); + } else { + switch(RelType) { + case ELF::R_AARCH64_LDST8_ABS_LO12_NC: + case ELF::R_AARCH64_LDST16_ABS_LO12_NC: + case ELF::R_AARCH64_LDST32_ABS_LO12_NC: + case ELF::R_AARCH64_LDST64_ABS_LO12_NC: + case ELF::R_AARCH64_LD64_GOT_LO12_NC: + case ELF::R_AARCH64_TLSDESC_LD64_LO12: + case ELF::R_AARCH64_LDST128_ABS_LO12_NC: + case ELF::R_AARCH64_ADD_ABS_LO12_NC: + return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_LO12, Ctx); + default: + break; + } + } + return Expr; + } + + const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override { + auto *AArchExpr = dyn_cast(Expr); + if (AArchExpr && AArchExpr->getSubExpr()) { + return getTargetSymbol(AArchExpr->getSubExpr()); + } + + auto *SymExpr = dyn_cast(Expr); + if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None) + return nullptr; + + return &SymExpr->getSymbol(); + } + + const MCSymbol *getTargetSymbol(const MCInst &Inst, + unsigned OpNum = 0) const override { + if (OpNum >= Inst.getNumOperands()) + return nullptr; + + // Auto-select correct operand number + if (OpNum == 0) { + if (isConditionalBranch(Inst)) + OpNum = 1; + if (isTB(Inst)) + OpNum = 2; + if (isMOVW(Inst)) + OpNum = 1; + } + + auto &Op = Inst.getOperand(OpNum); + if (!Op.isExpr()) + return nullptr; + + return getTargetSymbol(Op.getExpr()); + } + + bool evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size, + uint64_t &Target) const override { + size_t OpNum = 0; + + if (isConditionalBranch(Inst)) { + assert(Inst.getNumPrimeOperands() >= 2 && "Invalid number of operands"); + OpNum = 1; + } + + if (isTB(Inst)) { + assert(Inst.getNumPrimeOperands() >= 3 && "Invalid number of operands"); + OpNum = 2; + } + + if (Info->get(Inst.getOpcode()).OpInfo[OpNum].OperandType != + MCOI::OPERAND_PCREL) { + assert((isIndirectBranch(Inst) || isIndirectCall(Inst)) && + "FAILED evaluateBranch"); + return false; + } + + int64_t Imm = Inst.getOperand(OpNum).getImm() << 2; + Target = Addr + Imm; + return true; + } + + bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx) const override { + assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) && + "Invalid instruction"); + assert(Inst.getNumPrimeOperands() >= 1 && "Invalid number of operands"); + MCInst::iterator OI = Inst.begin(); + + if (isConditionalBranch(Inst)) { + assert(Inst.getNumPrimeOperands() >= 2 && "Invalid number of operands"); + ++OI; + } + + if (isTB(Inst)) { + assert(Inst.getNumPrimeOperands() >= 3 && "Invalid number of operands"); + OI = Inst.begin() + 2; + } + + OI = Inst.erase(OI); + Inst.insert(OI, MCOperand::createExpr(MCSymbolRefExpr::create( + TBB, MCSymbolRefExpr::VK_None, *Ctx))); + return true; + } + + /// Matches indirect branch patterns in AArch64 related to a jump table (JT), + /// helping us to build the complete CFG. A typical indirect branch to + /// a jump table entry in AArch64 looks like the following: + /// + /// adrp x1, #-7585792 # Get JT Page location + /// add x1, x1, #692 # Complement with JT Page offset + /// ldrh w0, [x1, w0, uxtw #1] # Loads JT entry + /// adr x1, #12 # Get PC + 12 (end of this BB) used next + /// add x0, x1, w0, sxth #2 # Finish building branch target + /// # (entries in JT are relative to the end + /// # of this BB) + /// br x0 # Indirect jump instruction + /// + bool analyzeIndirectBranchFragment( + const MCInst &Inst, + DenseMap> &UDChain, + const MCExpr *&JumpTable, int64_t &Offset, int64_t &ScaleValue, + MCInst *&PCRelBase) const { + // Expect AArch64 BR + assert(Inst.getOpcode() == AArch64::BR && "Unexpected opcode"); + + // Match the indirect branch pattern for aarch64 + auto &UsesRoot = UDChain[&Inst]; + if (UsesRoot.size() == 0 || UsesRoot[0] == nullptr) { + return false; + } + const auto *DefAdd = UsesRoot[0]; + + // Now we match an ADD + if (!isADD(*DefAdd)) { + // If the address is not broken up in two parts, this is not branching + // according to a jump table entry. Fail. + return false; + } + if (DefAdd->getOpcode() == AArch64::ADDXri) { + // This can happen when there is no offset, but a direct jump that was + // transformed into an indirect one (indirect tail call) : + // ADRP x2, Perl_re_compiler + // ADD x2, x2, :lo12:Perl_re_compiler + // BR x2 + return false; + } + assert(DefAdd->getOpcode() == AArch64::ADDXrx && + "Failed to match indirect branch!"); + + // Validate ADD operands + auto OperandExtension = DefAdd->getOperand(3).getImm(); + auto ShiftVal = AArch64_AM::getArithShiftValue(OperandExtension); + auto ExtendType = AArch64_AM::getArithExtendType(OperandExtension); + if (ShiftVal != 2) { + llvm_unreachable("Failed to match indirect branch! (fragment 2)"); + } + if (ExtendType == AArch64_AM::SXTB) { + ScaleValue = 1LL; + } else if (ExtendType == AArch64_AM::SXTH) { + ScaleValue = 2LL; + } else if (ExtendType == AArch64_AM::SXTW) { + ScaleValue = 4LL; + } else { + llvm_unreachable("Failed to match indirect branch! (fragment 3)"); + } + + // Match an ADR to load base address to be used when addressing JT targets + auto &UsesAdd = UDChain[DefAdd]; + assert(UsesAdd.size() > 1 && UsesAdd[1] != nullptr && + UsesAdd[2] != nullptr && "Expected definition"); + auto *DefBaseAddr = UsesAdd[1]; + assert(DefBaseAddr->getOpcode() == AArch64::ADR && + "Failed to match indirect branch pattern! (fragment 3)"); + + PCRelBase = DefBaseAddr; + // Match LOAD to load the jump table (relative) target + const auto *DefLoad = UsesAdd[2]; + assert(isLoad(*DefLoad) && + "Failed to match indirect branch load pattern! (1)"); + assert((ScaleValue != 1LL || isLDRB(*DefLoad)) && + "Failed to match indirect branch load pattern! (2)"); + assert((ScaleValue != 2LL || isLDRH(*DefLoad)) && + "Failed to match indirect branch load pattern! (3)"); + + // Match ADD that calculates the JumpTable Base Address (not the offset) + auto &UsesLoad = UDChain[DefLoad]; + const auto *DefJTBaseAdd = UsesLoad[1]; + MCPhysReg From, To; + if (DefJTBaseAdd == nullptr || isLoadFromStack(*DefJTBaseAdd) || + isRegToRegMove(*DefJTBaseAdd, From, To)) { + // Sometimes base address may have been defined in another basic block + // (hoisted). Return with no jump table info. + JumpTable = nullptr; + return true; + } + + assert(DefJTBaseAdd->getOpcode() == AArch64::ADDXri && + "Failed to match jump table base address pattern! (1)"); + + if (DefJTBaseAdd->getOperand(2).isImm()) + Offset = DefJTBaseAdd->getOperand(2).getImm(); + auto &UsesJTBaseAdd = UDChain[DefJTBaseAdd]; + const auto *DefJTBasePage = UsesJTBaseAdd[1]; + if (DefJTBasePage == nullptr || isLoadFromStack(*DefJTBasePage)) { + JumpTable = nullptr; + return true; + } + assert(DefJTBasePage->getOpcode() == AArch64::ADRP && + "Failed to match jump table base page pattern! (2)"); + assert(DefJTBasePage->getOperand(1).isExpr() && + "Failed to match jump table base page pattern! (3)"); + JumpTable = DefJTBasePage->getOperand(1).getExpr(); + return true; + } + + DenseMap> + computeLocalUDChain(const MCInst *CurInstr, + InstructionIterator Begin, + InstructionIterator End) const { + DenseMap RegAliasTable; + DenseMap> Uses; + + auto addInstrOperands = [&](const MCInst &Instr) { + // Update Uses table + for (unsigned OpNum = 0, OpEnd = Instr.getNumOperands(); OpNum != OpEnd; + ++OpNum) { + if (!Instr.getOperand(OpNum).isReg()) + continue; + Uses[&Instr].push_back(RegAliasTable[Instr.getOperand(OpNum).getReg()]); + DEBUG({ + dbgs() << "Adding reg operand " << Instr.getOperand(OpNum).getReg() + << " refs "; + if (RegAliasTable[Instr.getOperand(OpNum).getReg()] != nullptr) + RegAliasTable[Instr.getOperand(OpNum).getReg()]->dump(); + else + dbgs() << "\n"; + }); + } + }; + + DEBUG(dbgs() << "computeLocalUDChain\n"); + bool TerminatorSeen = false; + for (auto II = Begin; II != End; ++II) { + auto &Instr = *II; + // Ignore nops and CFIs + if (Info->get(Instr.getOpcode()).isPseudo() || isNoop(Instr)) + continue; + if (TerminatorSeen) { + RegAliasTable.clear(); + Uses.clear(); + } + + DEBUG(dbgs() << "Now updating for:\n "); + DEBUG(Instr.dump()); + addInstrOperands(Instr); + + BitVector Regs = BitVector(RegInfo->getNumRegs(), false); + getWrittenRegs(Instr, Regs); + + // Update register definitions after this point + int Idx = Regs.find_first(); + while (Idx != -1) { + RegAliasTable[Idx] = &Instr; + DEBUG(dbgs() << "Setting reg " << Idx << " def to current instr.\n"); + Idx = Regs.find_next(Idx); + } + + TerminatorSeen = isTerminator(Instr); + } + + // Process the last instruction, which is not currently added into the + // instruction stream + if (CurInstr) { + addInstrOperands(*CurInstr); + } + return Uses; + } + + IndirectBranchType analyzeIndirectBranch( + MCInst &Instruction, + InstructionIterator Begin, + InstructionIterator End, + const unsigned PtrSize, + MCInst *&MemLocInstrOut, + unsigned &BaseRegNumOut, + unsigned &IndexRegNumOut, + int64_t &DispValueOut, + const MCExpr *&DispExprOut, + MCInst *&PCRelBaseOut + ) const override { + MemLocInstrOut = nullptr; + BaseRegNumOut = AArch64::NoRegister; + IndexRegNumOut = AArch64::NoRegister; + DispValueOut = 0; + DispExprOut = nullptr; + + // An instruction referencing memory used by jump instruction (directly or + // via register). This location could be an array of function pointers + // in case of indirect tail call, or a jump table. + MCInst *MemLocInstr = nullptr; + + // Analyze the memory location. + int64_t ScaleValue, DispValue; + const MCExpr *DispExpr; + + auto UDChain = computeLocalUDChain(&Instruction, Begin, End); + MCInst *PCRelBase; + if (!analyzeIndirectBranchFragment(Instruction, UDChain, DispExpr, + DispValue, ScaleValue, PCRelBase)) { + return IndirectBranchType::UNKNOWN; + } + + MemLocInstrOut = MemLocInstr; + DispValueOut = DispValue; + DispExprOut = DispExpr; + PCRelBaseOut = PCRelBase; + return IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE; + } + + unsigned getInvertedBranchOpcode(unsigned Opcode) const { + switch (Opcode) { + default: + llvm_unreachable("Failed to invert branch opcode"); + return Opcode; + case AArch64::TBZW: return AArch64::TBNZW; + case AArch64::TBZX: return AArch64::TBNZX; + case AArch64::TBNZW: return AArch64::TBZW; + case AArch64::TBNZX: return AArch64::TBZX; + case AArch64::CBZW: return AArch64::CBNZW; + case AArch64::CBZX: return AArch64::CBNZX; + case AArch64::CBNZW: return AArch64::CBZW; + case AArch64::CBNZX: return AArch64::CBZX; + } + } + + bool createCFI(MCInst &Inst, int64_t Offset) const override { + Inst.clear(); + Inst.setOpcode(AArch64::CFI_INSTRUCTION); + Inst.addOperand(MCOperand::createImm(Offset)); + return true; + } + + bool isCFI(const MCInst &Inst) const override { + if (Inst.getOpcode() == AArch64::CFI_INSTRUCTION) + return true; + return false; + } + + bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx) const override { + if (isTB(Inst) || isCB(Inst)) { + Inst.setOpcode(getInvertedBranchOpcode(Inst.getOpcode())); + assert(Inst.getOpcode() != 0 && "Invalid branch instruction"); + } else if (Inst.getOpcode() == AArch64::Bcc) { + Inst.getOperand(0).setImm(AArch64CC::getInvertedCondCode( + static_cast(Inst.getOperand(0).getImm()))); + assert(Inst.getOperand(0).getImm() != AArch64CC::AL && + Inst.getOperand(0).getImm() != AArch64CC::NV && + "Can't reverse ALWAYS cond code"); + } else { + Inst.dump(); + llvm_unreachable("Unrecognized branch instruction"); + } + return replaceBranchTarget(Inst, TBB, Ctx); + } + + int getPCRelEncodingSize(MCInst &Inst) const override { + switch (Inst.getOpcode()) { + default: + llvm_unreachable("Failed to get pcrel encoding size"); + return 0; + case AArch64::TBZW: return 16; + case AArch64::TBZX: return 16; + case AArch64::TBNZW: return 16; + case AArch64::TBNZX: return 16; + case AArch64::CBZW: return 21; + case AArch64::CBZX: return 21; + case AArch64::CBNZW: return 21; + case AArch64::CBNZX: return 21; + case AArch64::B: return 28; + case AArch64::BL: return 28; + case AArch64::Bcc: return 21; + } + } + + int getShortJmpEncodingSize() const override { + return 32; + } + + int getUncondBranchEncodingSize() const override { + return 28; + } + + bool isTailCall(const MCInst &Inst) const override { + auto IsTCOrErr = tryGetAnnotationAs(Inst, "TC"); + if (IsTCOrErr) + return *IsTCOrErr; + if (getConditionalTailCall(Inst)) + return true; + return false; + } + + bool createTailCall(MCInst &Inst, const MCSymbol *Target, + MCContext *Ctx) const override { + Inst.setOpcode(AArch64::B); + Inst.addOperand(MCOperand::createExpr(getTargetExprFor( + Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), + *Ctx, 0))); + addAnnotation(Ctx, Inst, "TC", true); + return true; + } + + bool convertJmpToTailCall(MCInst &Inst, MCContext *Ctx) const override { + addAnnotation(Ctx, Inst, "TC", true); + return true; + } + + bool convertTailCallToJmp(MCInst &Inst) const override { + removeAnnotation(Inst, "TC"); + removeAnnotation(Inst, "Offset"); + if (getConditionalTailCall(Inst)) { + for (auto I = Inst.begin(), E = Inst.end(); I != E; ++I) { + if (I->isConditionalTailCall()) { + Inst.erase(I); + return true; + } + } + } + return true; + } + + bool lowerTailCall(MCInst &Inst) const override { + removeAnnotation(Inst, "TC"); + if (getConditionalTailCall(Inst)) { + for (auto I = Inst.begin(), E = Inst.end(); I != E; ++I) { + if (I->isConditionalTailCall()) { + Inst.erase(I); + return true; + } + } + llvm_unreachable("Tail call operand not found"); + } + return true; + } + + bool isNoop(const MCInst &Inst) const override { + return Inst.getOpcode() == AArch64::HINT && + Inst.getOperand(0).getImm() == 0; + } + + bool createNoop(MCInst &Inst) const override { + Inst.setOpcode(AArch64::HINT); + Inst.clear(); + Inst.addOperand(MCOperand::createImm(0)); + return true; + } + + bool isStore(const MCInst &Inst) const override { + return false; + } + + bool isInvoke(const MCInst &Inst) const override { + return false; + } + + bool analyzeBranch(InstructionIterator Begin, + InstructionIterator End, + const MCSymbol *&TBB, + const MCSymbol *&FBB, + MCInst *&CondBranch, + MCInst *&UncondBranch) const override { + auto I = End; + + while (I != Begin) { + --I; + + // Ignore nops and CFIs + if (Info->get(I->getOpcode()).isPseudo() || isNoop(*I)) + continue; + + // Stop when we find the first non-terminator + if (!isTerminator(*I) || isTailCall(*I) || !isBranch(*I)) + break; + + // Handle unconditional branches. + if (isUnconditionalBranch(*I)) { + // If any code was seen after this unconditional branch, we've seen + // unreachable code. Ignore them. + CondBranch = nullptr; + UncondBranch = &*I; + const auto *Sym = getTargetSymbol(*I); + assert(Sym != nullptr && + "Couldn't extract BB symbol from jump operand"); + TBB = Sym; + continue; + } + + // Handle conditional branches and ignore indirect branches + if (isIndirectBranch(*I)) { + return false; + } + + if (CondBranch == nullptr) { + const auto *TargetBB = getTargetSymbol(*I); + if (TargetBB == nullptr) { + // Unrecognized branch target + return false; + } + FBB = TBB; + TBB = TargetBB; + CondBranch = &*I; + continue; + } + + llvm_unreachable("multiple conditional branches in one BB"); + } + return true; + } + + void createLongJmp(std::vector &Seq, const MCSymbol *Target, + MCContext *Ctx) const override { + // ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call + // Standard for the ARM 64-bit Architecture (AArch64)". + // The sequence of instructions we create here is the following: + // movz ip0, #:abs_g3: + // movk ip0, #:abs_g2_nc: + // movk ip0, #:abs_g1_nc: + // movk ip0, #:abs_g0_nc: + // br ip0 + MCInst Inst; + Inst.setOpcode(AArch64::MOVZXi); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create( + MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), + AArch64MCExpr::VK_ABS_G3, *Ctx))); + Inst.addOperand(MCOperand::createImm(0x30)); + Seq.emplace_back(Inst); + + Inst.clear(); + Inst.setOpcode(AArch64::MOVKXi); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create( + MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), + AArch64MCExpr::VK_ABS_G2_NC, *Ctx))); + Inst.addOperand(MCOperand::createImm(0x20)); + Seq.emplace_back(Inst); + + Inst.clear(); + Inst.setOpcode(AArch64::MOVKXi); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create( + MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), + AArch64MCExpr::VK_ABS_G1_NC, *Ctx))); + Inst.addOperand(MCOperand::createImm(0x10)); + Seq.emplace_back(Inst); + + Inst.clear(); + Inst.setOpcode(AArch64::MOVKXi); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create( + MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), + AArch64MCExpr::VK_ABS_G0_NC, *Ctx))); + Inst.addOperand(MCOperand::createImm(0)); + Seq.emplace_back(Inst); + + Inst.clear(); + Inst.setOpcode(AArch64::BR); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Seq.emplace_back(Inst); + } + + void createShortJmp(std::vector &Seq, const MCSymbol *Target, + MCContext *Ctx) const override { + // ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call + // Standard for the ARM 64-bit Architecture (AArch64)". + // The sequence of instructions we create here is the following: + // movz ip0, #:abs_g1_nc: + // movk ip0, #:abs_g0_nc: + // br ip0 + MCInst Inst; + Inst.setOpcode(AArch64::MOVZXi); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create( + MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), + AArch64MCExpr::VK_ABS_G1_NC, *Ctx))); + Inst.addOperand(MCOperand::createImm(0x10)); + Seq.emplace_back(Inst); + + Inst.clear(); + Inst.setOpcode(AArch64::MOVKXi); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create( + MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx), + AArch64MCExpr::VK_ABS_G0_NC, *Ctx))); + Inst.addOperand(MCOperand::createImm(0)); + Seq.emplace_back(Inst); + + Inst.clear(); + Inst.setOpcode(AArch64::BR); + Inst.addOperand(MCOperand::createReg(AArch64::X16)); + Seq.emplace_back(Inst); + } + + bool replaceImmWithSymbol(MCInst &Inst, MCSymbol *Symbol, int64_t Addend, + MCContext *Ctx, int64_t &Value, + uint64_t RelType) const override { + unsigned ImmOpNo = -1U; + for (unsigned Index = 0; Index < Inst.getNumOperands(); ++Index) { + if (Inst.getOperand(Index).isImm()) { + ImmOpNo = Index; + break; + } + } + if (ImmOpNo == -1U) + return false; + + Value = Inst.getOperand(ImmOpNo).getImm(); + + MCOperand Operand; + if (!Addend) { + Operand = MCOperand::createExpr(getTargetExprFor( + Inst, MCSymbolRefExpr::create(Symbol, *Ctx), *Ctx, RelType)); + } else { + Operand = MCOperand::createExpr(getTargetExprFor( + Inst, + MCBinaryExpr::createAdd(MCSymbolRefExpr::create(Symbol, *Ctx), + MCConstantExpr::create(Addend, *Ctx), *Ctx), + *Ctx, RelType)); + } + Inst.getOperand(ImmOpNo) = Operand; + return true; + } + + bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx) const override { + Inst.setOpcode(AArch64::B); + Inst.clear(); + Inst.addOperand(MCOperand::createExpr(getTargetExprFor( + Inst, MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx), + *Ctx, 0))); + return true; + } + + bool isMoveMem2Reg(const MCInst &Inst) const override { + return false; + } + + bool isADD64rr(const MCInst &Inst) const override { + return false; + } + + bool isLeave(const MCInst &Inst) const override { + return false; + } + + bool isPop(const MCInst &Inst) const override { + return false; + } + + bool isPrefix(const MCInst &Inst) const override { + return false; + } + + bool deleteREPPrefix(MCInst &Inst) const override { + return false; + } + + bool createReturn(MCInst &Inst) const override { + Inst.setOpcode(AArch64::RET); + Inst.clear(); + Inst.addOperand(MCOperand::createReg(AArch64::LR)); + return true; + } +}; + +} // end anonymous namespace + +MCPlusBuilder *createAArch64MCPlusBuilder(const MCInstrAnalysis *Analysis, + const MCInstrInfo *Info, + const MCRegisterInfo *RegInfo) { + return new AArch64MCPlusBuilder(Analysis, Info, RegInfo); +} + diff --git a/bolt/Target/AArch64/CMakeLists.txt b/bolt/Target/AArch64/CMakeLists.txt new file mode 100644 index 000000000000..d3cedac53aa0 --- /dev/null +++ b/bolt/Target/AArch64/CMakeLists.txt @@ -0,0 +1,11 @@ +add_llvm_library(LLVMBOLTTargetAArch64 + AArch64MCPlusBuilder.cpp + + DEPENDS + intrinsics_gen + AArch64CommonTableGen + ) + +include_directories(${LLVM_MAIN_SRC_DIR}/lib/Target/AArch64 ${LLVM_BINARY_DIR}/lib/Target/AArch64) +include_directories(${LLVM_MAIN_SRC_DIR}/tools/llvm-bolt) + diff --git a/bolt/Target/CMakeLists.txt b/bolt/Target/CMakeLists.txt new file mode 100644 index 000000000000..f8e7a0bd9a0c --- /dev/null +++ b/bolt/Target/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(AArch64) +add_subdirectory(X86) diff --git a/bolt/Target/X86/CMakeLists.txt b/bolt/Target/X86/CMakeLists.txt new file mode 100644 index 000000000000..2a55ec2ca61d --- /dev/null +++ b/bolt/Target/X86/CMakeLists.txt @@ -0,0 +1,11 @@ +add_llvm_library(LLVMBOLTTargetX86 + X86MCPlusBuilder.cpp + + DEPENDS + intrinsics_gen + X86CommonTableGen + ) + +include_directories(${LLVM_MAIN_SRC_DIR}/lib/Target/X86 ${LLVM_BINARY_DIR}/lib/Target/X86) +include_directories(${LLVM_MAIN_SRC_DIR}/tools/llvm-bolt) + diff --git a/bolt/Target/X86/X86MCPlusBuilder.cpp b/bolt/Target/X86/X86MCPlusBuilder.cpp new file mode 100644 index 000000000000..3e4e03c33c27 --- /dev/null +++ b/bolt/Target/X86/X86MCPlusBuilder.cpp @@ -0,0 +1,2878 @@ +//===-- X86MCPlusBuilder.cpp ------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file provides X86-specific MC+ builder. +// +//===----------------------------------------------------------------------===// + +#include "MCPlusBuilder.h" +#include "llvm/ADT/Triple.h" +#include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCInstrAnalysis.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MC/MachineLocation.h" +#include "llvm/Support/DataExtractor.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/TargetRegistry.h" +#include "InstPrinter/X86ATTInstPrinter.h" +#include "InstPrinter/X86IntelInstPrinter.h" +#include "MCTargetDesc/X86MCTargetDesc.h" +#include "MCTargetDesc/X86BaseInfo.h" +#include "MCTargetDesc/X86MCAsmInfo.h" +#include + +#define DEBUG_TYPE "bolt-x86" + +using namespace llvm; +using namespace bolt; + +namespace { + +unsigned getShortBranchOpcode(unsigned Opcode) { + switch (Opcode) { + default: + return Opcode; + case X86::JMP_2: return X86::JMP_1; + case X86::JMP_4: return X86::JMP_1; + case X86::JE_2: return X86::JE_1; + case X86::JE_4: return X86::JE_1; + case X86::JNE_2: return X86::JNE_1; + case X86::JNE_4: return X86::JNE_1; + case X86::JL_2: return X86::JL_1; + case X86::JL_4: return X86::JL_1; + case X86::JLE_2: return X86::JLE_1; + case X86::JLE_4: return X86::JLE_1; + case X86::JG_2: return X86::JG_1; + case X86::JG_4: return X86::JG_1; + case X86::JGE_2: return X86::JGE_1; + case X86::JGE_4: return X86::JGE_1; + case X86::JB_2: return X86::JB_1; + case X86::JB_4: return X86::JB_1; + case X86::JBE_2: return X86::JBE_1; + case X86::JBE_4: return X86::JBE_1; + case X86::JA_2: return X86::JA_1; + case X86::JA_4: return X86::JA_1; + case X86::JAE_2: return X86::JAE_1; + case X86::JAE_4: return X86::JAE_1; + case X86::JS_2: return X86::JS_1; + case X86::JS_4: return X86::JS_1; + case X86::JNS_2: return X86::JNS_1; + case X86::JNS_4: return X86::JNS_1; + case X86::JP_2: return X86::JP_1; + case X86::JP_4: return X86::JP_1; + case X86::JNP_2: return X86::JNP_1; + case X86::JNP_4: return X86::JNP_1; + case X86::JO_2: return X86::JO_1; + case X86::JO_4: return X86::JO_1; + case X86::JNO_2: return X86::JNO_1; + case X86::JNO_4: return X86::JNO_1; + } +} + +unsigned getShortArithOpcode(unsigned Opcode) { + switch (Opcode) { + default: + return Opcode; + + // IMUL + case X86::IMUL16rri: return X86::IMUL16rri8; + case X86::IMUL16rmi: return X86::IMUL16rmi8; + case X86::IMUL32rri: return X86::IMUL32rri8; + case X86::IMUL32rmi: return X86::IMUL32rmi8; + case X86::IMUL64rri32: return X86::IMUL64rri8; + case X86::IMUL64rmi32: return X86::IMUL64rmi8; + + // OR + case X86::OR16ri: return X86::OR16ri8; + case X86::OR16mi: return X86::OR16mi8; + case X86::OR32ri: return X86::OR32ri8; + case X86::OR32mi: return X86::OR32mi8; + case X86::OR64ri32: return X86::OR64ri8; + case X86::OR64mi32: return X86::OR64mi8; + + // AND + case X86::AND16ri: return X86::AND16ri8; + case X86::AND16mi: return X86::AND16mi8; + case X86::AND32ri: return X86::AND32ri8; + case X86::AND32mi: return X86::AND32mi8; + case X86::AND64ri32: return X86::AND64ri8; + case X86::AND64mi32: return X86::AND64mi8; + + // XOR + case X86::XOR16ri: return X86::XOR16ri8; + case X86::XOR16mi: return X86::XOR16mi8; + case X86::XOR32ri: return X86::XOR32ri8; + case X86::XOR32mi: return X86::XOR32mi8; + case X86::XOR64ri32: return X86::XOR64ri8; + case X86::XOR64mi32: return X86::XOR64mi8; + + // ADD + case X86::ADD16ri: return X86::ADD16ri8; + case X86::ADD16mi: return X86::ADD16mi8; + case X86::ADD32ri: return X86::ADD32ri8; + case X86::ADD32mi: return X86::ADD32mi8; + case X86::ADD64ri32: return X86::ADD64ri8; + case X86::ADD64mi32: return X86::ADD64mi8; + + // SUB + case X86::SUB16ri: return X86::SUB16ri8; + case X86::SUB16mi: return X86::SUB16mi8; + case X86::SUB32ri: return X86::SUB32ri8; + case X86::SUB32mi: return X86::SUB32mi8; + case X86::SUB64ri32: return X86::SUB64ri8; + case X86::SUB64mi32: return X86::SUB64mi8; + + // CMP + case X86::CMP16ri: return X86::CMP16ri8; + case X86::CMP16mi: return X86::CMP16mi8; + case X86::CMP32ri: return X86::CMP32ri8; + case X86::CMP32mi: return X86::CMP32mi8; + case X86::CMP64ri32: return X86::CMP64ri8; + case X86::CMP64mi32: return X86::CMP64mi8; + + // PUSH + case X86::PUSHi32: return X86::PUSH32i8; + case X86::PUSHi16: return X86::PUSH16i8; + case X86::PUSH64i32: return X86::PUSH64i8; + } +} + +unsigned getInvertedBranchOpcode(unsigned Opcode) { + switch (Opcode) { + default: + return Opcode; + case X86::JE_1: return X86::JNE_1; + case X86::JE_2: return X86::JNE_2; + case X86::JE_4: return X86::JNE_4; + case X86::JNE_1: return X86::JE_1; + case X86::JNE_2: return X86::JE_2; + case X86::JNE_4: return X86::JE_4; + case X86::JL_1: return X86::JGE_1; + case X86::JL_2: return X86::JGE_2; + case X86::JL_4: return X86::JGE_4; + case X86::JLE_1: return X86::JG_1; + case X86::JLE_2: return X86::JG_2; + case X86::JLE_4: return X86::JG_4; + case X86::JG_1: return X86::JLE_1; + case X86::JG_2: return X86::JLE_2; + case X86::JG_4: return X86::JLE_4; + case X86::JGE_1: return X86::JL_1; + case X86::JGE_2: return X86::JL_2; + case X86::JGE_4: return X86::JL_4; + case X86::JB_1: return X86::JAE_1; + case X86::JB_2: return X86::JAE_2; + case X86::JB_4: return X86::JAE_4; + case X86::JBE_1: return X86::JA_1; + case X86::JBE_2: return X86::JA_2; + case X86::JBE_4: return X86::JA_4; + case X86::JA_1: return X86::JBE_1; + case X86::JA_2: return X86::JBE_2; + case X86::JA_4: return X86::JBE_4; + case X86::JAE_1: return X86::JB_1; + case X86::JAE_2: return X86::JB_2; + case X86::JAE_4: return X86::JB_4; + case X86::JS_1: return X86::JNS_1; + case X86::JS_2: return X86::JNS_2; + case X86::JS_4: return X86::JNS_4; + case X86::JNS_1: return X86::JS_1; + case X86::JNS_2: return X86::JS_2; + case X86::JNS_4: return X86::JS_4; + case X86::JP_1: return X86::JNP_1; + case X86::JP_2: return X86::JNP_2; + case X86::JP_4: return X86::JNP_4; + case X86::JNP_1: return X86::JP_1; + case X86::JNP_2: return X86::JP_2; + case X86::JNP_4: return X86::JP_4; + case X86::JO_1: return X86::JNO_1; + case X86::JO_2: return X86::JNO_2; + case X86::JO_4: return X86::JNO_4; + case X86::JNO_1: return X86::JO_1; + case X86::JNO_2: return X86::JO_2; + case X86::JNO_4: return X86::JO_4; + case X86::LOOP: + case X86::LOOPE: + case X86::LOOPNE: + case X86::JECXZ: + case X86::JRCXZ: + // Loop/JCXZ instructions don't have a direct inverse correspondent, so + // inverting them would require more complex code transformations. + llvm_unreachable("Support for properly inverting LOOP/JCXZ " + "instructions is currently unimplemented."); + } +} + +class X86MCPlusBuilder : public MCPlusBuilder { +public: + X86MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info, + const MCRegisterInfo *RegInfo) + : MCPlusBuilder(Analysis, Info, RegInfo) {} + + bool isNoop(const MCInst &Inst) const override { + switch (Inst.getOpcode()) { + case X86::NOOP: + case X86::NOOPL: + case X86::NOOPW: + return true; + } + return false; + } + + bool isPrefix(const MCInst &Inst) const override { + switch (Inst.getOpcode()) { + case X86::LOCK_PREFIX: + case X86::REPNE_PREFIX: + case X86::REP_PREFIX: + return true; + } + return false; + } + + bool deleteREPPrefix(MCInst &Inst) const override { + if (Inst.getFlags() == X86::IP_HAS_REPEAT) { + Inst.setFlags(0); + return true; + } + return false; + } + + // FIXME: For compatibility with old LLVM only! + bool isTerminator(const MCInst &Inst) const override { + return Info->get(Inst.getOpcode()).isTerminator() || + Inst.getOpcode() == X86::UD2B || Inst.getOpcode() == X86::TRAP; + } + + bool isEHLabel(const MCInst &Inst) const override { + return Inst.getOpcode() == X86::EH_LABEL; + } + + bool isIndirectCall(const MCInst &Inst) const override { + return isCall(Inst) && + ((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(0).isReg()); + } + + bool isPop(const MCInst &Inst) const override { + return getPopSize(Inst) == 0 ? false : true; + } + + int getPopSize(const MCInst &Inst) const override { + switch (Inst.getOpcode()) { + case X86::POP16r: + case X86::POP16rmm: + case X86::POP16rmr: + case X86::POPF16: + case X86::POPA16: + case X86::POPDS16: + case X86::POPES16: + case X86::POPFS16: + case X86::POPGS16: + case X86::POPSS16: + return 2; + case X86::POP32r: + case X86::POP32rmm: + case X86::POP32rmr: + case X86::POPA32: + case X86::POPDS32: + case X86::POPES32: + case X86::POPF32: + case X86::POPFS32: + case X86::POPGS32: + case X86::POPSS32: + return 4; + case X86::POP64r: + case X86::POP64rmm: + case X86::POP64rmr: + case X86::POPF64: + case X86::POPFS64: + case X86::POPGS64: + return 8; + } + return 0; + } + + bool isPush(const MCInst &Inst) const override { + return getPushSize(Inst) == 0 ? false : true; + } + + int getPushSize(const MCInst &Inst) const override { + switch (Inst.getOpcode()) { + case X86::PUSH16i8: + case X86::PUSH16r: + case X86::PUSH16rmm: + case X86::PUSH16rmr: + case X86::PUSHA16: + case X86::PUSHCS16: + case X86::PUSHDS16: + case X86::PUSHES16: + case X86::PUSHF16: + case X86::PUSHFS16: + case X86::PUSHGS16: + case X86::PUSHSS16: + case X86::PUSHi16: + return 2; + case X86::PUSH32i8: + case X86::PUSH32r: + case X86::PUSH32rmm: + case X86::PUSH32rmr: + case X86::PUSHA32: + case X86::PUSHCS32: + case X86::PUSHDS32: + case X86::PUSHES32: + case X86::PUSHF32: + case X86::PUSHFS32: + case X86::PUSHGS32: + case X86::PUSHSS32: + case X86::PUSHi32: + return 4; + case X86::PUSH64i32: + case X86::PUSH64i8: + case X86::PUSH64r: + case X86::PUSH64rmm: + case X86::PUSH64rmr: + case X86::PUSHF64: + case X86::PUSHFS64: + case X86::PUSHGS64: + return 8; + } + return 0; + } + + bool isADD64rr(const MCInst &Inst) const override { + return Inst.getOpcode() == X86::ADD64rr; + } + + bool isSUB(const MCInst &Inst) const override { + return Inst.getOpcode() == X86::SUB64rr || + Inst.getOpcode() == X86::SUB64ri32 || + Inst.getOpcode() == X86::SUB64ri8; + } + + bool isADDri(const MCInst &Inst) const { + return Inst.getOpcode() == X86::ADD64ri32 || + Inst.getOpcode() == X86::ADD64ri8; + } + + bool isLEA64r(const MCInst &Inst) const override { + return Inst.getOpcode() == X86::LEA64r; + } + + bool isMOVSX64rm32(const MCInst &Inst) const override { + return Inst.getOpcode() == X86::MOVSX64rm32; + } + + bool isLeave(const MCInst &Inst) const override { + return Inst.getOpcode() == X86::LEAVE || + Inst.getOpcode() == X86::LEAVE64; + } + + bool isEnter(const MCInst &Inst) const override { + return Inst.getOpcode() == X86::ENTER; + } + + bool isMoveMem2Reg(const MCInst &Inst) const override { + switch (Inst.getOpcode()) { + case X86::MOV16rm: + case X86::MOV32rm: + case X86::MOV64rm: + return true; + } + return false; + } + + bool isLoad(const MCInst &Inst) const override { + if (isPop(Inst)) + return true; + + auto MemOpNo = getMemoryOperandNo(Inst); + const auto MCII = Info->get(Inst.getOpcode()); + + if (MemOpNo == -1) + return false; + + return MCII.mayLoad(); + } + + bool isStore(const MCInst &Inst) const override { + if (isPush(Inst)) + return true; + + auto MemOpNo = getMemoryOperandNo(Inst); + const auto MCII = Info->get(Inst.getOpcode()); + + if (MemOpNo == -1) + return false; + + return MCII.mayStore(); + } + + bool isCleanRegXOR(const MCInst &Inst) const override { + switch (Inst.getOpcode()) { + case X86::XOR16rr: + case X86::XOR32rr: + case X86::XOR64rr: + break; + default: + return false; + } + return (Inst.getOperand(0).getReg() == + Inst.getOperand(2).getReg()); + } + + struct IndJmpMatcherFrag1 : MCInstMatcher { + std::unique_ptr Base; + std::unique_ptr Scale; + std::unique_ptr Index; + std::unique_ptr Offset; + + IndJmpMatcherFrag1(std::unique_ptr Base, + std::unique_ptr Scale, + std::unique_ptr Index, + std::unique_ptr Offset) + : Base(std::move(Base)), Scale(std::move(Scale)), + Index(std::move(Index)), Offset(std::move(Offset)) {} + + bool match(const MCRegisterInfo &MRI, const MCPlusBuilder &MIB, + MutableArrayRef InInstrWindow, int OpNum) override { + if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) + return false; + + if (CurInst->getOpcode() != X86::JMP64m) + return false; + + auto MemOpNo = MIB.getMemoryOperandNo(*CurInst); + if (MemOpNo == -1) + return false; + + if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) + return false; + if (!Scale->match(MRI, MIB, this->InstrWindow, + MemOpNo + X86::AddrScaleAmt)) + return false; + if (!Index->match(MRI, MIB, this->InstrWindow, + MemOpNo + X86::AddrIndexReg)) + return false; + if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) + return false; + return true; + } + + void annotate(const MCPlusBuilder &MIB, MCContext &Ctx, + StringRef Annotation) override { + MIB.addAnnotation(&Ctx, *CurInst, Annotation, true); + Base->annotate(MIB, Ctx, Annotation); + Scale->annotate(MIB, Ctx, Annotation); + Index->annotate(MIB, Ctx, Annotation); + Offset->annotate(MIB, Ctx, Annotation); + } + }; + + std::unique_ptr + matchIndJmp(std::unique_ptr Base, + std::unique_ptr Scale, + std::unique_ptr Index, + std::unique_ptr Offset) const override { + return std::unique_ptr( + new IndJmpMatcherFrag1(std::move(Base), std::move(Scale), + std::move(Index), std::move(Offset))); + } + + struct IndJmpMatcherFrag2 : MCInstMatcher { + std::unique_ptr Reg; + + IndJmpMatcherFrag2(std::unique_ptr Reg) + : Reg(std::move(Reg)) {} + + bool match(const MCRegisterInfo &MRI, const MCPlusBuilder &MIB, + MutableArrayRef InInstrWindow, int OpNum) override { + if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) + return false; + + if (CurInst->getOpcode() != X86::JMP64r) + return false; + + return Reg->match(MRI, MIB, this->InstrWindow, 0); + } + + void annotate(const MCPlusBuilder &MIB, MCContext &Ctx, + StringRef Annotation) override { + MIB.addAnnotation(&Ctx, *CurInst, Annotation, true); + Reg->annotate(MIB, Ctx, Annotation); + } + }; + + std::unique_ptr + matchIndJmp(std::unique_ptr Target) const override { + return std::unique_ptr( + new IndJmpMatcherFrag2(std::move(Target))); + } + + struct LoadMatcherFrag1 : MCInstMatcher { + std::unique_ptr Base; + std::unique_ptr Scale; + std::unique_ptr Index; + std::unique_ptr Offset; + + LoadMatcherFrag1(std::unique_ptr Base, + std::unique_ptr Scale, + std::unique_ptr Index, + std::unique_ptr Offset) + : Base(std::move(Base)), Scale(std::move(Scale)), + Index(std::move(Index)), Offset(std::move(Offset)) {} + + bool match(const MCRegisterInfo &MRI, const MCPlusBuilder &MIB, + MutableArrayRef InInstrWindow, int OpNum) override { + if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) + return false; + + if (CurInst->getOpcode() != X86::MOV64rm && + CurInst->getOpcode() != X86::MOVSX64rm32) + return false; + + auto MemOpNo = MIB.getMemoryOperandNo(*CurInst); + if (MemOpNo == -1) + return false; + + if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) + return false; + if (!Scale->match(MRI, MIB, this->InstrWindow, + MemOpNo + X86::AddrScaleAmt)) + return false; + if (!Index->match(MRI, MIB, this->InstrWindow, + MemOpNo + X86::AddrIndexReg)) + return false; + if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) + return false; + return true; + } + + void annotate(const MCPlusBuilder &MIB, MCContext &Ctx, + StringRef Annotation) override { + MIB.addAnnotation(&Ctx, *CurInst, Annotation, true); + Base->annotate(MIB, Ctx, Annotation); + Scale->annotate(MIB, Ctx, Annotation); + Index->annotate(MIB, Ctx, Annotation); + Offset->annotate(MIB, Ctx, Annotation); + } + }; + + std::unique_ptr + matchLoad(std::unique_ptr Base, + std::unique_ptr Scale, + std::unique_ptr Index, + std::unique_ptr Offset) const override { + return std::unique_ptr( + new LoadMatcherFrag1(std::move(Base), std::move(Scale), + std::move(Index), std::move(Offset))); + } + + struct AddMatcher : MCInstMatcher { + std::unique_ptr A; + std::unique_ptr B; + + AddMatcher(std::unique_ptr A, + std::unique_ptr B) + : A(std::move(A)), B(std::move(B)) {} + + bool match(const MCRegisterInfo &MRI, const MCPlusBuilder &MIB, + MutableArrayRef InInstrWindow, int OpNum) override { + if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) + return false; + + if (CurInst->getOpcode() == X86::ADD64rr || + CurInst->getOpcode() == X86::ADD64rr_DB || + CurInst->getOpcode() == X86::ADD64rr_REV) { + if (!A->match(MRI, MIB, this->InstrWindow, 1)) { + if (!B->match(MRI, MIB, this->InstrWindow, 1)) + return false; + return A->match(MRI, MIB, this->InstrWindow, 2); + } + + if (B->match(MRI, MIB, this->InstrWindow, 2)) + return true; + + if (!B->match(MRI, MIB, this->InstrWindow, 1)) + return false; + return A->match(MRI, MIB, this->InstrWindow, 2); + } + + return false; + } + + void annotate(const MCPlusBuilder &MIB, MCContext &Ctx, + StringRef Annotation) override { + MIB.addAnnotation(&Ctx, *CurInst, Annotation, true); + A->annotate(MIB, Ctx, Annotation); + B->annotate(MIB, Ctx, Annotation); + } + }; + + virtual std::unique_ptr + matchAdd(std::unique_ptr A, + std::unique_ptr B) const override { + return std::unique_ptr( + new AddMatcher(std::move(A), std::move(B))); + } + + struct LEAMatcher : MCInstMatcher { + std::unique_ptr Target; + + LEAMatcher(std::unique_ptr Target) + : Target(std::move(Target)) {} + + bool match(const MCRegisterInfo &MRI, const MCPlusBuilder &MIB, + MutableArrayRef InInstrWindow, int OpNum) override { + if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) + return false; + + if (CurInst->getOpcode() != X86::LEA64r) + return false; + + if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 || + CurInst->getOperand(1 + X86::AddrIndexReg).getReg() != + X86::NoRegister || + (CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != + X86::NoRegister && + CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP)) + return false; + + return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp); + } + + void annotate(const MCPlusBuilder &MIB, MCContext &Ctx, + StringRef Annotation) override { + MIB.addAnnotation(&Ctx, *CurInst, Annotation, true); + Target->annotate(MIB, Ctx, Annotation); + } + }; + + virtual std::unique_ptr + matchLoadAddr(std::unique_ptr Target) const override { + return std::unique_ptr(new LEAMatcher(std::move(Target))); + } + + bool hasPCRelOperand(const MCInst &Inst) const override { + for (const auto &Operand : Inst) { + if (Operand.isReg() && Operand.getReg() == X86::RIP) + return true; + } + return false; + } + + int getMemoryOperandNo(const MCInst &Inst) const override { + auto Opcode = Inst.getOpcode(); + auto const &Desc = Info->get(Opcode); + auto MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags); + if (MemOpNo >= 0) + MemOpNo += X86II::getOperandBias(Desc); + return MemOpNo; + } + + bool hasEVEXEncoding(const MCInst &Inst) const override { + auto const &Desc = Info->get(Inst.getOpcode()); + return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX; + } + + bool evaluateX86MemoryOperand(const MCInst &Inst, + unsigned *BaseRegNum, + int64_t *ScaleImm, + unsigned *IndexRegNum, + int64_t *DispImm, + unsigned *SegmentRegNum, + const MCExpr **DispExpr = nullptr) + const override { + assert(BaseRegNum && ScaleImm && IndexRegNum && SegmentRegNum && + "one of the input pointers is null"); + auto MemOpNo = getMemoryOperandNo(Inst); + if (MemOpNo < 0) + return false; + unsigned MemOpOffset = static_cast(MemOpNo); + + if (MemOpOffset + X86::AddrSegmentReg >= Inst.getNumPrimeOperands()) + return false; + + auto &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg); + auto &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt); + auto &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg); + auto &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp); + auto &Segment = Inst.getOperand(MemOpOffset + X86::AddrSegmentReg); + + // Make sure it is a well-formed memory operand. + if (!Base.isReg() || !Scale.isImm() || !Index.isReg() || + (!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg()) + return false; + + *BaseRegNum = Base.getReg(); + *ScaleImm = Scale.getImm(); + *IndexRegNum = Index.getReg(); + if (Disp.isImm()) { + assert(DispImm && "DispImm needs to be set"); + *DispImm = Disp.getImm(); + if (DispExpr) { + *DispExpr = nullptr; + } + } else { + assert(DispExpr && "DispExpr needs to be set"); + *DispExpr = Disp.getExpr(); + } + *SegmentRegNum = Segment.getReg(); + return true; + } + + bool evaluateMemOperandTarget(const MCInst &Inst, + uint64_t &Target, + uint64_t Address, + uint64_t Size) const override { + unsigned BaseRegNum; + int64_t ScaleValue; + unsigned IndexRegNum; + int64_t DispValue; + unsigned SegRegNum; + const MCExpr* DispExpr{nullptr}; + if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum, + &DispValue, &SegRegNum, &DispExpr)) { + return false; + } + + // Make sure it's a well-formed addressing we can statically evaluate. + if ((BaseRegNum != X86::RIP && BaseRegNum != X86::NoRegister) || + IndexRegNum != X86::NoRegister || SegRegNum != X86::NoRegister || + DispExpr) { + return false; + } + Target = DispValue; + if (BaseRegNum == X86::RIP) { + assert(Size != 0 && "instruction size required in order to statically " + "evaluate RIP-relative address"); + Target += Address + Size; + } + return true; + } + + MCInst::iterator getMemOperandDisp(MCInst &Inst) const override { + auto MemOpNo = getMemoryOperandNo(Inst); + if (MemOpNo < 0) + return Inst.end(); + return Inst.begin() + (MemOpNo + X86::AddrDisp); + } + + bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override { + auto OI = getMemOperandDisp(Inst); + if (OI == Inst.end()) + return false; + OI = Inst.erase(OI); + Inst.insert(OI, Operand); + return true; + } + +#if defined(__linux__) && \ + (defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) + /// Get the registers used as function parameters. + /// This function is specific to the x86_64 abi on Linux. + BitVector getRegsUsedAsParams() const override { + BitVector Regs = BitVector(RegInfo->getNumRegs(), false); + Regs |= getAliases(X86::RSI); + Regs |= getAliases(X86::RDI); + Regs |= getAliases(X86::RDX); + Regs |= getAliases(X86::RCX); + Regs |= getAliases(X86::R8); + Regs |= getAliases(X86::R9); + return Regs; + } + + void getCalleeSavedRegs(BitVector &Regs) const override { + Regs |= getAliases(X86::RBX); + Regs |= getAliases(X86::RBP); + Regs |= getAliases(X86::R12); + Regs |= getAliases(X86::R13); + Regs |= getAliases(X86::R14); + Regs |= getAliases(X86::R15); + } + + void getDefaultDefIn(BitVector &Regs) const override { + assert(Regs.size() >= RegInfo->getNumRegs() && + "The size of BitVector is less than RegInfo->getNumRegs()."); + Regs.set(X86::RAX); + Regs.set(X86::RCX); + Regs.set(X86::RDX); + Regs.set(X86::RSI); + Regs.set(X86::RDI); + Regs.set(X86::R8); + Regs.set(X86::R9); + Regs.set(X86::XMM0); + Regs.set(X86::XMM1); + Regs.set(X86::XMM2); + Regs.set(X86::XMM3); + Regs.set(X86::XMM4); + Regs.set(X86::XMM5); + Regs.set(X86::XMM6); + Regs.set(X86::XMM7); + } + + void getDefaultLiveOut(BitVector &Regs) const override { + assert(Regs.size() >= RegInfo->getNumRegs() && + "The size of BitVector is less than RegInfo->getNumRegs()."); + Regs |= getAliases(X86::RAX); + Regs |= getAliases(X86::RDX); + Regs |= getAliases(X86::RCX); + Regs |= getAliases(X86::XMM0); + Regs |= getAliases(X86::XMM1); + } + +#else + BitVector getRegsUsedAsParams() const override { + llvm_unreachable("not implemented"); + return BitVector(); + } + + void getCalleeSavedRegs(BitVector &Regs) const override { + llvm_unreachable("not implemented"); + } +#endif + + void getGPRegs(BitVector &Regs, bool IncludeAlias) const override { + if (IncludeAlias) { + Regs |= getAliases(X86::RAX); + Regs |= getAliases(X86::RBX); + Regs |= getAliases(X86::RBP); + Regs |= getAliases(X86::RSI); + Regs |= getAliases(X86::RDI); + Regs |= getAliases(X86::RDX); + Regs |= getAliases(X86::RCX); + Regs |= getAliases(X86::R8); + Regs |= getAliases(X86::R9); + Regs |= getAliases(X86::R10); + Regs |= getAliases(X86::R11); + Regs |= getAliases(X86::R12); + Regs |= getAliases(X86::R13); + Regs |= getAliases(X86::R14); + Regs |= getAliases(X86::R15); + return; + } + Regs.set(X86::RAX); + Regs.set(X86::RBX); + Regs.set(X86::RBP); + Regs.set(X86::RSI); + Regs.set(X86::RDI); + Regs.set(X86::RDX); + Regs.set(X86::RCX); + Regs.set(X86::R8); + Regs.set(X86::R9); + Regs.set(X86::R10); + Regs.set(X86::R11); + Regs.set(X86::R12); + Regs.set(X86::R13); + Regs.set(X86::R14); + Regs.set(X86::R15); + } + + void getClassicGPRegs(BitVector &Regs) const override { + Regs |= getAliases(X86::RAX); + Regs |= getAliases(X86::RBX); + Regs |= getAliases(X86::RBP); + Regs |= getAliases(X86::RSI); + Regs |= getAliases(X86::RDI); + Regs |= getAliases(X86::RDX); + Regs |= getAliases(X86::RCX); + } + + MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override { + switch (Reg) { + case X86::RAX: case X86::EAX: case X86::AX: case X86::AL: case X86::AH: + switch (Size) { + case 8: return X86::RAX; case 4: return X86::EAX; + case 2: return X86::AX; case 1: return X86::AL; + default: llvm_unreachable("Unexpected size"); + } + case X86::RBX: case X86::EBX: case X86::BX: case X86::BL: case X86::BH: + switch (Size) { + case 8: return X86::RBX; case 4: return X86::EBX; + case 2: return X86::BX; case 1: return X86::BL; + default: llvm_unreachable("Unexpected size"); + } + case X86::RDX: case X86::EDX: case X86::DX: case X86::DL: case X86::DH: + switch (Size) { + case 8: return X86::RDX; case 4: return X86::EDX; + case 2: return X86::DX; case 1: return X86::DL; + default: llvm_unreachable("Unexpected size"); + } + case X86::RDI: case X86::EDI: case X86::DI: case X86::DIL: + switch (Size) { + case 8: return X86::RDI; case 4: return X86::EDI; + case 2: return X86::DI; case 1: return X86::DIL; + default: llvm_unreachable("Unexpected size"); + } + case X86::RSI: case X86::ESI: case X86::SI: case X86::SIL: + switch (Size) { + case 8: return X86::RSI; case 4: return X86::ESI; + case 2: return X86::SI; case 1: return X86::SIL; + default: llvm_unreachable("Unexpected size"); + } + case X86::RCX: case X86::ECX: case X86::CX: case X86::CL: case X86::CH: + switch (Size) { + case 8: return X86::RCX; case 4: return X86::ECX; + case 2: return X86::CX; case 1: return X86::CL; + default: llvm_unreachable("Unexpected size"); + } + case X86::R8: case X86::R8D: case X86::R8W: case X86::R8B: + switch (Size) { + case 8: return X86::R8; case 4: return X86::R8D; + case 2: return X86::R8W; case 1: return X86::R8B; + default: llvm_unreachable("Unexpected size"); + } + case X86::R9: case X86::R9D: case X86::R9W: case X86::R9B: + switch (Size) { + case 8: return X86::R9; case 4: return X86::R9D; + case 2: return X86::R9W; case 1: return X86::R9B; + default: llvm_unreachable("Unexpected size"); + } + case X86::R10: case X86::R10D: case X86::R10W: case X86::R10B: + switch (Size) { + case 8: return X86::R10; case 4: return X86::R10D; + case 2: return X86::R10W; case 1: return X86::R10B; + default: llvm_unreachable("Unexpected size"); + } + case X86::R11: case X86::R11D: case X86::R11W: case X86::R11B: + switch (Size) { + case 8: return X86::R11; case 4: return X86::R11D; + case 2: return X86::R11W; case 1: return X86::R11B; + default: llvm_unreachable("Unexpected size"); + } + case X86::R12: case X86::R12D: case X86::R12W: case X86::R12B: + switch (Size) { + case 8: return X86::R12; case 4: return X86::R12D; + case 2: return X86::R12W; case 1: return X86::R12B; + default: llvm_unreachable("Unexpected size"); + } + case X86::R13: case X86::R13D: case X86::R13W: case X86::R13B: + switch (Size) { + case 8: return X86::R13; case 4: return X86::R13D; + case 2: return X86::R13W; case 1: return X86::R13B; + default: llvm_unreachable("Unexpected size"); + } + case X86::R14: case X86::R14D: case X86::R14W: case X86::R14B: + switch (Size) { + case 8: return X86::R14; case 4: return X86::R14D; + case 2: return X86::R14W; case 1: return X86::R14B; + default: llvm_unreachable("Unexpected size"); + } + case X86::R15: case X86::R15D: case X86::R15W: case X86::R15B: + switch (Size) { + case 8: return X86::R15; case 4: return X86::R15D; + case 2: return X86::R15W; case 1: return X86::R15B; + default: llvm_unreachable("Unexpected size"); + } + default: + dbgs() << Reg << " (get alias sized)\n"; + llvm_unreachable("Unexpected reg number"); + break; + } + } + + bool isUpper8BitReg(MCPhysReg Reg) const override { + switch (Reg) { + case X86::AH: case X86::BH: case X86::CH: case X86::DH: + return true; + default: + return false; + } + } + + bool cannotUseREX(const MCInst &Inst) const override { + switch(Inst.getOpcode()) { + case X86::MOV8mr_NOREX: + case X86::MOV8rm_NOREX: + case X86::MOV8rr_NOREX: + case X86::MOVSX32_NOREXrm8: + case X86::MOVSX32_NOREXrr8: + case X86::MOVZX32_NOREXrm8: + case X86::MOVZX32_NOREXrr8: + case X86::MOV8mr: + case X86::MOV8rm: + case X86::MOV8rr: + case X86::MOVSX32rm8: + case X86::MOVSX32rr8: + case X86::MOVZX32rm8: + case X86::MOVZX32rr8: + case X86::TEST8ri: + for (int I = 0, E = Inst.getNumPrimeOperands(); I != E; ++I) { + const auto &Operand = Inst.getOperand(I); + if (!Operand.isReg()) + continue; + if (isUpper8BitReg(Operand.getReg())) + return true; + } + // Fall-through + default: + return false; + } + } + + bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore, + bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm, + uint16_t &StackPtrReg, int64_t &StackOffset, + uint8_t &Size, bool &IsSimple, + bool &IsIndexed) const override { + // Detect simple push/pop cases first + if (auto Sz = getPushSize(Inst)) { + IsLoad = false; + IsStore = true; + IsStoreFromReg = true; + StackPtrReg = X86::RSP; + StackOffset = -Sz; + Size = Sz; + IsSimple = true; + if (Inst.getOperand(0).isImm()) { + SrcImm = Inst.getOperand(0).getImm(); + } else if (Inst.getOperand(0).isReg()) { + Reg = Inst.getOperand(0).getReg(); + } else { + IsSimple = false; + } + return true; + } + if (auto Sz = getPopSize(Inst)) { + assert(Inst.getOperand(0).isReg() && + "Expected register operand for push"); + IsLoad = true; + IsStore = false; + Reg = Inst.getOperand(0).getReg(); + StackPtrReg = X86::RSP; + StackOffset = 0; + Size = Sz; + IsSimple = true; + return true; + } + + struct InstInfo { + // Size in bytes that Inst loads from memory. + uint8_t DataSize; + bool IsLoad; + bool IsStore; + bool StoreFromReg; + bool Simple; + }; + + InstInfo I; + auto MemOpNo = getMemoryOperandNo(Inst); + const auto MCII = Info->get(Inst.getOpcode()); + // If it is not dealing with a memory operand, we discard it + if (MemOpNo == -1 || MCII.isCall()) + return false; + + switch (Inst.getOpcode()) { + default: { + uint8_t Sz = 0; + bool IsLoad = MCII.mayLoad(); + bool IsStore = MCII.mayStore(); + // Is it LEA? (deals with memory but is not loading nor storing) + if (!IsLoad && !IsStore) + return false; + + // Try to guess data size involved in the load/store by looking at the + // register size. If there's no reg involved, return 0 as size, meaning + // we don't know. + for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) { + if (MCII.OpInfo[I].OperandType != MCOI::OPERAND_REGISTER) + continue; + if (static_cast(I) >= MemOpNo && I < X86::AddrNumOperands) + continue; + Sz = RegInfo->getRegClass(MCII.OpInfo[I].RegClass).getPhysRegSize(); + break; + } + I = {Sz, IsLoad, IsStore, false, false}; + break; + } + case X86::MOV16rm: I = {2, true, false, false, true}; break; + case X86::MOV32rm: I = {4, true, false, false, true}; break; + case X86::MOV64rm: I = {8, true, false, false, true}; break; + case X86::MOV16mr: I = {2, false, true, true, true}; break; + case X86::MOV32mr: I = {4, false, true, true, true}; break; + case X86::MOV64mr: I = {8, false, true, true, true}; break; + case X86::MOV16mi: I = {2, false, true, false, true}; break; + case X86::MOV32mi: I = {4, false, true, false, true}; break; + } // end switch (Inst.getOpcode()) + + unsigned BaseRegNum; + int64_t ScaleValue; + unsigned IndexRegNum; + int64_t DispValue; + unsigned SegRegNum; + const MCExpr *DispExpr; + if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum, + &DispValue, &SegRegNum, &DispExpr)) { + DEBUG(dbgs() << "Evaluate failed on "); + DEBUG(Inst.dump()); + return false; + } + + // Make sure it's a stack access + if (BaseRegNum != X86::RBP && BaseRegNum != X86::RSP) { + return false; + } + + IsLoad = I.IsLoad; + IsStore = I.IsStore; + IsStoreFromReg = I.StoreFromReg; + Size = I.DataSize; + IsSimple = I.Simple; + StackPtrReg = BaseRegNum; + StackOffset = DispValue; + IsIndexed = IndexRegNum != X86::NoRegister || SegRegNum != X86::NoRegister; + + if (!I.Simple) + return true; + + // Retrieve related register in simple MOV from/to stack operations. + unsigned MemOpOffset = static_cast(MemOpNo); + if (I.IsLoad) { + auto RegOpnd = Inst.getOperand(0); + assert(RegOpnd.isReg() && "unexpected destination operand"); + Reg = RegOpnd.getReg(); + } else if (I.IsStore) { + auto SrcOpnd = Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); + if (I.StoreFromReg) { + assert(SrcOpnd.isReg() && "unexpected source operand"); + Reg = SrcOpnd.getReg(); + } else { + assert(SrcOpnd.isImm() && "unexpected source operand"); + SrcImm = SrcOpnd.getImm(); + } + } + + return true; + } + + void changeToPushOrPop(MCInst &Inst) const override { + assert(!isPush(Inst) && !isPop(Inst)); + + struct InstInfo { + // Size in bytes that Inst loads from memory. + uint8_t DataSize; + bool IsLoad; + bool StoreFromReg; + }; + + InstInfo I; + switch (Inst.getOpcode()) { + default: { + llvm_unreachable("Unhandled opcode"); + return; + } + case X86::MOV16rm: I = {2, true, false}; break; + case X86::MOV32rm: I = {4, true, false}; break; + case X86::MOV64rm: I = {8, true, false}; break; + case X86::MOV16mr: I = {2, false, true}; break; + case X86::MOV32mr: I = {4, false, true}; break; + case X86::MOV64mr: I = {8, false, true}; break; + case X86::MOV16mi: I = {2, false, false}; break; + case X86::MOV32mi: I = {4, false, false}; break; + } // end switch (Inst.getOpcode()) + + unsigned BaseRegNum; + int64_t ScaleValue; + unsigned IndexRegNum; + int64_t DispValue; + unsigned SegRegNum; + const MCExpr *DispExpr; + if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, &IndexRegNum, + &DispValue, &SegRegNum, &DispExpr)) { + llvm_unreachable("Evaluate failed"); + return; + } + // Make sure it's a stack access + if (BaseRegNum != X86::RBP && BaseRegNum != X86::RSP) { + llvm_unreachable("Not a stack access"); + return; + } + + unsigned MemOpOffset = getMemoryOperandNo(Inst); + unsigned NewOpcode = 0; + if (I.IsLoad) { + switch (I.DataSize) { + case 2: NewOpcode = X86::POP16r; break; + case 4: NewOpcode = X86::POP32r; break; + case 8: NewOpcode = X86::POP64r; break; + default: + assert(false); + } + auto RegOpndNum = Inst.getOperand(0).getReg(); + Inst.clear(); + Inst.setOpcode(NewOpcode); + Inst.addOperand(MCOperand::createReg(RegOpndNum)); + } else { + auto SrcOpnd = Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); + if (I.StoreFromReg) { + switch (I.DataSize) { + case 2: NewOpcode = X86::PUSH16r; break; + case 4: NewOpcode = X86::PUSH32r; break; + case 8: NewOpcode = X86::PUSH64r; break; + default: + assert(false); + } + assert(SrcOpnd.isReg() && "unexpected source operand"); + auto RegOpndNum = SrcOpnd.getReg(); + Inst.clear(); + Inst.setOpcode(NewOpcode); + Inst.addOperand(MCOperand::createReg(RegOpndNum)); + } else { + switch (I.DataSize) { + case 2: NewOpcode = X86::PUSH16i8; break; + case 4: NewOpcode = X86::PUSH32i8; break; + case 8: NewOpcode = X86::PUSH64i32; break; + default: + assert(false); + } + assert(SrcOpnd.isImm() && "unexpected source operand"); + auto SrcImm = SrcOpnd.getImm(); + Inst.clear(); + Inst.setOpcode(NewOpcode); + Inst.addOperand(MCOperand::createImm(SrcImm)); + } + } + } + + bool isStackAdjustment(const MCInst &Inst) const override { + switch (Inst.getOpcode()) { + default: + return false; + case X86::SUB64ri32: + case X86::SUB64ri8: + case X86::ADD64ri32: + case X86::ADD64ri8: + case X86::LEA64r: + break; + } + + const auto MCII = Info->get(Inst.getOpcode()); + for (int I = 0, E = MCII.getNumDefs(); I != E; ++I) { + const auto &Operand = Inst.getOperand(I); + if (Operand.isReg() && Operand.getReg() == X86::RSP) { + return true; + } + } + return false; + } + + bool evaluateSimple(const MCInst &Inst, int64_t &Output, + std::pair Input1, + std::pair Input2) const override { + + auto getOperandVal = [&] (MCPhysReg Reg) -> ErrorOr { + if (Reg == Input1.first) + return Input1.second; + if (Reg == Input2.first) + return Input2.second; + return make_error_code(errc::result_out_of_range); + }; + + switch (Inst.getOpcode()) { + default: + return false; + + case X86::AND64ri32: + case X86::AND64ri8: + assert(Inst.getOperand(2).isImm()); + if (auto InputVal = getOperandVal(Inst.getOperand(1).getReg())) { + Output = *InputVal & Inst.getOperand(2).getImm(); + } else { + return false; + } + break; + case X86::SUB64ri32: + case X86::SUB64ri8: + assert(Inst.getOperand(2).isImm()); + if (auto InputVal = getOperandVal(Inst.getOperand(1).getReg())) { + Output = *InputVal - Inst.getOperand(2).getImm(); + } else { + return false; + } + break; + case X86::ADD64ri32: + case X86::ADD64ri8: + assert(Inst.getOperand(2).isImm()); + if (auto InputVal = getOperandVal(Inst.getOperand(1).getReg())) { + Output = *InputVal + Inst.getOperand(2).getImm(); + } else { + return false; + } + break; + + case X86::LEA64r: { + unsigned BaseRegNum; + int64_t ScaleValue; + unsigned IndexRegNum; + int64_t DispValue; + unsigned SegRegNum; + const MCExpr *DispExpr{nullptr}; + if (!evaluateX86MemoryOperand(Inst, &BaseRegNum, &ScaleValue, + &IndexRegNum, &DispValue, &SegRegNum, + &DispExpr)) { + return false; + } + + if (BaseRegNum == X86::NoRegister || IndexRegNum != X86::NoRegister || + SegRegNum != X86::NoRegister || DispExpr) { + return false; + } + + if (auto InputVal = getOperandVal(BaseRegNum)) { + Output = *InputVal + DispValue; + } else { + return false; + } + + break; + } + } + return true; + } + + bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From, + MCPhysReg &To) const override { + switch (Inst.getOpcode()) { + default: + return false; + case X86::LEAVE: + case X86::LEAVE64: + To = getStackPointer(); + From = getFramePointer(); + return true; + case X86::MOV64rr: + To = Inst.getOperand(0).getReg(); + From = Inst.getOperand(1).getReg(); + return true; + } + } + + MCPhysReg getStackPointer() const override { return X86::RSP; } + MCPhysReg getFramePointer() const override { return X86::RBP; } + MCPhysReg getFlagsReg() const override { return X86::EFLAGS; } + + bool escapesVariable(const MCInst &Inst, bool HasFramePointer) const override { + auto MemOpNo = getMemoryOperandNo(Inst); + const auto MCII = Info->get(Inst.getOpcode()); + const auto NumDefs = MCII.getNumDefs(); + static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) | + getAliases(X86::RBP)); + static BitVector SPAliases(getAliases(X86::RSP)); + + // FIXME: PUSH can be technically a leak, but let's ignore this for now + // because a lot of harmless prologue code will spill SP to the stack + // (t15117648) - Unless push is clearly pushing an object address to the + // stack as demonstrated by having a MemOp. + bool IsPush = isPush(Inst); + if (IsPush && MemOpNo == -1) + return false; + + // We use this to detect LEA (has memop but does not access mem) + bool AccessMem = MCII.mayLoad() || MCII.mayStore(); + bool DoesLeak = false; + for (int I = 0, E = Inst.getNumPrimeOperands(); I != E; ++I) { + // Ignore if SP/BP is used to derefence memory -- that's fine + if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo && + I <= MemOpNo + 5) + continue; + // Ignore if someone is writing to SP/BP + if (I < static_cast(NumDefs)) + continue; + + const auto &Operand = Inst.getOperand(I); + if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) { + DoesLeak = true; + break; + } + if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) { + DoesLeak = true; + break; + } + } + + // If potential leak, check if it is not just writing to itself/sp/bp + if (DoesLeak) { + for (int I = 0, E = NumDefs; I != E; ++I) { + const auto &Operand = Inst.getOperand(I); + if (HasFramePointer && Operand.isReg() && + SPBPAliases[Operand.getReg()]) { + DoesLeak = false; + break; + } + if (!HasFramePointer && Operand.isReg() && + SPAliases[Operand.getReg()]) { + DoesLeak = false; + break; + } + } + } + return DoesLeak; + } + + bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override { + unsigned ImmOpNo = -1U; + auto MemOpNo = getMemoryOperandNo(Inst); + if (MemOpNo != -1) { + ImmOpNo = MemOpNo + X86::AddrDisp; + } else { + for (unsigned Index = 0; Index < Inst.getNumPrimeOperands(); ++Index) { + if (Inst.getOperand(Index).isImm()) { + ImmOpNo = Index; + } + } + } + if (ImmOpNo == -1U) + return false; + + MCOperand &Operand = Inst.getOperand(ImmOpNo); + Amt += Operand.getImm(); + Operand.setImm(Amt); + // Check for the need for relaxation + if (int64_t(Amt) == int64_t(int8_t(Amt))) + return true; + + // Relax instruction + switch (Inst.getOpcode()) { + case X86::SUB64ri8: + Inst.setOpcode(X86::SUB64ri32); + break; + case X86::ADD64ri8: + Inst.setOpcode(X86::ADD64ri32); + break; + default: + // No need for relaxation + break; + } + return true; + } + + /// TODO: this implementation currently works for the most common opcodes that + /// load from memory. It can be extended to work with memory store opcodes as + /// well as more memory load opcodes. + bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData, + uint32_t Offset) const override { + enum CheckSignExt : uint8_t { + NOCHECK = 0, + CHECK8, + CHECK32, + }; + + using CheckList = std::vector>; + struct InstInfo { + // Size in bytes that Inst loads from memory. + uint8_t DataSize; + + // True when the target operand has to be duplicated because the opcode + // expects a LHS operand. + bool HasLHS; + + // List of checks and corresponding opcodes to be used. We try to use the + // smallest possible immediate value when various sizes are available, + // hence we may need to check whether a larger constant fits in a smaller + // immediate. + CheckList Checks; + }; + + InstInfo I; + + switch (Inst.getOpcode()) { + default: { + switch (getPopSize(Inst)) { + case 2: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; + case 4: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; + case 8: I = {8, false, {{CHECK32, X86::MOV64ri32}, + {NOCHECK, X86::MOV64rm}}}; break; + default: return false; + } + break; + } + + // MOV + case X86::MOV8rm: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break; + case X86::MOV16rm: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; + case X86::MOV32rm: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; + case X86::MOV64rm: I = {8, false, {{CHECK32, X86::MOV64ri32}, + {NOCHECK, X86::MOV64rm}}}; break; + + // MOVZX + case X86::MOVZX16rm8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break; + case X86::MOVZX32rm8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break; + case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break; + + // CMP + case X86::CMP8rm: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break; + case X86::CMP16rm: I = {2, false, {{CHECK8, X86::CMP16ri8}, + {NOCHECK, X86::CMP16ri}}}; break; + case X86::CMP32rm: I = {4, false, {{CHECK8, X86::CMP32ri8}, + {NOCHECK, X86::CMP32ri}}}; break; + case X86::CMP64rm: I = {8, false, {{CHECK8, X86::CMP64ri8}, + {CHECK32, X86::CMP64ri32}, + {NOCHECK, X86::CMP64rm}}}; break; + + // TEST + case X86::TEST8mr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break; + case X86::TEST16mr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break; + case X86::TEST32mr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break; + case X86::TEST64mr: I = {8, false, {{CHECK32, X86::TEST64ri32}, + {NOCHECK, X86::TEST64mr}}}; break; + + // ADD + case X86::ADD8rm: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break; + case X86::ADD16rm: I = {2, true, {{CHECK8, X86::ADD16ri8}, + {NOCHECK, X86::ADD16ri}}}; break; + case X86::ADD32rm: I = {4, true, {{CHECK8, X86::ADD32ri8}, + {NOCHECK, X86::ADD32ri}}}; break; + case X86::ADD64rm: I = {8, true, {{CHECK8, X86::ADD64ri8}, + {CHECK32, X86::ADD64ri32}, + {NOCHECK, X86::ADD64rm}}}; break; + + // SUB + case X86::SUB8rm: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break; + case X86::SUB16rm: I = {2, true, {{CHECK8, X86::SUB16ri8}, + {NOCHECK, X86::SUB16ri}}}; break; + case X86::SUB32rm: I = {4, true, {{CHECK8, X86::SUB32ri8}, + {NOCHECK, X86::SUB32ri}}}; break; + case X86::SUB64rm: I = {8, true, {{CHECK8, X86::SUB64ri8}, + {CHECK32, X86::SUB64ri32}, + {NOCHECK, X86::SUB64rm}}}; break; + + // AND + case X86::AND8rm: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break; + case X86::AND16rm: I = {2, true, {{CHECK8, X86::AND16ri8}, + {NOCHECK, X86::AND16ri}}}; break; + case X86::AND32rm: I = {4, true, {{CHECK8, X86::AND32ri8}, + {NOCHECK, X86::AND32ri}}}; break; + case X86::AND64rm: I = {8, true, {{CHECK8, X86::AND64ri8}, + {CHECK32, X86::AND64ri32}, + {NOCHECK, X86::AND64rm}}}; break; + + // OR + case X86::OR8rm: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break; + case X86::OR16rm: I = {2, true, {{CHECK8, X86::OR16ri8}, + {NOCHECK, X86::OR16ri}}}; break; + case X86::OR32rm: I = {4, true, {{CHECK8, X86::OR32ri8}, + {NOCHECK, X86::OR32ri}}}; break; + case X86::OR64rm: I = {8, true, {{CHECK8, X86::OR64ri8}, + {CHECK32, X86::OR64ri32}, + {NOCHECK, X86::OR64rm}}}; break; + + // XOR + case X86::XOR8rm: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break; + case X86::XOR16rm: I = {2, true, {{CHECK8, X86::XOR16ri8}, + {NOCHECK, X86::XOR16ri}}}; break; + case X86::XOR32rm: I = {4, true, {{CHECK8, X86::XOR32ri8}, + {NOCHECK, X86::XOR32ri}}}; break; + case X86::XOR64rm: I = {8, true, {{CHECK8, X86::XOR64ri8}, + {CHECK32, X86::XOR64ri32}, + {NOCHECK, X86::XOR64rm}}}; break; + } + + // Compute the immediate value. + assert(Offset + I.DataSize <= ConstantData.size() && + "invalid offset for given constant data"); + int64_t ImmVal = + DataExtractor(ConstantData, true, 64).getSigned(&Offset, I.DataSize); + + // Compute the new opcode. + unsigned NewOpcode = 0; + for (const auto &Check : I.Checks) { + NewOpcode = Check.second; + if (Check.first == NOCHECK) + break; + else if (Check.first == CHECK8 && + ImmVal >= std::numeric_limits::min() && + ImmVal <= std::numeric_limits::max()) + break; + else if (Check.first == CHECK32 && + ImmVal >= std::numeric_limits::min() && + ImmVal <= std::numeric_limits::max()) + break; + } + if (NewOpcode == Inst.getOpcode()) + return false; + + // Modify the instruction. + MCOperand ImmOp = MCOperand::createImm(ImmVal); + uint32_t TargetOpNum{0}; + // Test instruction does not follow the regular pattern of putting the + // memory reference of a load (5 MCOperands) last in the list of operands. + // Since it is not modifying the register operand, it is not treated as + // a destination operand and it is not the first operand as it is in the + // other instructions we treat here. + if (NewOpcode == X86::TEST8ri || + NewOpcode == X86::TEST16ri || + NewOpcode == X86::TEST32ri || + NewOpcode == X86::TEST64ri32) { + TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands; + } + MCOperand TargetOp = Inst.getOperand(TargetOpNum); + Inst.clear(); + Inst.setOpcode(NewOpcode); + Inst.addOperand(TargetOp); + if (I.HasLHS) + Inst.addOperand(TargetOp); + Inst.addOperand(ImmOp); + + return true; + } + + /// TODO: this implementation currently works for the most common opcodes that + /// load from memory. It can be extended to work with memory store opcodes as + /// well as more memory load opcodes. + bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override { + unsigned NewOpcode; + + switch (Inst.getOpcode()) { + default: { + switch (getPopSize(Inst)) { + case 2: NewOpcode = X86::MOV16rr; break; + case 4: NewOpcode = X86::MOV32rr; break; + case 8: NewOpcode = X86::MOV64rr; break; + default: return false; + } + break; + } + + // MOV + case X86::MOV8rm: NewOpcode = X86::MOV8rr; break; + case X86::MOV16rm: NewOpcode = X86::MOV16rr; break; + case X86::MOV32rm: NewOpcode = X86::MOV32rr; break; + case X86::MOV64rm: NewOpcode = X86::MOV64rr; break; + } + + // Modify the instruction. + MCOperand RegOp = MCOperand::createReg(RegNum); + MCOperand TargetOp = Inst.getOperand(0); + Inst.clear(); + Inst.setOpcode(NewOpcode); + Inst.addOperand(TargetOp); + Inst.addOperand(RegOp); + + return true; + } + + bool isRedundantMove(const MCInst &Inst) const override { + switch (Inst.getOpcode()) { + default: + return false; + + // MOV + case X86::MOV8rr: + case X86::MOV16rr: + case X86::MOV32rr: + case X86::MOV64rr: + break; + } + + assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg()); + return Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg(); + } + + bool isTailCall(const MCInst &Inst) const override { + switch (Inst.getOpcode()) { + case X86::TAILJMPd: + case X86::TAILJMPm: + case X86::TAILJMPr: + return true; + } + + if (getConditionalTailCall(Inst)) + return true; + + return false; + } + + bool requiresAlignedAddress(const MCInst &Inst) const override { + auto const &Desc = Info->get(Inst.getOpcode()); + for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) { + const auto &Op = Desc.OpInfo[I]; + if (Op.OperandType != MCOI::OPERAND_REGISTER) + continue; + if (Op.RegClass == X86::VR128RegClassID) + return true; + } + return false; + } + + bool convertJmpToTailCall(MCInst &Inst, MCContext *) const override { + int NewOpcode; + switch (Inst.getOpcode()) { + default: + return false; + case X86::JMP_1: + case X86::JMP_2: + case X86::JMP_4: + NewOpcode = X86::TAILJMPd; + break; + case X86::JMP16m: + case X86::JMP32m: + case X86::JMP64m: + NewOpcode = X86::TAILJMPm; + break; + case X86::JMP16r: + case X86::JMP32r: + case X86::JMP64r: + NewOpcode = X86::TAILJMPr; + break; + } + + Inst.setOpcode(NewOpcode); + return true; + } + + bool convertTailCallToCall(MCInst &Inst) const override { + int NewOpcode; + switch (Inst.getOpcode()) { + default: + return false; + case X86::TAILJMPd: + NewOpcode = X86::CALL64pcrel32; + break; + case X86::TAILJMPm: + NewOpcode = X86::CALL64m; + break; + case X86::TAILJMPr: + NewOpcode = X86::CALL64r; + break; + } + + Inst.setOpcode(NewOpcode); + return true; + } + + bool convertCallToIndirectCall(MCInst &Inst, + const MCSymbol *TargetLocation, + MCContext *Ctx) const override { + assert((Inst.getOpcode() == X86::CALL64pcrel32 || + Inst.getOpcode() == X86::TAILJMPd) && + "64-bit direct (tail) call instruction expected"); + const auto NewOpcode = (Inst.getOpcode() == X86::CALL64pcrel32) + ? X86::CALL64m + : X86::TAILJMPm; + Inst.setOpcode(NewOpcode); + + // Replace the first operand and preserve auxiliary operands of + // the instruction. + Inst.erase(Inst.begin()); + Inst.insert(Inst.begin(), + MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg + Inst.insert(Inst.begin(), + MCOperand::createExpr( // Displacement + MCSymbolRefExpr::create(TargetLocation, + MCSymbolRefExpr::VK_None, + *Ctx))); + Inst.insert(Inst.begin(), + MCOperand::createReg(X86::NoRegister)); // IndexReg + Inst.insert(Inst.begin(), + MCOperand::createImm(1)); // ScaleAmt + Inst.insert(Inst.begin(), + MCOperand::createReg(X86::RIP)); // BaseReg + + return true; + } + + bool shortenInstruction(MCInst &Inst) const override { + unsigned OldOpcode = Inst.getOpcode(); + unsigned NewOpcode = OldOpcode; + + if (isBranch(Inst)) { + NewOpcode = getShortBranchOpcode(OldOpcode); + } else if (OldOpcode == X86::MOV64ri) { + if (Inst.getOperand(Inst.getNumPrimeOperands() - 1).isImm()) { + const auto Imm = + Inst.getOperand(Inst.getNumPrimeOperands() - 1).getImm(); + if (int64_t(Imm) == int64_t(int32_t(Imm))) { + NewOpcode = X86::MOV64ri32; + } + } + } else { + // If it's arithmetic instruction check if signed operand fits in 1 byte. + const auto ShortOpcode = getShortArithOpcode(OldOpcode); + if (ShortOpcode != OldOpcode && + Inst.getOperand(Inst.getNumPrimeOperands() - 1).isImm()) { + auto Imm = Inst.getOperand(Inst.getNumPrimeOperands() - 1).getImm(); + if (int64_t(Imm) == int64_t(int8_t(Imm))) { + NewOpcode = ShortOpcode; + } + } + } + + if (NewOpcode == OldOpcode) + return false; + + Inst.setOpcode(NewOpcode); + return true; + } + + bool lowerTailCall(MCInst &Inst) const override { + if (Inst.getOpcode() == X86::TAILJMPd) { + Inst.setOpcode(X86::JMP_1); + return true; + } + return false; + } + + const MCSymbol *getTargetSymbol(const MCInst &Inst, + unsigned OpNum = 0) const override { + if (OpNum >= Inst.getNumPrimeOperands()) + return nullptr; + + auto &Op = Inst.getOperand(OpNum); + if (!Op.isExpr()) + return nullptr; + + auto *SymExpr = dyn_cast(Op.getExpr()); + if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None) + return nullptr; + + return &SymExpr->getSymbol(); + } + + bool analyzeBranch(InstructionIterator Begin, + InstructionIterator End, + const MCSymbol *&TBB, + const MCSymbol *&FBB, + MCInst *&CondBranch, + MCInst *&UncondBranch) const override { + auto I = End; + + // Bottom-up analysis + while (I != Begin) { + --I; + + // Ignore nops and CFIs + if (Info->get(I->getOpcode()).isPseudo()) + continue; + + // Stop when we find the first non-terminator + if (!isTerminator(*I)) + break; + + if (!isBranch(*I)) + break; + + // Handle unconditional branches. + if (I->getOpcode() == X86::JMP_1 || + I->getOpcode() == X86::JMP_2 || + I->getOpcode() == X86::JMP_4) { + // If any code was seen after this unconditional branch, we've seen + // unreachable code. Ignore them. + CondBranch = nullptr; + UncondBranch = &*I; + const auto *Sym = getTargetSymbol(*I); + assert(Sym != nullptr && + "Couldn't extract BB symbol from jump operand"); + TBB = Sym; + continue; + } + + // Handle conditional branches and ignore indirect branches + if (I->getOpcode() != X86::LOOP && + I->getOpcode() != X86::LOOPE && + I->getOpcode() != X86::LOOPNE && + I->getOpcode() != X86::JECXZ && + I->getOpcode() != X86::JRCXZ && + getInvertedBranchOpcode(I->getOpcode()) == I->getOpcode()) { + // Indirect branch + return false; + } + + if (CondBranch == nullptr) { + const auto *TargetBB = getTargetSymbol(*I); + if (TargetBB == nullptr) { + // Unrecognized branch target + return false; + } + FBB = TBB; + TBB = TargetBB; + CondBranch = &*I; + continue; + } + + llvm_unreachable("multiple conditional branches in one BB"); + } + return true; + } + + template + std::pair + analyzePICJumpTable(Itr II, + Itr IE, + MCPhysReg R1, + MCPhysReg R2) const { + // Analyze PIC-style jump table code template: + // + // lea PIC_JUMP_TABLE(%rip), {%r1|%r2} <- MemLocInstr + // mov ({%r1|%r2}, %index, 4), {%r2|%r1} + // add %r2, %r1 + // jmp *%r1 + // + // (with any irrelevant instructions in-between) + // + // When we call this helper we've already determined %r1 and %r2, and + // reverse instruction iterator \p II is pointing to the ADD instruction. + // + // PIC jump table looks like following: + // + // JT: ---------- + // E1:| L1 - JT | + // |----------| + // E2:| L2 - JT | + // |----------| + // | | + // ...... + // En:| Ln - JT | + // ---------- + // + // Where L1, L2, ..., Ln represent labels in the function. + // + // The actual relocations in the table will be of the form: + // + // Ln - JT + // = (Ln - En) + (En - JT) + // = R_X86_64_PC32(Ln) + En - JT + // = R_X86_64_PC32(Ln + offsetof(En)) + // + DEBUG(dbgs() << "Checking for PIC jump table\n"); + MCInst *MemLocInstr = nullptr; + const MCInst *MovInstr = nullptr; + while (++II != IE) { + auto &Instr = *II; + const auto &InstrDesc = Info->get(Instr.getOpcode()); + if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo) && + !InstrDesc.hasDefOfPhysReg(Instr, R2, *RegInfo)) { + // Ignore instructions that don't affect R1, R2 registers. + continue; + } else if (!MovInstr) { + // Expect to see MOV instruction. + if (!isMOVSX64rm32(Instr)) { + DEBUG(dbgs() << "MOV instruction expected.\n"); + break; + } + + // Check if it's setting %r1 or %r2. In canonical form it sets %r2. + // If it sets %r1 - rename the registers so we have to only check + // a single form. + auto MovDestReg = Instr.getOperand(0).getReg(); + if (MovDestReg != R2) + std::swap(R1, R2); + if (MovDestReg != R2) { + DEBUG(dbgs() << "MOV instruction expected to set %r2\n"); + break; + } + + // Verify operands for MOV. + unsigned BaseRegNum; + int64_t ScaleValue; + unsigned IndexRegNum; + int64_t DispValue; + unsigned SegRegNum; + if (!evaluateX86MemoryOperand(Instr, &BaseRegNum, + &ScaleValue, &IndexRegNum, + &DispValue, &SegRegNum)) + break; + if (BaseRegNum != R1 || + ScaleValue != 4 || + IndexRegNum == X86::NoRegister || + DispValue != 0 || + SegRegNum != X86::NoRegister) + break; + MovInstr = &Instr; + } else { + assert(MovInstr && "MOV instruction expected to be set"); + if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo)) + continue; + if (!isLEA64r(Instr)) { + DEBUG(dbgs() << "LEA instruction expected\n"); + break; + } + if (Instr.getOperand(0).getReg() != R1) { + DEBUG(dbgs() << "LEA instruction expected to set %r1\n"); + break; + } + + // Verify operands for LEA. + unsigned BaseRegNum; + int64_t ScaleValue; + unsigned IndexRegNum; + const MCExpr *DispExpr = nullptr; + unsigned SegRegNum; + if (!evaluateX86MemoryOperand(Instr, &BaseRegNum, + &ScaleValue, &IndexRegNum, + nullptr, &SegRegNum, &DispExpr)) + break; + if (BaseRegNum != RegInfo->getProgramCounter() || + IndexRegNum != X86::NoRegister || + SegRegNum != X86::NoRegister || + DispExpr == nullptr) + break; + MemLocInstr = &Instr; + break; + } + } + + if (!MemLocInstr) + return std::make_pair(IndirectBranchType::UNKNOWN, nullptr); + + DEBUG(dbgs() << "checking potential PIC jump table\n"); + return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE, + MemLocInstr); + } + + IndirectBranchType analyzeIndirectBranch( + MCInst &Instruction, + InstructionIterator Begin, + InstructionIterator End, + const unsigned PtrSize, + MCInst *&MemLocInstrOut, + unsigned &BaseRegNumOut, + unsigned &IndexRegNumOut, + int64_t &DispValueOut, + const MCExpr *&DispExprOut, + MCInst *&PCRelBaseOut + ) const override { + // Try to find a (base) memory location from where the address for + // the indirect branch is loaded. For X86-64 the memory will be specified + // in the following format: + // + // {%rip}/{%basereg} + Imm + IndexReg * Scale + // + // We are interested in the cases where Scale == sizeof(uintptr_t) and + // the contents of the memory are presumably a function array. + // + // Normal jump table: + // + // jmp *(JUMP_TABLE, %index, Scale) + // + // or + // + // mov (JUMP_TABLE, %index, Scale), %r1 + // ... + // jmp %r1 + // + // We handle PIC-style jump tables separately. + // + MemLocInstrOut = nullptr; + BaseRegNumOut = X86::NoRegister; + IndexRegNumOut = X86::NoRegister; + DispValueOut = 0; + DispExprOut = nullptr; + + std::reverse_iterator II(End); + std::reverse_iterator IE(Begin); + + IndirectBranchType Type = IndirectBranchType::UNKNOWN; + + // An instruction referencing memory used by jump instruction (directly or + // via register). This location could be an array of function pointers + // in case of indirect tail call, or a jump table. + MCInst *MemLocInstr = nullptr; + + if (Instruction.getNumPrimeOperands() == 1) { + // If the indirect jump is on register - try to detect if the + // register value is loaded from a memory location. + assert(Instruction.getOperand(0).isReg() && "register operand expected"); + const auto R1 = Instruction.getOperand(0).getReg(); + // Check if one of the previous instructions defines the jump-on register. + // We will check that this instruction belongs to the same basic block + // in postProcessIndirectBranches(). + for (auto PrevII = II; PrevII != IE; ++PrevII) { + auto &PrevInstr = *PrevII; + const auto &PrevInstrDesc = Info->get(PrevInstr.getOpcode()); + + if (!PrevInstrDesc.hasDefOfPhysReg(PrevInstr, R1, *RegInfo)) + continue; + + if (isMoveMem2Reg(PrevInstr)) { + MemLocInstr = &PrevInstr; + break; + } else if (isADD64rr(PrevInstr)) { + auto R2 = PrevInstr.getOperand(2).getReg(); + if (R1 == R2) + return IndirectBranchType::UNKNOWN; + std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2); + break; + } else { + return IndirectBranchType::UNKNOWN; + } + } + if (!MemLocInstr) { + // No definition seen for the register in this function so far. Could be + // an input parameter - which means it is an external code reference. + // It also could be that the definition happens to be in the code that + // we haven't processed yet. Since we have to be conservative, return + // as UNKNOWN case. + return IndirectBranchType::UNKNOWN; + } + } else { + MemLocInstr = &Instruction; + } + + const auto RIPRegister = RegInfo->getProgramCounter(); + + // Analyze the memory location. + unsigned BaseRegNum, IndexRegNum, SegRegNum; + int64_t ScaleValue, DispValue; + const MCExpr *DispExpr; + + if (!evaluateX86MemoryOperand(*MemLocInstr, &BaseRegNum, + &ScaleValue, &IndexRegNum, + &DispValue, &SegRegNum, + &DispExpr)) + return IndirectBranchType::UNKNOWN; + + if ((BaseRegNum != X86::NoRegister && BaseRegNum != RIPRegister) || + SegRegNum != X86::NoRegister) + return IndirectBranchType::UNKNOWN; + + if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && + (ScaleValue != 1 || BaseRegNum != RIPRegister)) + return IndirectBranchType::UNKNOWN; + + if (Type != IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && + ScaleValue != PtrSize) + return IndirectBranchType::UNKNOWN; + + MemLocInstrOut = MemLocInstr; + BaseRegNumOut = BaseRegNum; + IndexRegNumOut = IndexRegNum; + DispValueOut = DispValue; + DispExprOut = DispExpr; + + return Type; + } + + /// Analyze a callsite to see if it could be a virtual method call. This only + /// checks to see if the overall pattern is satisfied, it does not guarantee + /// that the callsite is a true virtual method call. + /// The format of virtual method calls that are recognized is one of the + /// following: + /// + /// Form 1: (found in debug code) + /// add METHOD_OFFSET, %VtableReg + /// mov (%VtableReg), %MethodReg + /// ... + /// call or jmp *%MethodReg + /// + /// Form 2: + /// mov METHOD_OFFSET(%VtableReg), %MethodReg + /// ... + /// call or jmp *%MethodReg + /// + /// Form 3: + /// ... + /// call or jmp *METHOD_OFFSET(%VtableReg) + /// + bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin, + InstructionIterator ForwardEnd, + std::vector &MethodFetchInsns, + unsigned &VtableRegNum, + unsigned &MethodRegNum, + uint64_t &MethodOffset) const override { + VtableRegNum = X86::NoRegister; + MethodRegNum = X86::NoRegister; + MethodOffset = 0; + + std::reverse_iterator Itr(ForwardEnd); + std::reverse_iterator End(ForwardBegin); + + auto &CallInst = *Itr++; + assert(isIndirectBranch(CallInst) || isCall(CallInst)); + + unsigned BaseReg, IndexReg, SegmentReg; + int64_t Scale, Disp; + const MCExpr *DispExpr; + + // The call can just be jmp offset(reg) + if (evaluateX86MemoryOperand(CallInst, + &BaseReg, + &Scale, + &IndexReg, + &Disp, + &SegmentReg, + &DispExpr)) { + if (!DispExpr && + BaseReg != X86::RIP && + BaseReg != X86::RBP && + BaseReg != X86::NoRegister) { + MethodRegNum = BaseReg; + if (Scale == 1 && + IndexReg == X86::NoRegister && + SegmentReg == X86::NoRegister) { + VtableRegNum = MethodRegNum; + MethodOffset = Disp; + MethodFetchInsns.push_back(&CallInst); + return true; + } + } + return false; + } else if (CallInst.getOperand(0).isReg()) { + MethodRegNum = CallInst.getOperand(0).getReg(); + } else { + return false; + } + + if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) { + VtableRegNum = X86::NoRegister; + MethodRegNum = X86::NoRegister; + return false; + } + + // find load from vtable, this may or may not include the method offset + while (Itr != End) { + auto &CurInst = *Itr++; + const auto &Desc = Info->get(CurInst.getOpcode()); + if (Desc.hasDefOfPhysReg(CurInst, MethodRegNum, *RegInfo)) { + if (isLoad(CurInst) && + evaluateX86MemoryOperand(CurInst, + &BaseReg, + &Scale, + &IndexReg, + &Disp, + &SegmentReg, + &DispExpr)) { + if (!DispExpr && + Scale == 1 && + BaseReg != X86::RIP && + BaseReg != X86::RBP && + BaseReg != X86::NoRegister && + IndexReg == X86::NoRegister && + SegmentReg == X86::NoRegister && + BaseReg != X86::RIP) { + VtableRegNum = BaseReg; + MethodOffset = Disp; + MethodFetchInsns.push_back(&CurInst); + if (MethodOffset != 0) + return true; + break; + } + } + return false; + } + } + + if (!VtableRegNum) + return false; + + // look for any adds affecting the method register. + while (Itr != End) { + auto &CurInst = *Itr++; + const auto &Desc = Info->get(CurInst.getOpcode()); + if (Desc.hasDefOfPhysReg(CurInst, VtableRegNum, *RegInfo)) { + if (isADDri(CurInst)) { + assert(!MethodOffset); + MethodOffset = CurInst.getOperand(2).getImm(); + MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst); + break; + } + } + } + + return true; + } + + bool createStackPointerIncrement(MCInst &Inst, int Size, + bool NoFlagsClobber) const override { + if (NoFlagsClobber) { + Inst.setOpcode(X86::LEA64r); + Inst.clear(); + Inst.addOperand(MCOperand::createReg(X86::RSP)); + Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg + Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt + Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg + Inst.addOperand(MCOperand::createImm(-Size)); // Displacement + Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg + return true; + } + Inst.setOpcode(X86::SUB64ri8); + Inst.clear(); + Inst.addOperand(MCOperand::createReg(X86::RSP)); + Inst.addOperand(MCOperand::createReg(X86::RSP)); + Inst.addOperand(MCOperand::createImm(Size)); + return true; + } + + bool createStackPointerDecrement(MCInst &Inst, int Size, + bool NoFlagsClobber) const override { + if (NoFlagsClobber) { + Inst.setOpcode(X86::LEA64r); + Inst.clear(); + Inst.addOperand(MCOperand::createReg(X86::RSP)); + Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg + Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt + Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg + Inst.addOperand(MCOperand::createImm(Size)); // Displacement + Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg + return true; + } + Inst.setOpcode(X86::ADD64ri8); + Inst.clear(); + Inst.addOperand(MCOperand::createReg(X86::RSP)); + Inst.addOperand(MCOperand::createReg(X86::RSP)); + Inst.addOperand(MCOperand::createImm(Size)); + return true; + } + + bool createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset, + const MCPhysReg &SrcReg, int Size) const override { + unsigned NewOpcode; + switch (Size) { + default: + return false; + case 2: NewOpcode = X86::MOV16mr; break; + case 4: NewOpcode = X86::MOV32mr; break; + case 8: NewOpcode = X86::MOV64mr; break; + } + Inst.setOpcode(NewOpcode); + Inst.clear(); + Inst.addOperand(MCOperand::createReg(StackReg)); // BaseReg + Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt + Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg + Inst.addOperand(MCOperand::createImm(Offset)); // Displacement + Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg + Inst.addOperand(MCOperand::createReg(SrcReg)); + return true; + } + + bool createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg, + int Offset, const MCPhysReg &DstReg, + int Size) const override { + return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister, + Offset, DstReg, Size); + } + + bool createLoad(MCInst &Inst, const MCPhysReg &BaseReg, + int Scale, const MCPhysReg &IndexReg, + int Offset, const MCPhysReg &DstReg, + int Size) const override { + unsigned NewOpcode; + switch (Size) { + default: + return false; + case 2: NewOpcode = X86::MOV16rm; break; + case 4: NewOpcode = X86::MOV32rm; break; + case 8: NewOpcode = X86::MOV64rm; break; + } + Inst.setOpcode(NewOpcode); + Inst.clear(); + Inst.addOperand(MCOperand::createReg(DstReg)); + Inst.addOperand(MCOperand::createReg(BaseReg)); + Inst.addOperand(MCOperand::createImm(Scale)); + Inst.addOperand(MCOperand::createReg(IndexReg)); + Inst.addOperand(MCOperand::createImm(Offset)); // Displacement + Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg + return true; + } + + bool createIJmp32Frag(SmallVectorImpl &Insts, + const MCOperand &BaseReg, const MCOperand &Scale, + const MCOperand &IndexReg, const MCOperand &Offset, + const MCOperand &TmpReg) const override { + // The code fragment we emit here is: + // + // mov32 (%base, %index, scale), %tmpreg + // ijmp *(%tmpreg) + // + MCInst IJmp; + IJmp.setOpcode(X86::JMP64r); + IJmp.addOperand(TmpReg); + + MCInst Load; + Load.setOpcode(X86::MOV32rm); + Load.addOperand(TmpReg); + Load.addOperand(BaseReg); + Load.addOperand(Scale); + Load.addOperand(IndexReg); + Load.addOperand(Offset); + Load.addOperand(MCOperand::createReg(X86::NoRegister)); + + Insts.push_back(Load); + Insts.push_back(IJmp); + return true; + } + + bool createNoop(MCInst &Inst) const override { + Inst.setOpcode(X86::NOOP); + return true; + } + + bool createReturn(MCInst &Inst) const override { + Inst.setOpcode(X86::RETQ); + return true; + } + + bool replaceImmWithSymbol(MCInst &Inst, MCSymbol *Symbol, int64_t Addend, + MCContext *Ctx, int64_t &Value, + uint64_t RelType) const override { + unsigned ImmOpNo = -1U; + + for (unsigned Index = 0; Index < Inst.getNumPrimeOperands(); ++Index) { + if (Inst.getOperand(Index).isImm()) { + ImmOpNo = Index; + // TODO: this is a bit hacky. It finds the correct operand by + // searching for a specific immediate value. If no value is + // provided it defaults to the last immediate operand found. + // This could lead to unexpected results if the instruction + // has more than one immediate with the same value. + if (Inst.getOperand(ImmOpNo).getImm() == Value) + break; + } + } + + if (ImmOpNo == -1U) + return false; + + Value = Inst.getOperand(ImmOpNo).getImm(); + + MCOperand Operand; + if (!Addend) { + Operand = MCOperand::createExpr(getTargetExprFor( + Inst, MCSymbolRefExpr::create(Symbol, *Ctx), *Ctx, RelType)); + } else { + Operand = MCOperand::createExpr(getTargetExprFor( + Inst, + MCBinaryExpr::createAdd(MCSymbolRefExpr::create(Symbol, *Ctx), + MCConstantExpr::create(Addend, *Ctx), *Ctx), + *Ctx, RelType)); + } + Inst.getOperand(ImmOpNo) = Operand; + return true; + } + + bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx) const override { + Inst.setOpcode(X86::JMP_1); + Inst.addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx))); + return true; + } + + bool createTailCall(MCInst &Inst, const MCSymbol *Target, + MCContext *Ctx) const override { + Inst.setOpcode(X86::TAILJMPd); + Inst.addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); + return true; + } + + bool createTrap(MCInst &Inst) const override { + Inst.clear(); + Inst.setOpcode(X86::TRAP); + return true; + } + + bool createCFI(MCInst &Inst, int64_t Offset) const override { + Inst.clear(); + Inst.setOpcode(X86::CFI_INSTRUCTION); + Inst.addOperand(MCOperand::createImm(Offset)); + return true; + } + + bool isCFI(const MCInst &Inst) const override { + if (Inst.getOpcode() == X86::CFI_INSTRUCTION) + return true; + return false; + } + + bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx) const override { + Inst.setOpcode(getInvertedBranchOpcode(Inst.getOpcode())); + assert(Inst.getOpcode() != 0 && "invalid branch instruction"); + Inst.getOperand(0) = MCOperand::createExpr( + MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); + return true; + } + + unsigned getCanonicalBranchOpcode(unsigned Opcode) const override { + switch (Opcode) { + default: + return Opcode; + + case X86::JE_1: return X86::JE_1; + case X86::JE_2: return X86::JE_2; + case X86::JE_4: return X86::JE_4; + case X86::JNE_1: return X86::JE_1; + case X86::JNE_2: return X86::JE_2; + case X86::JNE_4: return X86::JE_4; + + case X86::JL_1: return X86::JL_1; + case X86::JL_2: return X86::JL_2; + case X86::JL_4: return X86::JL_4; + case X86::JGE_1: return X86::JL_1; + case X86::JGE_2: return X86::JL_2; + case X86::JGE_4: return X86::JL_4; + + case X86::JLE_1: return X86::JG_1; + case X86::JLE_2: return X86::JG_2; + case X86::JLE_4: return X86::JG_4; + case X86::JG_1: return X86::JG_1; + case X86::JG_2: return X86::JG_2; + case X86::JG_4: return X86::JG_4; + + case X86::JB_1: return X86::JB_1; + case X86::JB_2: return X86::JB_2; + case X86::JB_4: return X86::JB_4; + case X86::JAE_1: return X86::JB_1; + case X86::JAE_2: return X86::JB_2; + case X86::JAE_4: return X86::JB_4; + + case X86::JBE_1: return X86::JA_1; + case X86::JBE_2: return X86::JA_2; + case X86::JBE_4: return X86::JA_4; + case X86::JA_1: return X86::JA_1; + case X86::JA_2: return X86::JA_2; + case X86::JA_4: return X86::JA_4; + + case X86::JS_1: return X86::JS_1; + case X86::JS_2: return X86::JS_2; + case X86::JS_4: return X86::JS_4; + case X86::JNS_1: return X86::JS_1; + case X86::JNS_2: return X86::JS_2; + case X86::JNS_4: return X86::JS_4; + + case X86::JP_1: return X86::JP_1; + case X86::JP_2: return X86::JP_2; + case X86::JP_4: return X86::JP_4; + case X86::JNP_1: return X86::JP_1; + case X86::JNP_2: return X86::JP_2; + case X86::JNP_4: return X86::JP_4; + + case X86::JO_1: return X86::JO_1; + case X86::JO_2: return X86::JO_2; + case X86::JO_4: return X86::JO_4; + case X86::JNO_1: return X86::JO_1; + case X86::JNO_2: return X86::JO_2; + case X86::JNO_4: return X86::JO_4; + } + } + + bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx) const override { + assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) && + "Invalid instruction"); + Inst.getOperand(0) = MCOperand::createExpr( + MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); + return true; + } + + bool createEHLabel(MCInst &Inst, const MCSymbol *Label, + MCContext *Ctx) const override { + Inst.setOpcode(X86::EH_LABEL); + Inst.clear(); + Inst.addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(Label, MCSymbolRefExpr::VK_None, *Ctx))); + return true; + } + + ICPdata indirectCallPromotion( + const MCInst &CallInst, + const std::vector>& Targets, + const std::vector &VtableAddrs, + const std::vector &MethodFetchInsns, + const bool MinimizeCodeSize, + MCContext *Ctx + ) const override { + const bool IsTailCall = isTailCall(CallInst); + const bool IsJumpTable = getJumpTable(CallInst) != 0; + ICPdata Results; + + // Label for the current code block. + MCSymbol* NextTarget = nullptr; + + // The join block which contains all the instructions following CallInst. + // MergeBlock remains null if CallInst is a tail call. + MCSymbol* MergeBlock = nullptr; + + unsigned FuncAddrReg = X86::R10; + + const bool LoadElim = !VtableAddrs.empty(); + assert((!LoadElim || VtableAddrs.size() == Targets.size()) && + "There must be a vtable entry for every method " + "in the targets vector."); + + if (MinimizeCodeSize && !LoadElim) { + std::set UsedRegs; + + for (unsigned int i = 0; i < CallInst.getNumPrimeOperands(); ++i) { + const auto &Op = CallInst.getOperand(i); + if (Op.isReg()) { + UsedRegs.insert(Op.getReg()); + } + } + + if (UsedRegs.count(X86::R10) == 0) + FuncAddrReg = X86::R10; + else if (UsedRegs.count(X86::R11) == 0) + FuncAddrReg = X86::R11; + else + return Results; + } + + const auto jumpToMergeBlock = [&](std::vector *NewCall) { + assert(MergeBlock); + NewCall->push_back(CallInst); + MCInst& Merge = NewCall->back(); + Merge.clear(); + createUncondBranch(Merge, MergeBlock, Ctx); + }; + + for (unsigned int i = 0; i < Targets.size(); ++i) { + Results.push_back(std::make_pair(NextTarget, std::vector())); + std::vector* NewCall = &Results.back().second; + + if (MinimizeCodeSize && !LoadElim) { + // Load the call target into FuncAddrReg. + NewCall->push_back(CallInst); // Copy CallInst in order to get SMLoc + MCInst& Target = NewCall->back(); + Target.clear(); + Target.setOpcode(X86::MOV64ri32); + Target.addOperand(MCOperand::createReg(FuncAddrReg)); + if (Targets[i].first) { + // Is this OK? + Target.addOperand( + MCOperand::createExpr( + MCSymbolRefExpr::create(Targets[i].first, + MCSymbolRefExpr::VK_None, + *Ctx))); + } else { + const auto Addr = Targets[i].second; + // Immediate address is out of sign extended 32 bit range. + if (int64_t(Addr) != int64_t(int32_t(Addr))) { + return ICPdata(); + } + Target.addOperand(MCOperand::createImm(Addr)); + } + + // Compare current call target to a specific address. + NewCall->push_back(CallInst); + MCInst& Compare = NewCall->back(); + Compare.clear(); + if (CallInst.getOpcode() == X86::CALL64r || + CallInst.getOpcode() == X86::JMP64r || + CallInst.getOpcode() == X86::TAILJMPr) { + Compare.setOpcode(X86::CMP64rr); + } else if (CallInst.getOpcode() == X86::CALL64pcrel32) { + Compare.setOpcode(X86::CMP64ri32); + } else { + Compare.setOpcode(X86::CMP64rm); + } + Compare.addOperand(MCOperand::createReg(FuncAddrReg)); + + // TODO: Would be preferable to only load this value once. + for (unsigned i = 0; + i < Info->get(CallInst.getOpcode()).getNumOperands(); + ++i) { + if (!CallInst.getOperand(i).isJumpTable()) + Compare.addOperand(CallInst.getOperand(i)); + } + } else { + // Compare current call target to a specific address. + NewCall->push_back(CallInst); + MCInst& Compare = NewCall->back(); + Compare.clear(); + if (CallInst.getOpcode() == X86::CALL64r || + CallInst.getOpcode() == X86::JMP64r || + CallInst.getOpcode() == X86::TAILJMPr) { + Compare.setOpcode(X86::CMP64ri32); + } else { + Compare.setOpcode(X86::CMP64mi32); + } + + // Original call address. + for (unsigned i = 0; + i < Info->get(CallInst.getOpcode()).getNumOperands(); + ++i) { + if (!CallInst.getOperand(i).isJumpTable()) + Compare.addOperand(CallInst.getOperand(i)); + } + + // Target address. + if (Targets[i].first && !LoadElim) { + Compare.addOperand( + MCOperand::createExpr( + MCSymbolRefExpr::create(Targets[i].first, + MCSymbolRefExpr::VK_None, + *Ctx))); + } else { + const auto Addr = LoadElim ? VtableAddrs[i] : Targets[i].second; + // Immediate address is out of sign extended 32 bit range. + if (int64_t(Addr) != int64_t(int32_t(Addr))) { + return ICPdata(); + } + Compare.addOperand(MCOperand::createImm(Addr)); + } + } + + // jump to next target compare. + NextTarget = Ctx->createTempSymbol(); // generate label for the next block + NewCall->push_back(CallInst); + + if (IsJumpTable) { + MCInst& Je = NewCall->back(); + + // Jump to next compare if target addresses don't match. + Je.clear(); + Je.setOpcode(X86::JE_1); + if (Targets[i].first) { + Je.addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(Targets[i].first, + MCSymbolRefExpr::VK_None, + *Ctx))); + } else { + Je.addOperand(MCOperand::createImm(Targets[i].second)); + } + assert(!isInvoke(CallInst)); + } else { + MCInst& Jne = NewCall->back(); + + // Jump to next compare if target addresses don't match. + Jne.clear(); + Jne.setOpcode(X86::JNE_1); + Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( + NextTarget, MCSymbolRefExpr::VK_None, *Ctx))); + + // Call specific target directly. + Results.push_back( + std::make_pair(Ctx->createTempSymbol(), std::vector())); + NewCall = &Results.back().second; + NewCall->push_back(CallInst); + MCInst &CallOrJmp = NewCall->back(); + + CallOrJmp.clear(); + + if (MinimizeCodeSize && !LoadElim) { + CallOrJmp.setOpcode(IsTailCall ? X86::TAILJMPr : X86::CALL64r); + CallOrJmp.addOperand(MCOperand::createReg(FuncAddrReg)); + } else { + CallOrJmp.setOpcode(IsTailCall ? X86::TAILJMPd : X86::CALL64pcrel32); + + if (Targets[i].first) { + CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( + Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); + } else { + CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second)); + } + } + + if (isInvoke(CallInst) && !isInvoke(CallOrJmp)) { + // Copy over any EH or GNU args size information from the original call. + addEHInfo(CallOrJmp, getEHInfo(CallInst), Ctx); + auto GnuArgsSize = getGnuArgsSize(CallInst); + if (GnuArgsSize >= 0) + addGnuArgsSize(CallOrJmp, GnuArgsSize); + } + + if (!IsTailCall) { + // The fallthrough block for the most common target should be + // the merge block. + if (i == 0) { + // Fallthrough to merge block. + MergeBlock = Ctx->createTempSymbol(); + } else { + // Insert jump to the merge block if we are not doing a fallthrough. + jumpToMergeBlock(NewCall); + } + } + } + } + + // Cold call block. + Results.push_back(std::make_pair(NextTarget, std::vector())); + std::vector& NewCall = Results.back().second; + for (auto *Inst : MethodFetchInsns) { + if (Inst != &CallInst) + NewCall.push_back(*Inst); + } + NewCall.push_back(CallInst); + + // Jump to merge block from cold call block + if (!IsTailCall && !IsJumpTable) { + jumpToMergeBlock(&NewCall); + + // Record merge block + Results.push_back(std::make_pair(MergeBlock, std::vector())); + } + + return Results; + } + + ICPdata jumpTablePromotion( + const MCInst &IJmpInst, + const std::vector>& Targets, + const std::vector &TargetFetchInsns, + MCContext *Ctx + ) const override { + assert(getJumpTable(IJmpInst) != 0); + uint16_t IndexReg = getAnnotationAs(IJmpInst, "JTIndexReg"); + if (IndexReg == 0) + return ICPdata(); + + ICPdata Results; + + // Label for the current code block. + MCSymbol* NextTarget = nullptr; + + for (unsigned int i = 0; i < Targets.size(); ++i) { + Results.push_back(std::make_pair(NextTarget, std::vector())); + std::vector* CurBB = &Results.back().second; + + // Compare current index to a specific index. + CurBB->emplace_back(MCInst()); + MCInst &CompareInst = CurBB->back(); + CompareInst.setLoc(IJmpInst.getLoc()); + CompareInst.setOpcode(X86::CMP64ri32); + CompareInst.addOperand(MCOperand::createReg(IndexReg)); + + const auto CaseIdx = Targets[i].second; + // Immediate address is out of sign extended 32 bit range. + if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx))) { + return ICPdata(); + } + CompareInst.addOperand(MCOperand::createImm(CaseIdx)); + shortenInstruction(CompareInst); + + // jump to next target compare. + NextTarget = Ctx->createTempSymbol(); // generate label for the next block + CurBB->push_back(MCInst()); + + MCInst &JEInst = CurBB->back(); + JEInst.setLoc(IJmpInst.getLoc()); + + // Jump to target if indices match + JEInst.setOpcode(X86::JE_1); + JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( + Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); + } + + // Cold call block. + Results.push_back(std::make_pair(NextTarget, std::vector())); + std::vector& CurBB = Results.back().second; + for (auto *Inst : TargetFetchInsns) { + if (Inst != &IJmpInst) + CurBB.push_back(*Inst); + } + CurBB.push_back(IJmpInst); + + return Results; + } + +}; + +} + +MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis, + const MCInstrInfo *Info, + const MCRegisterInfo *RegInfo) { + return new X86MCPlusBuilder(Analysis, Info, RegInfo); +} +