From c040431fe6d5e49f5c090f0e14c74888380d3c9e Mon Sep 17 00:00:00 2001 From: Vladislav Khmelevsky Date: Fri, 20 Aug 2021 03:07:01 +0300 Subject: [PATCH] [PR] AArch64: Fix ADR instruction handling Summary: There are 2 problems found when handling ADR instruction: 1. When extracting value from the ADR instruction we need to do it another way, then we do it for ADRP instruction. 2. When creating target expression the VariantKind should be other for ADR instruction. And we introduces R_AARCH64_ADR_PREL_LO21, R_AARCH64_TLSDESC_ADR_PREL21 and R_AARCH64_ADR_PREL_PG_HI21_NC relocations support. Also this patch introduces AdrPass, which will replace non-local pointing ADR instructions with ADRP + ADD instructions sequence due to small offset range of ADR instruction, so after BOLT magic there are no guarantees that ADR instruction will still be in the range of just +- 1MB from its target. The instruction replacement needs relocations to be avalailable, so we won't remove "IsFromCode" relocations after disassembly from BF anymore. Also we need original offset of ADR instruction to be available so we add offset annotation for these instructions. The last thing this patch adds is ARM testing directory, which will be used only on ARM testing servers. The common tests (non-assembler tests which are platform-independent) might be moved from the X86 directory to the parent one in the future, so such tests could be tested on both X86 and ARM machines. Vladislav Khmelevsky, Advanced Software Technology Lab, Huawei (cherry picked from FBD30497379) --- bolt/src/BinaryFunction.cpp | 8 +- bolt/src/BinaryFunction.h | 3 + bolt/src/BinaryPassManager.cpp | 16 ++- bolt/src/MCPlusBuilder.h | 28 +++++ bolt/src/Passes/ADRRelaxationPass.cpp | 72 +++++++++++ bolt/src/Passes/ADRRelaxationPass.h | 41 +++++++ bolt/src/Passes/CMakeLists.txt | 1 + bolt/src/Relocation.cpp | 23 +++- bolt/src/RewriteInstance.cpp | 3 +- .../Target/AArch64/AArch64MCPlusBuilder.cpp | 116 ++++++++++++++---- bolt/test/AArch64/adrrelaxationpass.s | 49 ++++++++ bolt/test/AArch64/lit.local.cfg | 2 + bolt/test/X86/lit.local.cfg | 2 + 13 files changed, 332 insertions(+), 32 deletions(-) create mode 100644 bolt/src/Passes/ADRRelaxationPass.cpp create mode 100644 bolt/src/Passes/ADRRelaxationPass.h create mode 100644 bolt/test/AArch64/adrrelaxationpass.s create mode 100644 bolt/test/AArch64/lit.local.cfg create mode 100644 bolt/test/X86/lit.local.cfg diff --git a/bolt/src/BinaryFunction.cpp b/bolt/src/BinaryFunction.cpp index c3c1f01118da..ff307bbb9840 100644 --- a/bolt/src/BinaryFunction.cpp +++ b/bolt/src/BinaryFunction.cpp @@ -4018,7 +4018,9 @@ bool BinaryFunction::isDataMarker(const SymbolRef &Symbol, // code section (see IHI0056B). $d identifies a symbol starting data contents. if (BC.isAArch64() && Symbol.getType() && cantFail(Symbol.getType()) == SymbolRef::ST_Unknown && SymbolSize == 0 && - Symbol.getName() && cantFail(Symbol.getName()) == "$d") + Symbol.getName() && + (cantFail(Symbol.getName()) == "$d" || + cantFail(Symbol.getName()).startswith("$d."))) return true; return false; } @@ -4030,7 +4032,9 @@ bool BinaryFunction::isCodeMarker(const SymbolRef &Symbol, // end of a data chunk inside code. if (BC.isAArch64() && Symbol.getType() && cantFail(Symbol.getType()) == SymbolRef::ST_Unknown && SymbolSize == 0 && - Symbol.getName() && cantFail(Symbol.getName()) == "$x") + Symbol.getName() && + (cantFail(Symbol.getName()) == "$x" || + cantFail(Symbol.getName()).startswith("$x."))) return true; return false; } diff --git a/bolt/src/BinaryFunction.h b/bolt/src/BinaryFunction.h index 093861265e5a..6e697b45b493 100644 --- a/bolt/src/BinaryFunction.h +++ b/bolt/src/BinaryFunction.h @@ -1315,8 +1315,11 @@ public: case ELF::R_AARCH64_LDST8_ABS_LO12_NC: case ELF::R_AARCH64_LDST128_ABS_LO12_NC: case ELF::R_AARCH64_ADR_GOT_PAGE: + case ELF::R_AARCH64_TLSDESC_ADR_PREL21: case ELF::R_AARCH64_TLSDESC_ADR_PAGE21: + case ELF::R_AARCH64_ADR_PREL_LO21: case ELF::R_AARCH64_ADR_PREL_PG_HI21: + case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC: Relocations[Offset] = Relocation{Offset, Symbol, RelType, Addend, Value}; break; case ELF::R_X86_64_PC32: diff --git a/bolt/src/BinaryPassManager.cpp b/bolt/src/BinaryPassManager.cpp index 8b32062a76d5..9aacff6d793e 100644 --- a/bolt/src/BinaryPassManager.cpp +++ b/bolt/src/BinaryPassManager.cpp @@ -9,6 +9,7 @@ //===----------------------------------------------------------------------===// #include "BinaryPassManager.h" +#include "Passes/ADRRelaxationPass.h" #include "Passes/Aligner.h" #include "Passes/AllocCombiner.h" #include "Passes/FrameOptimizer.h" @@ -415,8 +416,8 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) { opts::ICF); if (BC.isAArch64()) - Manager.registerPass( - std::make_unique(PrintVeneerElimination)); + Manager.registerPass( + std::make_unique(PrintVeneerElimination)); Manager.registerPass( std::make_unique(NeverPrint, opts::SpecializeMemcpy1), @@ -521,11 +522,14 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) { if (BC.HasRelocations) Manager.registerPass(std::make_unique()); - // Tighten branches according to offset differences between branch and - // targets. No extra instructions after this pass, otherwise we may have - // relocations out of range and crash during linking. - if (BC.isAArch64()) + if (BC.isAArch64()) { + Manager.registerPass(std::make_unique()); + + // Tighten branches according to offset differences between branch and + // targets. No extra instructions after this pass, otherwise we may have + // relocations out of range and crash during linking. Manager.registerPass(std::make_unique(PrintLongJmp)); + } // This pass turns tail calls into jumps which makes them invisible to // function reordering. It's unsafe to use any CFG or instruction analysis diff --git a/bolt/src/MCPlusBuilder.h b/bolt/src/MCPlusBuilder.h index 703773eb52b1..0da004000627 100644 --- a/bolt/src/MCPlusBuilder.h +++ b/bolt/src/MCPlusBuilder.h @@ -559,6 +559,10 @@ public: return false; } + virtual void getADRReg(const MCInst &Inst, MCPhysReg &RegName) const { + llvm_unreachable("not implemented"); + } + virtual bool isMoveMem2Reg(const MCInst &Inst) const { llvm_unreachable("not implemented"); return false; @@ -1096,6 +1100,21 @@ public: return &cast(Expr)->getSymbol(); } + /// Return addend that represents an offset from MCSymbol target + /// of this instruction at a given operand number \p OpNum. + /// If there's no symbol associated with the operand - return 0 + virtual int64_t getTargetAddend(const MCInst &Inst, + unsigned OpNum = 0) const { + llvm_unreachable("not implemented"); + return 0; + } + + /// Return MCSymbol addend extracted from a target expression + virtual int64_t getTargetAddend(const MCExpr *Expr) const { + llvm_unreachable("not implemented"); + return 0; + } + /// Return MCSymbol/offset extracted from a target expression virtual std::pair getTargetSymbolInfo(const MCExpr *Expr) const { @@ -1382,6 +1401,15 @@ public: return false; } + /// Store \p Target absolute adddress to \p RegName + virtual std::vector materializeAddress(const MCSymbol *Target, + MCContext *Ctx, + MCPhysReg RegName, + int64_t Addend = 0) const { + llvm_unreachable("not implemented"); + return {}; + } + /// Creates a new unconditional branch instruction in Inst and set its operand /// to TBB. /// diff --git a/bolt/src/Passes/ADRRelaxationPass.cpp b/bolt/src/Passes/ADRRelaxationPass.cpp new file mode 100644 index 000000000000..5b65fe248bb2 --- /dev/null +++ b/bolt/src/Passes/ADRRelaxationPass.cpp @@ -0,0 +1,72 @@ +//===--- ADRRelaxationPass.cpp --------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +//===----------------------------------------------------------------------===// + +#include "ADRRelaxationPass.h" +#include "ParallelUtilities.h" + +using namespace llvm; + +namespace opts { +extern cl::OptionCategory BoltCategory; + +static cl::opt + AdrPassOpt("adr-relaxation", + cl::desc("Replace ARM non-local ADR instructions with ADRP"), + cl::init(true), cl::cat(BoltCategory), cl::ReallyHidden); +} // namespace opts + +namespace llvm { +namespace bolt { + +void ADRRelaxationPass::runOnFunction(BinaryContext &BC, BinaryFunction &BF) { + for (BinaryBasicBlock *BB : BF.layout()) { + for (auto It = BB->begin(); It != BB->end(); ++It) { + MCInst &Inst = *It; + if (!BC.MIB->isADR(Inst)) + continue; + + const MCSymbol *Symbol = BC.MIB->getTargetSymbol(Inst); + if (!Symbol) + continue; + + BinaryFunction::IslandInfo &Islands = BF.getIslandInfo(); + if (Islands.Symbols.count(Symbol) || Islands.ProxySymbols.count(Symbol)) + continue; + + BinaryFunction *TargetBF = BC.getFunctionForSymbol(Symbol); + if (TargetBF && TargetBF == &BF) + continue; + + MCPhysReg Reg; + BC.MIB->getADRReg(Inst, Reg); + int64_t Addend = BC.MIB->getTargetAddend(Inst); + std::vector Addr = + BC.MIB->materializeAddress(Symbol, BC.Ctx.get(), Reg, Addend); + It = BB->replaceInstruction(It, Addr); + } + } +} + +void ADRRelaxationPass::runOnFunctions(BinaryContext &BC) { + if (!opts::AdrPassOpt || !BC.HasRelocations) + return; + + ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) { + runOnFunction(BC, BF); + }; + + ParallelUtilities::runOnEachFunction( + BC, ParallelUtilities::SchedulingPolicy::SP_TRIVIAL, WorkFun, nullptr, + "ADRRelaxationPass", /* ForceSequential */ true); +} + +} // end namespace bolt +} // end namespace llvm diff --git a/bolt/src/Passes/ADRRelaxationPass.h b/bolt/src/Passes/ADRRelaxationPass.h new file mode 100644 index 000000000000..49944a4a68b3 --- /dev/null +++ b/bolt/src/Passes/ADRRelaxationPass.h @@ -0,0 +1,41 @@ +//===--------- Passes/ADRRelaxationPass.h ---------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_LLVM_BOLT_PASSES_ADRRELAXATIONPASS_H +#define LLVM_TOOLS_LLVM_BOLT_PASSES_ADRRELAXATIONPASS_H + +#include "BinaryPasses.h" + +// This pass replaces AArch64 non-local ADR instructions +// with ADRP + ADD due to small offset range of ADR instruction +// (+- 1MB) which could be easely overflowed after BOLT optimizations +// Such problems are usually connected with errata 843419 +// https://developer.arm.com/documentation/epm048406/2100/ +// The linker could replace ADRP instruction with ADR in some cases. + +namespace llvm { +namespace bolt { + +class ADRRelaxationPass : public BinaryFunctionPass { +public: + explicit ADRRelaxationPass() : BinaryFunctionPass(false) {} + + const char *getName() const override { return "adr-relaxation"; } + + /// Pass entry point + void runOnFunctions(BinaryContext &BC) override; + void runOnFunction(BinaryContext &BC, BinaryFunction &BF); +}; + +} // namespace bolt +} // namespace llvm + +#endif diff --git a/bolt/src/Passes/CMakeLists.txt b/bolt/src/Passes/CMakeLists.txt index 71956e5985fb..19203acc137b 100644 --- a/bolt/src/Passes/CMakeLists.txt +++ b/bolt/src/Passes/CMakeLists.txt @@ -1,4 +1,5 @@ add_llvm_library(LLVMBOLTPasses + ADRRelaxationPass.cpp Aligner.cpp AllocCombiner.cpp BinaryPasses.cpp diff --git a/bolt/src/Relocation.cpp b/bolt/src/Relocation.cpp index e688c5953c28..b56fd089d6ad 100644 --- a/bolt/src/Relocation.cpp +++ b/bolt/src/Relocation.cpp @@ -46,7 +46,9 @@ bool isSupportedAArch64(uint64_t Type) { default: return false; case ELF::R_AARCH64_CALL26: + case ELF::R_AARCH64_ADR_PREL_LO21: case ELF::R_AARCH64_ADR_PREL_PG_HI21: + case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC: case ELF::R_AARCH64_LDST64_ABS_LO12_NC: case ELF::R_AARCH64_ADD_ABS_LO12_NC: case ELF::R_AARCH64_LDST128_ABS_LO12_NC: @@ -54,6 +56,7 @@ bool isSupportedAArch64(uint64_t Type) { case ELF::R_AARCH64_LDST16_ABS_LO12_NC: case ELF::R_AARCH64_LDST8_ABS_LO12_NC: case ELF::R_AARCH64_ADR_GOT_PAGE: + case ELF::R_AARCH64_TLSDESC_ADR_PREL21: case ELF::R_AARCH64_TLSDESC_ADR_PAGE21: case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12: @@ -101,7 +104,9 @@ size_t getSizeForTypeAArch64(uint64_t Type) { dbgs() << "Reloc num: " << Type << "\n"; llvm_unreachable("unsupported relocation type"); case ELF::R_AARCH64_CALL26: + case ELF::R_AARCH64_ADR_PREL_LO21: case ELF::R_AARCH64_ADR_PREL_PG_HI21: + case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC: case ELF::R_AARCH64_LDST64_ABS_LO12_NC: case ELF::R_AARCH64_ADD_ABS_LO12_NC: case ELF::R_AARCH64_LDST128_ABS_LO12_NC: @@ -109,6 +114,7 @@ size_t getSizeForTypeAArch64(uint64_t Type) { case ELF::R_AARCH64_LDST16_ABS_LO12_NC: case ELF::R_AARCH64_LDST8_ABS_LO12_NC: case ELF::R_AARCH64_ADR_GOT_PAGE: + case ELF::R_AARCH64_TLSDESC_ADR_PREL21: case ELF::R_AARCH64_TLSDESC_ADR_PAGE21: case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12: @@ -147,15 +153,23 @@ uint64_t extractValueAArch64(uint64_t Type, uint64_t Contents, uint64_t PC) { Contents &= ~0xfffffffffc000000ULL; return static_cast(PC) + SignExtend64<28>(Contents << 2); case ELF::R_AARCH64_ADR_GOT_PAGE: + case ELF::R_AARCH64_TLSDESC_ADR_PREL21: case ELF::R_AARCH64_TLSDESC_ADR_PAGE21: case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: - case ELF::R_AARCH64_ADR_PREL_PG_HI21: { + case ELF::R_AARCH64_ADR_PREL_LO21: + case ELF::R_AARCH64_ADR_PREL_PG_HI21: + case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC: { // Bits 32:12 of Symbol address goes in bits 30:29 + 23:5 of ADRP - // instruction + // and ADR instructions + bool IsAdr = !!(((Contents >> 31) & 0x1) == 0); Contents &= ~0xffffffff9f00001fUll; uint64_t LowBits = (Contents >> 29) & 0x3; uint64_t HighBits = (Contents >> 5) & 0x7ffff; Contents = LowBits | (HighBits << 2); + if (IsAdr) + return static_cast(PC) + SignExtend64<21>(Contents); + + // ADRP instruction Contents = static_cast(PC) + SignExtend64<33>(Contents << 12); Contents &= ~0xfffUll; return Contents; @@ -234,6 +248,7 @@ bool isGOTAArch64(uint64_t Type) { case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12: case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: + case ELF::R_AARCH64_TLSDESC_ADR_PREL21: case ELF::R_AARCH64_TLSDESC_ADR_PAGE21: case ELF::R_AARCH64_TLSDESC_LD64_LO12: case ELF::R_AARCH64_TLSDESC_ADD_LO12: @@ -257,6 +272,7 @@ bool isTLSAArch64(uint64_t Type) { switch (Type) { default: return false; + case ELF::R_AARCH64_TLSDESC_ADR_PREL21: case ELF::R_AARCH64_TLSDESC_ADR_PAGE21: case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12: @@ -312,9 +328,12 @@ bool isPCRelativeAArch64(uint64_t Type) { return false; case ELF::R_AARCH64_TLSDESC_CALL: case ELF::R_AARCH64_CALL26: + case ELF::R_AARCH64_ADR_PREL_LO21: case ELF::R_AARCH64_ADR_PREL_PG_HI21: + case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC: case ELF::R_AARCH64_ADR_GOT_PAGE: case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: + case ELF::R_AARCH64_TLSDESC_ADR_PREL21: case ELF::R_AARCH64_TLSDESC_ADR_PAGE21: case ELF::R_AARCH64_JUMP26: case ELF::R_AARCH64_PREL32: diff --git a/bolt/src/RewriteInstance.cpp b/bolt/src/RewriteInstance.cpp index 3d4d6b7f8568..fe68c7854341 100644 --- a/bolt/src/RewriteInstance.cpp +++ b/bolt/src/RewriteInstance.cpp @@ -998,7 +998,8 @@ void RewriteInstance::discoverFileObjects() { [](const SymbolRef &Symbol) { StringRef Name = cantFail(Symbol.getName()); return !(cantFail(Symbol.getType()) == SymbolRef::ST_Unknown && - (Name == "$d" || Name == "$x")); + (Name == "$d" || Name.startswith("$d.") || Name == "$x" || + Name.startswith("$x."))); }); --LastSymbol; } diff --git a/bolt/src/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/src/Target/AArch64/AArch64MCPlusBuilder.cpp index 144140ed4916..4a945ff6d3a2 100644 --- a/bolt/src/Target/AArch64/AArch64MCPlusBuilder.cpp +++ b/bolt/src/Target/AArch64/AArch64MCPlusBuilder.cpp @@ -66,6 +66,15 @@ public: return Inst.getOpcode() == AArch64::ADR; } + void getADRReg(const MCInst &Inst, MCPhysReg &RegName) const override { + assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction"); + assert(MCPlus::getNumPrimeOperands(Inst) != 0 && + "No operands for ADR instruction"); + assert(Inst.getOperand(0).isReg() && + "Unexpected operand in ADR instruction"); + RegName = Inst.getOperand(0).getReg(); + } + bool isTB(const MCInst &Inst) const { return (Inst.getOpcode() == AArch64::TBNZW || Inst.getOpcode() == AArch64::TBNZX || @@ -312,13 +321,18 @@ public: const MCExpr *getTargetExprFor(MCInst &Inst, const MCExpr *Expr, MCContext &Ctx, uint64_t RelType) const override { - if (RelType == ELF::R_AARCH64_ADR_GOT_PAGE || - RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21) { + + if (isADR(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_LO21 || + RelType == ELF::R_AARCH64_TLSDESC_ADR_PREL21) { + return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS, Ctx); + } else if (isADRP(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21 || + RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21_NC || + RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21 || + RelType == ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 || + RelType == ELF::R_AARCH64_ADR_GOT_PAGE) { // Never emit a GOT reloc, we handled this in // RewriteInstance::readRelocations(). return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx); - } else if (Inst.getOpcode() == AArch64::ADRP) { - return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx); } else { switch(RelType) { case ELF::R_AARCH64_LDST8_ABS_LO12_NC: @@ -337,27 +351,13 @@ public: return Expr; } - const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override { - auto *AArchExpr = dyn_cast(Expr); - if (AArchExpr && AArchExpr->getSubExpr()) { - return getTargetSymbol(AArchExpr->getSubExpr()); - } - - auto *SymExpr = dyn_cast(Expr); - if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None) - return nullptr; - - return &SymExpr->getSymbol(); - } - - const MCSymbol *getTargetSymbol(const MCInst &Inst, - unsigned OpNum = 0) const override { + bool getSymbolRefOperandNum(const MCInst &Inst, unsigned &OpNum) const { if (OpNum >= MCPlus::getNumPrimeOperands(Inst)) - return nullptr; + return false; // Auto-select correct operand number if (OpNum == 0) { - if (isConditionalBranch(Inst)) + if (isConditionalBranch(Inst) || isADR(Inst) || isADRP(Inst)) OpNum = 1; if (isTB(Inst)) OpNum = 2; @@ -365,6 +365,30 @@ public: OpNum = 1; } + return true; + } + + const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override { + auto *AArchExpr = dyn_cast(Expr); + if (AArchExpr && AArchExpr->getSubExpr()) + return getTargetSymbol(AArchExpr->getSubExpr()); + + auto *BinExpr = dyn_cast(Expr); + if (BinExpr) + return getTargetSymbol(BinExpr->getLHS()); + + auto *SymExpr = dyn_cast(Expr); + if (SymExpr && SymExpr->getKind() == MCSymbolRefExpr::VK_None) + return &SymExpr->getSymbol(); + + return nullptr; + } + + const MCSymbol *getTargetSymbol(const MCInst &Inst, + unsigned OpNum = 0) const override { + if (!getSymbolRefOperandNum(Inst, OpNum)) + return nullptr; + const MCOperand &Op = Inst.getOperand(OpNum); if (!Op.isExpr()) return nullptr; @@ -372,6 +396,34 @@ public: return getTargetSymbol(Op.getExpr()); } + int64_t getTargetAddend(const MCExpr *Expr) const override { + auto *AArchExpr = dyn_cast(Expr); + if (AArchExpr && AArchExpr->getSubExpr()) + return getTargetAddend(AArchExpr->getSubExpr()); + + auto *BinExpr = dyn_cast(Expr); + if (BinExpr && BinExpr->getOpcode() == MCBinaryExpr::Add) + return getTargetAddend(BinExpr->getRHS()); + + auto *ConstExpr = dyn_cast(Expr); + if (ConstExpr) + return ConstExpr->getValue(); + + return 0; + } + + int64_t getTargetAddend(const MCInst &Inst, + unsigned OpNum = 0) const override { + if (!getSymbolRefOperandNum(Inst, OpNum)) + return 0; + + const MCOperand &Op = Inst.getOperand(OpNum); + if (!Op.isExpr()) + return 0; + + return getTargetAddend(Op.getExpr()); + } + bool evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size, uint64_t &Target) const override { size_t OpNum = 0; @@ -1037,6 +1089,28 @@ public: Inst.addOperand(MCOperand::createReg(AArch64::LR)); return true; } + + std::vector materializeAddress(const MCSymbol *Target, MCContext *Ctx, + MCPhysReg RegName, + int64_t Addend) const override { + // Get page-aligned address and add page offset + std::vector Insts(2); + Insts[0].setOpcode(AArch64::ADRP); + Insts[0].clear(); + Insts[0].addOperand(MCOperand::createReg(RegName)); + Insts[0].addOperand(MCOperand::createImm(0)); + setOperandToSymbolRef(Insts[0], /* OpNum */ 1, Target, Addend, Ctx, + ELF::R_AARCH64_NONE); + Insts[1].setOpcode(AArch64::ADDXri); + Insts[1].clear(); + Insts[1].addOperand(MCOperand::createReg(RegName)); + Insts[1].addOperand(MCOperand::createReg(RegName)); + Insts[1].addOperand(MCOperand::createImm(0)); + Insts[1].addOperand(MCOperand::createImm(0)); + setOperandToSymbolRef(Insts[1], /* OpNum */ 2, Target, Addend, Ctx, + ELF::R_AARCH64_ADD_ABS_LO12_NC); + return Insts; + } }; } // end anonymous namespace diff --git a/bolt/test/AArch64/adrrelaxationpass.s b/bolt/test/AArch64/adrrelaxationpass.s new file mode 100644 index 000000000000..4a74a6b59cba --- /dev/null +++ b/bolt/test/AArch64/adrrelaxationpass.s @@ -0,0 +1,49 @@ +# The second and third ADR instructions are non-local to functions +# and must be replaced with ADRP + ADD by BOLT + +# REQUIRES: system-linux + +# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown \ +# RUN: %s -o %t.o +# RUN: %host_cc %cflags %t.o -o %t.exe -Wl,-q +# RUN: llvm-bolt %t.exe -o %t.bolt -adr-relaxation=true +# RUN: llvm-objdump -d --disassemble-symbols=main %t.bolt | FileCheck %s +# RUN: %t.bolt + + .data + .align 8 + .global Gvar +Gvar: .dword 0x0 + .global Gvar2 +Gvar2: .dword 0x42 + + .text + .align 4 + .global test + .type test, %function +test: + mov x0, xzr + ret + .size test, .-test + + .align 4 + .global main + .type main, %function +main: + adr x0, .CI + adr x1, test + adr x2, Gvar2 + adr x3, br +br: + br x1 + .size main, .-main +.CI: + .word 0xff + +# CHECK:
: +# CHECK-NEXT: adr x0, #28 +# CHECK-NEXT: adrp x1, 0x{{[1-8a-f][0-9a-f]*}} +# CHECK-NEXT: add x1, x1, #{{[1-8a-f][0-9a-f]*}} +# CHECK-NEXT: adrp x2, 0x{{[1-8a-f][0-9a-f]*}} +# CHECK-NEXT: add x2, x2, #{{[1-8a-f][0-9a-f]*}} +# CHECK-NEXT: adr x3, #4 diff --git a/bolt/test/AArch64/lit.local.cfg b/bolt/test/AArch64/lit.local.cfg new file mode 100644 index 000000000000..c4f2399f8fdd --- /dev/null +++ b/bolt/test/AArch64/lit.local.cfg @@ -0,0 +1,2 @@ +if config.host_arch not in ['AArch64']: + config.unsupported = True diff --git a/bolt/test/X86/lit.local.cfg b/bolt/test/X86/lit.local.cfg new file mode 100644 index 000000000000..7d0b9ea5a8b5 --- /dev/null +++ b/bolt/test/X86/lit.local.cfg @@ -0,0 +1,2 @@ +if config.host_arch not in ['x86', 'X86', 'x86_64']: + config.unsupported = True