diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h index e605ce4c704a..0a593b2b4a31 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h @@ -26,6 +26,7 @@ enum ELFX86RelocationKind : Edge::Kind { Pointer64, Pointer64Anon, PCRel32, + PCRel64, PCRel32Minus1, PCRel32Minus2, PCRel32Minus4, @@ -35,6 +36,8 @@ enum ELFX86RelocationKind : Edge::Kind { PCRel32Minus4Anon, PCRel32GOTLoad, PCRel32GOT, + PCRel64GOT, + GOT64, PCRel32TLV, Delta32, Delta64, diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h index 032cd58db968..c2335b024198 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -674,7 +674,7 @@ public: assert((First || !Last) && "Last can not be null if start is non-null"); return Last; } - bool isEmpty() const { + bool empty() const { assert((First || !Last) && "Last can not be null if start is non-null"); return !First; } diff --git a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h index 2777bbaea9ff..8ae3bc2bf61d 100644 --- a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h +++ b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h @@ -40,18 +40,24 @@ public: : F(std::move(F)) {} Error operator()(LinkGraph &G) { - for (auto *Sym : G.external_symbols()) { + + // This pass will affect the external symbols set, so copy them out into a + // vector and iterate over that. + std::vector Externals(G.external_symbols().begin(), + G.external_symbols().end()); + + for (auto *Sym : Externals) { SectionRangeSymbolDesc D = F(G, *Sym); if (D.Sec) { auto &SR = getSectionRange(*D.Sec); if (D.IsStart) { - if (SR.isEmpty()) + if (SR.empty()) G.makeAbsolute(*Sym, 0); else G.makeDefined(*Sym, *SR.getFirstBlock(), 0, 0, Linkage::Strong, Scope::Local, false); } else { - if (SR.isEmpty()) + if (SR.empty()) G.makeAbsolute(*Sym, 0); else G.makeDefined(*Sym, *SR.getLastBlock(), diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index 8bba5dd77f8f..eb38313b4280 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -40,7 +40,8 @@ public: PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder; bool isGOTEdgeToFix(Edge &E) const { - return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; + return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad || + E.getKind() == PCRel64GOT || E.getKind() == GOT64; } Symbol &createGOTEntry(Symbol &Target) { @@ -51,14 +52,25 @@ public: } void fixGOTEdge(Edge &E, Symbol &GOTEntry) { - assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && - "Not a GOT edge?"); - // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is - // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to - // check for GOT optimization opportunities in the + // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary + // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now: + // We will use the kind to check for GOT optimization opportunities in the // optimizeMachO_x86_64_GOTAndStubs pass below. - if (E.getKind() == PCRel32GOT) + // If it's a GOT64 leave it as is. + switch (E.getKind()) { + case PCRel32GOT: E.setKind(PCRel32); + break; + case PCRel64GOT: + E.setKind(PCRel64); + break; + case GOT64: + break; + case PCRel32GOTLoad: + break; + default: + llvm_unreachable("Unexpected GOT edge kind"); + } E.setTarget(GOTEntry); // Leave the edge addend as-is. @@ -117,7 +129,8 @@ private: Section *StubsSection = nullptr; }; -StringRef ELFGOTSectionName = "$__GOT"; +constexpr StringRef ELFGOTSectionName = "$__GOT"; +constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_"; const char *const DwarfSectionNames[] = { #define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION) \ @@ -245,6 +258,7 @@ private: case ELF::R_X86_64_PC32: return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32; case ELF::R_X86_64_PC64: + case ELF::R_X86_64_GOTPC64: return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64; case ELF::R_X86_64_64: return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64; @@ -252,6 +266,10 @@ private: case ELF::R_X86_64_GOTPCRELX: case ELF::R_X86_64_REX_GOTPCRELX: return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad; + case ELF::R_X86_64_GOTPCREL64: + return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT; + case ELF::R_X86_64_GOT64: + return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64; case ELF::R_X86_64_PLT32: return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32; } @@ -699,9 +717,57 @@ public: ELFJITLinker_x86_64(std::unique_ptr Ctx, std::unique_ptr G, PassConfiguration PassConfig) - : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) { + getPassConfig().PostAllocationPasses.push_back( + [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); }); + } private: + Symbol *GOTSymbol = nullptr; + + Error getOrCreateGOTSymbol(LinkGraph &G) { + Section *GOTSection = nullptr; + + auto DefineExternalGOTSymbolIfPresent = + createDefineExternalSectionStartAndEndSymbolsPass( + [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc { + if (Sym.getName() == ELFGOTSymbolName) + if ((GOTSection = G.findSectionByName(ELFGOTSectionName))) { + GOTSymbol = &Sym; + return {*GOTSection, true}; + } + return {}; + }); + + // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an + // external. + if (auto Err = DefineExternalGOTSymbolIfPresent(G)) + return Err; + + // If there's a GOT section but we didn't find an external GOT symbol... + if (GOTSection && !GOTSymbol) { + + // Check for an existing defined symbol. + for (auto *Sym : GOTSection->symbols()) + if (Sym->getName() == ELFGOTSymbolName) { + GOTSymbol = Sym; + return Error::success(); + } + + // If there's no defined symbol then create one. + SectionRange SR(*GOTSection); + if (SR.empty()) + GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0, + Linkage::Strong, Scope::Local, true); + else + GOTSymbol = + &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0, + Linkage::Strong, Scope::Local, false, true); + } + + return Error::success(); + } + Error applyFixup(LinkGraph &G, Block &B, const Edge &E, char *BlockWorkingMem) const { using namespace ELF_x86_64_Edges; @@ -720,6 +786,11 @@ private: return makeTargetOutOfRangeError(G, B, E); break; } + case ELFX86RelocationKind::PCRel64: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + *(little64_t *)FixupPtr = Value; + break; + } case ELFX86RelocationKind::Pointer64: { int64_t Value = E.getTarget().getAddress() + E.getAddend(); *(ulittle64_t *)FixupPtr = Value; @@ -751,6 +822,13 @@ private: *(little64_t *)FixupPtr = Value; break; } + case ELFX86RelocationKind::GOT64: { + assert(GOTSymbol && "No GOT section symbol"); + int64_t Value = + E.getTarget().getAddress() - GOTSymbol->getAddress() + E.getAddend(); + *(little64_t *)FixupPtr = Value; + break; + } default: LLVM_DEBUG({ dbgs() << "Bad edge: " << getELFX86RelocationKindName(E.getKind()) @@ -793,9 +871,6 @@ identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { if (auto *Sec = G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) return {*Sec, false}; - } else if (SymName == "_GLOBAL_OFFSET_TABLE_") { - if (auto *GOTSec = G.findSectionByName(ELFGOTSectionName)) - return {*GOTSec, true}; } return {}; } @@ -822,13 +897,13 @@ void link_ELF_x86_64(std::unique_ptr G, Config.PostPrunePasses.push_back( PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass); - // Add GOT/Stubs optimizer pass. - Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs); - // Resolve any external section start / end symbols. - Config.PreFixupPasses.push_back( + Config.PostAllocationPasses.push_back( createDefineExternalSectionStartAndEndSymbolsPass( identifyELFSectionStartAndEndSymbols)); + + // Add GOT/Stubs optimizer pass. + Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs); } if (auto Err = Ctx->modifyPassConfig(*G, Config)) diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index 26e07be52820..ef68f96149ab 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -51,6 +51,12 @@ protected: using SegmentLayoutMap = DenseMap; + // Returns the PassConfiguration for this instance. This can be used by + // JITLinkerBase implementations to add late passes that reference their + // own data structures (e.g. for ELF implementations to locate / construct + // a GOT start symbol prior to fixup). + PassConfiguration &getPassConfig() { return Passes; } + // Phase 1: // 1.1: Run pre-prune passes // 1.2: Prune graph diff --git a/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_large_pic_relocations.s b/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_large_pic_relocations.s new file mode 100644 index 000000000000..b70debd8565b --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_large_pic_relocations.s @@ -0,0 +1,58 @@ +# RUN: rm -rf %t && mkdir -p %t +# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent -filetype=obj \ +# RUN: -large-code-model -o %t/elf_lg_pic_reloc.o %s +# RUN: llvm-jitlink -noexec -slab-allocate 100Kb -slab-address 0xfff00000 \ +# RUN: -check %s %t/elf_lg_pic_reloc.o +# +# Test ELF large/PIC relocations. + + .text + .file "testcase.c" + + # Empty main entry point. + .globl main + .p2align 4, 0x90 + .type main,@function +main: + retq + + .size main, .-main + +# Test R_X86_64_GOTPC64 handling. We want to check that the offset of the +# operand is the 64-bit delta to the start of the GOT. +# jitlink-check: decode_operand(test_gotpc64, 1) = \ +# jitlink-check: _GLOBAL_OFFSET_TABLE_ - test_lg_pic_GOT +# jitlink-check: decode_operand(test_got64, 1) = \ +# jitlink-check: got_addr(elf_lg_pic_reloc.o, named_data) - \ +# jitlink-check: _GLOBAL_OFFSET_TABLE_ + .globl test_lg_pic_GOT + .p2align 4, 0x90 + .type test_lg_pic_GOT,@function +test_lg_pic_GOT: +.L0$pb: + leaq .L0$pb(%rip), %rax + + .globl test_gotpc64 +test_gotpc64: + movabsq $_GLOBAL_OFFSET_TABLE_-.L0$pb, %rcx + .size test_gotpc64, .-test_gotpc64 + + addq %rax, %rcx + .globl test_got64 +test_got64: + movabsq $named_data@GOT, %rax + .size test_got64, .-test_got64 + + .size test_lg_pic_GOT, .-test_lg_pic_GOT + + .data + + .type named_data,@object + .p2align 3 +named_data: + .quad 42 + .size named_data, 8 + + .ident "clang version 10.0.0-4ubuntu1 " + .section ".note.GNU-stack","",@progbits + .addrsig diff --git a/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_relocations.s b/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_small_pic_relocations.s similarity index 83% rename from llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_relocations.s rename to llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_small_pic_relocations.s index c2039bd64264..50f073beb662 100644 --- a/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_relocations.s +++ b/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_small_pic_relocations.s @@ -1,12 +1,13 @@ # RUN: rm -rf %t && mkdir -p %t -# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent -filetype=obj -o %t/elf_reloc.o %s +# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent -filetype=obj \ +# RUN: -o %t/elf_sm_pic_reloc.o %s # RUN: llvm-jitlink -noexec -slab-allocate 100Kb -slab-address 0xfff00000 \ # RUN: -define-abs external_data=0x1 \ # RUN: -define-abs extern_in_range32=0xffe00000 \ # RUN: -define-abs extern_out_of_range32=0x7fff00000000 \ -# RUN: -check %s %t/elf_reloc.o +# RUN: -check %s %t/elf_sm_pic_reloc.o # -# Test standard ELF relocations. +# Test ELF small/PIC relocations. .text .file "testcase.c" @@ -55,7 +56,8 @@ test_call_local: # resolution, the target turns out to be in-range from the callsite and so the # edge is relaxed in post-allocation optimization. # -# jitlink-check: decode_operand(test_call_extern, 0) = extern_in_range32 - next_pc(test_call_extern) +# jitlink-check: decode_operand(test_call_extern, 0) = \ +# jitlink-check: extern_in_range32 - next_pc(test_call_extern) .globl test_call_extern .p2align 4, 0x90 .type test_call_extern,@function @@ -70,8 +72,10 @@ test_call_extern: # entry. # # jitlink-check: decode_operand(test_call_extern_plt, 0) = \ -# jitlink-check: stub_addr(elf_reloc.o, extern_out_of_range32) - next_pc(test_call_extern_plt) -# jitlink-check: *{8}(got_addr(elf_reloc.o, extern_out_of_range32)) = extern_out_of_range32 +# jitlink-check: stub_addr(elf_sm_pic_reloc.o, extern_out_of_range32) - \ +# jitlink-check: next_pc(test_call_extern_plt) +# jitlink-check: *{8}(got_addr(elf_sm_pic_reloc.o, extern_out_of_range32)) = \ +# jitlink-check: extern_out_of_range32 .globl test_call_extern_plt .p2align 4, 0x90 .type test_call_extern_plt,@function @@ -82,8 +86,9 @@ test_call_extern_plt: # Test GOTPCREL handling. We want to check both the offset to the GOT entry and its # contents. -# jitlink-check: decode_operand(test_gotpcrel, 4) = got_addr(elf_reloc.o, named_data) - next_pc(test_gotpcrel) -# jitlink-check: *{8}(got_addr(elf_reloc.o, named_data)) = named_data +# jitlink-check: decode_operand(test_gotpcrel, 4) = \ +# jitlink-check: got_addr(elf_sm_pic_reloc.o, named_data) - next_pc(test_gotpcrel) +# jitlink-check: *{8}(got_addr(elf_sm_pic_reloc.o, named_data)) = named_data .globl test_gotpcrel .p2align 4, 0x90 @@ -96,7 +101,7 @@ test_gotpcrel: # Test REX_GOTPCRELX handling. We want to check both the offset to the GOT entry and its # contents. # jitlink-check: decode_operand(test_rex_gotpcrelx, 4) = \ -# jitlink-check: got_addr(elf_reloc.o, external_data) - next_pc(test_rex_gotpcrelx) +# jitlink-check: got_addr(elf_sm_pic_reloc.o, external_data) - next_pc(test_rex_gotpcrelx) .globl test_rex_gotpcrelx .p2align 4, 0x90 diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink-elf.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink-elf.cpp index beb73fb8edf7..efa39b02b325 100644 --- a/llvm/tools/llvm-jitlink/llvm-jitlink-elf.cpp +++ b/llvm/tools/llvm-jitlink/llvm-jitlink-elf.cpp @@ -133,8 +133,9 @@ Error registerELFGraphInfo(Session &S, LinkGraph &G) { else return TS.takeError(); SectionContainsContent = true; - } else if (Sym->hasName()) { - dbgs() << "Symbol: " << Sym->getName() << "\n"; + } + + if (Sym->hasName()) { if (Sym->isSymbolZeroFill()) { S.SymbolInfos[Sym->getName()] = {Sym->getSize(), Sym->getAddress()}; SectionContainsZeroFill = true;