From 0e940d55f8a9388c42cc5998ea05212a983f05a7 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Thu, 16 Jul 2020 12:51:14 -0700 Subject: [PATCH] [ORC] Add TargetProcessControl and TPCIndirectionUtils APIs. TargetProcessControl is a new API for communicating with JIT target processes. It supports memory allocation and access, and inspection of some process properties, e.g. the target proces triple and page size. Centralizing these APIs allows utilities written against TargetProcessControl to remain independent of the communication procotol with the target process (which may be direct memory access/allocation for in-process JITing, or may involve some form of IPC or RPC). An initial set of TargetProcessControl-based utilities for lazy compilation is provided by the TPCIndirectionUtils class. An initial implementation of TargetProcessControl for in-process JITing is provided by the SelfTargetProcessControl class. An example program showing how the APIs can be used is provided in llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl. --- llvm/examples/OrcV2Examples/CMakeLists.txt | 1 + .../CMakeLists.txt | 12 + .../LLJITWithTargetProcessControl.cpp | 178 ++++++++ .../llvm/ExecutionEngine/Orc/LazyReexports.h | 6 +- .../ExecutionEngine/Orc/TPCIndirectionUtils.h | 209 +++++++++ .../Orc/TargetProcessControl.h | 162 +++++++ llvm/lib/ExecutionEngine/Orc/CMakeLists.txt | 4 +- .../Orc/TPCIndirectionUtils.cpp | 420 ++++++++++++++++++ .../Orc/TargetProcessControl.cpp | 79 ++++ 9 files changed, 1067 insertions(+), 4 deletions(-) create mode 100644 llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/CMakeLists.txt create mode 100644 llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/LLJITWithTargetProcessControl.cpp create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h create mode 100644 llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp create mode 100644 llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp diff --git a/llvm/examples/OrcV2Examples/CMakeLists.txt b/llvm/examples/OrcV2Examples/CMakeLists.txt index 2c737296ae10..0f1be0e35f23 100644 --- a/llvm/examples/OrcV2Examples/CMakeLists.txt +++ b/llvm/examples/OrcV2Examples/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(LLJITWithInitializers) add_subdirectory(LLJITWithLazyReexports) add_subdirectory(LLJITWithObjectCache) add_subdirectory(LLJITWithObjectLinkingLayerPlugin) +add_subdirectory(LLJITWithTargetProcessControl) add_subdirectory(OrcV2CBindingsAddObjectFile) add_subdirectory(OrcV2CBindingsBasicUsage) add_subdirectory(OrcV2CBindingsReflectProcessSymbols) diff --git a/llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/CMakeLists.txt b/llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/CMakeLists.txt new file mode 100644 index 000000000000..10e0de8d64bd --- /dev/null +++ b/llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/CMakeLists.txt @@ -0,0 +1,12 @@ +set(LLVM_LINK_COMPONENTS + Core + ExecutionEngine + IRReader + OrcJIT + Support + nativecodegen + ) + +add_llvm_example(LLJITWithTargetProcessControl + LLJITWithTargetProcessControl.cpp + ) diff --git a/llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/LLJITWithTargetProcessControl.cpp b/llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/LLJITWithTargetProcessControl.cpp new file mode 100644 index 000000000000..a88360e3a8cd --- /dev/null +++ b/llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/LLJITWithTargetProcessControl.cpp @@ -0,0 +1,178 @@ +//===--- LLJITWithLazyReexports.cpp - LLJIT example with custom laziness --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// In this example we will use the lazy re-exports utility to lazily compile +// IR modules. We will do this in seven steps: +// +// 1. Create an LLJIT instance. +// 2. Install a transform so that we can see what is being compiled. +// 3. Create an indirect stubs manager and lazy call-through manager. +// 4. Add two modules that will be conditionally compiled, plus a main module. +// 5. Add lazy-rexports of the symbols in the conditionally compiled modules. +// 6. Dump the ExecutionSession state to see the symbol table prior to +// executing any code. +// 7. Verify that only modules containing executed code are compiled. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringMap.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/Orc/OrcABISupport.h" +#include "llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h" +#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/raw_ostream.h" + +#include "../ExampleModules.h" + +#include + +using namespace llvm; +using namespace llvm::orc; + +ExitOnError ExitOnErr; + +// Example IR modules. +// +// Note that in the conditionally compiled modules, FooMod and BarMod, functions +// have been given an _body suffix. This is to ensure that their names do not +// clash with their lazy-reexports. +// For clients who do not wish to rename function bodies (e.g. because they want +// to re-use cached objects between static and JIT compiles) techniques exist to +// avoid renaming. See the lazy-reexports section of the ORCv2 design doc. + +const llvm::StringRef FooMod = + R"( + define i32 @foo_body() { + entry: + ret i32 1 + } +)"; + +const llvm::StringRef BarMod = + R"( + define i32 @bar_body() { + entry: + ret i32 2 + } +)"; + +const llvm::StringRef MainMod = + R"( + + define i32 @entry(i32 %argc) { + entry: + %and = and i32 %argc, 1 + %tobool = icmp eq i32 %and, 0 + br i1 %tobool, label %if.end, label %if.then + + if.then: ; preds = %entry + %call = tail call i32 @foo() #2 + br label %return + + if.end: ; preds = %entry + %call1 = tail call i32 @bar() #2 + br label %return + + return: ; preds = %if.end, %if.then + %retval.0 = phi i32 [ %call, %if.then ], [ %call1, %if.end ] + ret i32 %retval.0 + } + + declare i32 @foo() + declare i32 @bar() +)"; + +static void *reenter(void *Ctx, void *TrampolineAddr) { + std::promise LandingAddressP; + auto LandingAddressF = LandingAddressP.get_future(); + + auto *TPCIU = static_cast(Ctx); + TPCIU->getLazyCallThroughManager().resolveTrampolineLandingAddress( + pointerToJITTargetAddress(TrampolineAddr), + [&](JITTargetAddress LandingAddress) { + LandingAddressP.set_value( + jitTargetAddressToPointer(LandingAddress)); + }); + return LandingAddressF.get(); +} + +cl::list InputArgv(cl::Positional, + cl::desc("...")); + +int main(int argc, char *argv[]) { + // Initialize LLVM. + InitLLVM X(argc, argv); + + InitializeNativeTarget(); + InitializeNativeTargetAsmPrinter(); + + cl::ParseCommandLineOptions(argc, argv, "LLJITWithLazyReexports"); + ExitOnErr.setBanner(std::string(argv[0]) + ": "); + + // (1) Create LLJIT instance. + auto J = ExitOnErr(LLJITBuilder().create()); + + // (2) Install transform to print modules as they are compiled: + J->getIRTransformLayer().setTransform( + [](ThreadSafeModule TSM, + const MaterializationResponsibility &R) -> Expected { + TSM.withModuleDo([](Module &M) { dbgs() << "---Compiling---\n" << M; }); + return std::move(TSM); // Not a redundant move: fix build on gcc-7.5 + }); + + // (3) Create stubs and call-through managers: + + auto TPC = ExitOnErr(SelfTargetProcessControl::Create()); + auto TPCIU = ExitOnErr(TPCIndirectionUtils::Create(*TPC)); + ExitOnErr(TPCIU->writeResolverBlock(pointerToJITTargetAddress(&reenter), + pointerToJITTargetAddress(TPCIU.get()))); + TPCIU->createLazyCallThroughManager(J->getExecutionSession(), 0); + auto ISM = TPCIU->createIndirectStubsManager(); + + // (4) Add modules. + ExitOnErr(J->addIRModule(ExitOnErr(parseExampleModule(FooMod, "foo-mod")))); + ExitOnErr(J->addIRModule(ExitOnErr(parseExampleModule(BarMod, "bar-mod")))); + ExitOnErr(J->addIRModule(ExitOnErr(parseExampleModule(MainMod, "main-mod")))); + + // (5) Add lazy reexports. + MangleAndInterner Mangle(J->getExecutionSession(), J->getDataLayout()); + SymbolAliasMap ReExports( + {{Mangle("foo"), + {Mangle("foo_body"), + JITSymbolFlags::Exported | JITSymbolFlags::Callable}}, + {Mangle("bar"), + {Mangle("bar_body"), + JITSymbolFlags::Exported | JITSymbolFlags::Callable}}}); + ExitOnErr(J->getMainJITDylib().define( + lazyReexports(TPCIU->getLazyCallThroughManager(), *ISM, + J->getMainJITDylib(), std::move(ReExports)))); + + // (6) Dump the ExecutionSession state. + dbgs() << "---Session state---\n"; + J->getExecutionSession().dump(dbgs()); + dbgs() << "\n"; + + // (7) Execute the JIT'd main function and pass the example's command line + // arguments unmodified. This should cause either ExampleMod1 or ExampleMod2 + // to be compiled, and either "1" or "2" returned depending on the number of + // arguments passed. + + // Look up the JIT'd function, cast it to a function pointer, then call it. + auto EntrySym = ExitOnErr(J->lookup("entry")); + auto *Entry = (int (*)(int))EntrySym.getAddress(); + + int Result = Entry(argc); + outs() << "---Result---\n" + << "entry(" << argc << ") = " << Result << "\n"; + + return 0; +} diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h index 0d3ccecdf121..9206e40fffb1 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h @@ -40,6 +40,9 @@ public: using NotifyResolvedFunction = unique_function; + LazyCallThroughManager(ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddr, TrampolinePool *TP); + // Return a free call-through trampoline and bind it to look up and call // through to the given symbol. Expected @@ -56,9 +59,6 @@ protected: using NotifyLandingResolvedFunction = TrampolinePool::NotifyLandingResolvedFunction; - LazyCallThroughManager(ExecutionSession &ES, - JITTargetAddress ErrorHandlerAddr, TrampolinePool *TP); - struct ReexportsEntry { JITDylib *SourceJD; SymbolStringPtr SymbolName; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h new file mode 100644 index 000000000000..db9cd1b98cf9 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h @@ -0,0 +1,209 @@ +//===--- TPCIndirectionUtils.h - TPC based indirection utils ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Indirection utilities (stubs, trampolines, lazy call-throughs) that use the +// TargetProcessControl API to interact with the target process. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TPCINDIRECTIONUTILS_H +#define LLVM_EXECUTIONENGINE_ORC_TPCINDIRECTIONUTILS_H + +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" +#include "llvm/ExecutionEngine/Orc/LazyReexports.h" + +#include + +namespace llvm { +namespace orc { + +class TargetProcessControl; + +/// Provides TargetProcessControl based indirect stubs, trampoline pool and +/// lazy call through manager. +class TPCIndirectionUtils { + friend class TPCIndirectionUtilsAccess; + +public: + /// ABI support base class. Used to write resolver, stub, and trampoline + /// blocks. + class ABISupport { + protected: + ABISupport(unsigned PointerSize, unsigned TrampolineSize, unsigned StubSize, + unsigned StubToPointerMaxDisplacement, unsigned ResolverCodeSize) + : PointerSize(PointerSize), TrampolineSize(TrampolineSize), + StubSize(StubSize), + StubToPointerMaxDisplacement(StubToPointerMaxDisplacement), + ResolverCodeSize(ResolverCodeSize) {} + + public: + virtual ~ABISupport(); + + unsigned getPointerSize() const { return PointerSize; } + unsigned getTrampolineSize() const { return TrampolineSize; } + unsigned getStubSize() const { return StubSize; } + unsigned getStubToPointerMaxDisplacement() const { + return StubToPointerMaxDisplacement; + } + unsigned getResolverCodeSize() const { return ResolverCodeSize; } + + virtual void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddr, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) const = 0; + + virtual void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTragetAddr, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines) const = 0; + + virtual void + writeIndirectStubsBlock(char *StubsBlockWorkingMem, + JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, + unsigned NumStubs) const = 0; + + private: + unsigned PointerSize = 0; + unsigned TrampolineSize = 0; + unsigned StubSize = 0; + unsigned StubToPointerMaxDisplacement = 0; + unsigned ResolverCodeSize = 0; + }; + + /// Create using the given ABI class. + template + static std::unique_ptr + CreateWithABI(TargetProcessControl &TPC); + + /// Create based on the TargetProcessControl triple. + static Expected> + Create(TargetProcessControl &TPC); + + /// Return a reference to the TargetProcessControl object. + TargetProcessControl &getTargetProcessControl() const { return TPC; } + + /// Return a reference to the ABISupport object for this instance. + ABISupport &getABISupport() const { return *ABI; } + + /// Release memory for resources held by this instance. This *must* be called + /// prior to destruction of the class. + Error cleanup(); + + /// Write resolver code to the target process and return its address. + /// This must be called before any call to createTrampolinePool or + /// createLazyCallThroughManager. + Expected + writeResolverBlock(JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr); + + /// Returns the address of the Resolver block. Returns zero if the + /// writeResolverBlock method has not previously been called. + JITTargetAddress getResolverBlockAddress() const { return ResolverBlockAddr; } + + /// Create an IndirectStubsManager for the target process. + std::unique_ptr createIndirectStubsManager(); + + /// Create a TrampolinePool for the target process. + TrampolinePool &getTrampolinePool(); + + /// Create a LazyCallThroughManager. + /// This function should only be called once. + LazyCallThroughManager & + createLazyCallThroughManager(ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddr); + + /// Create a LazyCallThroughManager for the target process. + LazyCallThroughManager &getLazyCallThroughManager() { + assert(LCTM && "createLazyCallThroughManager must be called first"); + return *LCTM; + } + +private: + using Allocation = jitlink::JITLinkMemoryManager::Allocation; + + struct IndirectStubInfo { + IndirectStubInfo() = default; + IndirectStubInfo(JITTargetAddress StubAddress, + JITTargetAddress PointerAddress) + : StubAddress(StubAddress), PointerAddress(PointerAddress) {} + JITTargetAddress StubAddress = 0; + JITTargetAddress PointerAddress = 0; + }; + + using IndirectStubInfoVector = std::vector; + + /// Create a TPCIndirectionUtils instance. + TPCIndirectionUtils(TargetProcessControl &TPC, + std::unique_ptr ABI); + + Expected getIndirectStubs(unsigned NumStubs); + + std::mutex TPCUIMutex; + TargetProcessControl &TPC; + std::unique_ptr ABI; + JITTargetAddress ResolverBlockAddr; + std::unique_ptr ResolverBlock; + std::unique_ptr TP; + std::unique_ptr LCTM; + + std::vector AvailableIndirectStubs; + std::vector> IndirectStubAllocs; +}; + +namespace detail { + +template +class ABISupportImpl : public TPCIndirectionUtils::ABISupport { +public: + ABISupportImpl() + : ABISupport(ORCABI::PointerSize, ORCABI::TrampolineSize, + ORCABI::StubSize, ORCABI::StubToPointerMaxDisplacement, + ORCABI::ResolverCodeSize) {} + + void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddr, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) const override { + ORCABI::writeResolverCode(ResolverWorkingMem, ResolverTargetAddr, + ReentryFnAddr, ReentryCtxAddr); + } + + void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddr, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines) const override { + ORCABI::writeTrampolines(TrampolineBlockWorkingMem, + TrampolineBlockTargetAddr, ResolverAddr, + NumTrampolines); + } + + void writeIndirectStubsBlock(char *StubsBlockWorkingMem, + JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, + unsigned NumStubs) const override { + ORCABI::writeIndirectStubsBlock(StubsBlockWorkingMem, + StubsBlockTargetAddress, + PointersBlockTargetAddress, NumStubs); + } +}; + +} // end namespace detail + +template +std::unique_ptr +TPCIndirectionUtils::CreateWithABI(TargetProcessControl &TPC) { + return std::unique_ptr(new TPCIndirectionUtils( + TPC, std::make_unique>())); +} + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_T_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h new file mode 100644 index 000000000000..facafd883653 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h @@ -0,0 +1,162 @@ +//===--- TargetProcessControl.h - Target process control APIs ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for interacting with target processes. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESSCONTROL_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESSCONTROL_H + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/Support/MSVCErrorWorkarounds.h" + +#include + +namespace llvm { +namespace orc { + +/// TargetProcessControl supports interaction with a JIT target process. +class TargetProcessControl { +public: + /// APIs for manipulating memory in the target process. + class MemoryAccess { + public: + template struct UIntWrite { + UIntWrite() = default; + UIntWrite(JITTargetAddress Address, T Value) + : Address(Address), Value(Value) {} + + JITTargetAddress Address = 0; + T Value = 0; + }; + + using UInt8Write = UIntWrite; + using UInt16Write = UIntWrite; + using UInt32Write = UIntWrite; + using UInt64Write = UIntWrite; + + struct BufferWrite { + BufferWrite(JITTargetAddress Address, StringRef Buffer) + : Address(Address), Buffer(Buffer) {} + + JITTargetAddress Address = 0; + StringRef Buffer; + }; + + using WriteResultFn = unique_function; + + virtual ~MemoryAccess(); + + virtual void writeUInt8s(ArrayRef Ws, + WriteResultFn OnWriteComplete) = 0; + + virtual void writeUInt16s(ArrayRef Ws, + WriteResultFn OnWriteComplete) = 0; + + virtual void writeUInt32s(ArrayRef Ws, + WriteResultFn OnWriteComplete) = 0; + + virtual void writeUInt64s(ArrayRef Ws, + WriteResultFn OnWriteComplete) = 0; + + virtual void writeBuffers(ArrayRef Ws, + WriteResultFn OnWriteComplete) = 0; + + Error writeUInt8s(ArrayRef Ws) { + std::promise ResultP; + auto ResultF = ResultP.get_future(); + writeUInt8s(Ws, [&](Error Err) { ResultP.set_value(std::move(Err)); }); + return ResultF.get(); + } + + Error writeUInt16s(ArrayRef Ws) { + std::promise ResultP; + auto ResultF = ResultP.get_future(); + writeUInt16s(Ws, [&](Error Err) { ResultP.set_value(std::move(Err)); }); + return ResultF.get(); + } + + Error writeUInt32s(ArrayRef Ws) { + std::promise ResultP; + auto ResultF = ResultP.get_future(); + writeUInt32s(Ws, [&](Error Err) { ResultP.set_value(std::move(Err)); }); + return ResultF.get(); + } + + Error writeUInt64s(ArrayRef Ws) { + std::promise ResultP; + auto ResultF = ResultP.get_future(); + writeUInt64s(Ws, [&](Error Err) { ResultP.set_value(std::move(Err)); }); + return ResultF.get(); + } + + Error writeBuffers(ArrayRef Ws) { + std::promise ResultP; + auto ResultF = ResultP.get_future(); + writeBuffers(Ws, [&](Error Err) { ResultP.set_value(std::move(Err)); }); + return ResultF.get(); + } + }; + + virtual ~TargetProcessControl(); + + /// Return the Triple for the target process. + const Triple &getTargetTriple() const { return TT; } + + /// Get the page size for the target process. + unsigned getPageSize() const { return PageSize; } + + /// Return a JITLinkMemoryManager for the target process. + jitlink::JITLinkMemoryManager &getMemMgr() const { return *MemMgr; } + + /// Return a MemoryAccess object for the target process. + MemoryAccess &getMemoryAccess() const { return *MemAccess; } + +protected: + TargetProcessControl(Triple TT, unsigned PageSize); + + Triple TT; + unsigned PageSize = 0; + jitlink::JITLinkMemoryManager *MemMgr = nullptr; + MemoryAccess *MemAccess = nullptr; +}; + +/// A TargetProcessControl +class SelfTargetProcessControl : public TargetProcessControl, + private TargetProcessControl::MemoryAccess { +public: + SelfTargetProcessControl(Triple TT, unsigned PageSize); + + static Expected> Create(); + +private: + void writeUInt8s(ArrayRef Ws, + WriteResultFn OnWriteComplete) override; + + void writeUInt16s(ArrayRef Ws, + WriteResultFn OnWriteComplete) override; + + void writeUInt32s(ArrayRef Ws, + WriteResultFn OnWriteComplete) override; + + void writeUInt64s(ArrayRef Ws, + WriteResultFn OnWriteComplete) override; + + void writeBuffers(ArrayRef Ws, + WriteResultFn OnWriteComplete) override; + + std::unique_ptr IPMM = + std::make_unique(); +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESSCONTROL_H diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt index 473cf9299523..f9d7924cd5e8 100644 --- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -22,9 +22,11 @@ add_llvm_component_library(LLVMOrcJIT OrcV2CBindings.cpp OrcMCJITReplacement.cpp RTDyldObjectLinkingLayer.cpp - ThreadSafeModule.cpp Speculation.cpp SpeculateAnalyses.cpp + TargetProcessControl.cpp + ThreadSafeModule.cpp + TPCIndirectionUtils.cpp ADDITIONAL_HEADER_DIRS ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc diff --git a/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp new file mode 100644 index 000000000000..51e6928c39e0 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp @@ -0,0 +1,420 @@ +//===------ TargetProcessControl.cpp -- Target process control APIs -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h" + +#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" +#include "llvm/Support/MathExtras.h" + +using namespace llvm; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class TPCIndirectionUtilsAccess { +public: + using IndirectStubInfo = TPCIndirectionUtils::IndirectStubInfo; + using IndirectStubInfoVector = TPCIndirectionUtils::IndirectStubInfoVector; + + static Expected + getIndirectStubs(TPCIndirectionUtils &TPCIU, unsigned NumStubs) { + return TPCIU.getIndirectStubs(NumStubs); + }; +}; + +} // end namespace orc +} // end namespace llvm + +namespace { + +class TPCTrampolinePool : public TrampolinePool { +public: + TPCTrampolinePool(TPCIndirectionUtils &TPCIU); + Error deallocatePool(); + Expected getTrampoline() override; + void releaseTrampoline(JITTargetAddress TrampolineAddr); + +protected: + Error grow(); + + using Allocation = jitlink::JITLinkMemoryManager::Allocation; + + std::mutex TPMutex; + TPCIndirectionUtils &TPCIU; + unsigned TrampolineSize = 0; + unsigned TrampolinesPerPage = 0; + std::vector> TrampolineBlocks; + std::vector AvailableTrampolines; +}; + +class TPCIndirectStubsManager : public IndirectStubsManager, + private TPCIndirectionUtilsAccess { +public: + TPCIndirectStubsManager(TPCIndirectionUtils &TPCIU) : TPCIU(TPCIU) {} + + Error deallocateStubs(); + + Error createStub(StringRef StubName, JITTargetAddress StubAddr, + JITSymbolFlags StubFlags) override; + + Error createStubs(const StubInitsMap &StubInits) override; + + JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; + + JITEvaluatedSymbol findPointer(StringRef Name) override; + + Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; + +private: + using StubInfo = std::pair; + + std::mutex ISMMutex; + TPCIndirectionUtils &TPCIU; + StringMap StubInfos; +}; + +TPCTrampolinePool::TPCTrampolinePool(TPCIndirectionUtils &TPCIU) + : TPCIU(TPCIU) { + auto &TPC = TPCIU.getTargetProcessControl(); + auto &ABI = TPCIU.getABISupport(); + + TrampolineSize = ABI.getTrampolineSize(); + TrampolinesPerPage = + (TPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; +} + +Error TPCTrampolinePool::deallocatePool() { + Error Err = Error::success(); + for (auto &Alloc : TrampolineBlocks) + Err = joinErrors(std::move(Err), Alloc->deallocate()); + return Err; +} + +Expected TPCTrampolinePool::getTrampoline() { + std::lock_guard Lock(TPMutex); + if (AvailableTrampolines.empty()) { + if (auto Err = grow()) + return std::move(Err); + } + + assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool"); + auto TrampolineAddr = AvailableTrampolines.back(); + AvailableTrampolines.pop_back(); + return TrampolineAddr; +} + +void TPCTrampolinePool::releaseTrampoline(JITTargetAddress TrampolineAddr) { + std::lock_guard Lock(TPMutex); + AvailableTrampolines.push_back(TrampolineAddr); +} + +Error TPCTrampolinePool::grow() { + assert(this->AvailableTrampolines.empty() && + "Grow called with trampolines still available"); + + auto ResolverAddress = TPCIU.getResolverBlockAddress(); + assert(ResolverAddress && "Resolver address can not be null"); + + auto &TPC = TPCIU.getTargetProcessControl(); + constexpr auto TrampolinePagePermissions = + static_cast(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + auto PageSize = TPC.getPageSize(); + auto Alloc = TPC.getMemMgr().allocate( + {{TrampolinePagePermissions, {PageSize, PageSize, 0}}}); + + if (!Alloc) + return Alloc.takeError(); + + unsigned NumTrampolines = TrampolinesPerPage; + + auto WorkingMemory = (*Alloc)->getWorkingMemory(TrampolinePagePermissions); + auto TargetAddress = (*Alloc)->getTargetMemory(TrampolinePagePermissions); + + TPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress, + ResolverAddress, NumTrampolines); + + auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions); + for (unsigned I = 0; I < NumTrampolines; ++I) + this->AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize)); + + if (auto Err = (*Alloc)->finalize()) + return Err; + + TrampolineBlocks.push_back(std::move(*Alloc)); + + return Error::success(); +} + +Error TPCIndirectStubsManager::createStub(StringRef StubName, + JITTargetAddress StubAddr, + JITSymbolFlags StubFlags) { + StubInitsMap SIM; + SIM[StubName] = std::make_pair(StubAddr, StubFlags); + return createStubs(SIM); +} + +Error TPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { + auto AvailableStubInfos = getIndirectStubs(TPCIU, StubInits.size()); + if (!AvailableStubInfos) + return AvailableStubInfos.takeError(); + + { + std::lock_guard Lock(ISMMutex); + unsigned ASIdx = 0; + for (auto &SI : StubInits) { + auto &A = (*AvailableStubInfos)[ASIdx++]; + StubInfos[SI.first()] = std::make_pair(A, SI.second.second); + } + } + + auto &MemAccess = TPCIU.getTargetProcessControl().getMemoryAccess(); + switch (TPCIU.getABISupport().getPointerSize()) { + case 4: { + unsigned ASIdx = 0; + std::vector PtrUpdates; + for (auto &SI : StubInits) + PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, + static_cast(SI.second.first)}); + return MemAccess.writeUInt32s(PtrUpdates); + } + case 8: { + unsigned ASIdx = 0; + std::vector PtrUpdates; + for (auto &SI : StubInits) + PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, + static_cast(SI.second.first)}); + return MemAccess.writeUInt64s(PtrUpdates); + } + default: + return make_error("Unsupported pointer size", + inconvertibleErrorCode()); + } +} + +JITEvaluatedSymbol TPCIndirectStubsManager::findStub(StringRef Name, + bool ExportedStubsOnly) { + std::lock_guard Lock(ISMMutex); + auto I = StubInfos.find(Name); + if (I == StubInfos.end()) + return nullptr; + return {I->second.first.StubAddress, I->second.second}; +} + +JITEvaluatedSymbol TPCIndirectStubsManager::findPointer(StringRef Name) { + std::lock_guard Lock(ISMMutex); + auto I = StubInfos.find(Name); + if (I == StubInfos.end()) + return nullptr; + return {I->second.first.PointerAddress, I->second.second}; +} + +Error TPCIndirectStubsManager::updatePointer(StringRef Name, + JITTargetAddress NewAddr) { + + JITTargetAddress PtrAddr = 0; + { + std::lock_guard Lock(ISMMutex); + auto I = StubInfos.find(Name); + if (I == StubInfos.end()) + return make_error("Unknown stub name", + inconvertibleErrorCode()); + PtrAddr = I->second.first.PointerAddress; + } + + auto &MemAccess = TPCIU.getTargetProcessControl().getMemoryAccess(); + switch (TPCIU.getABISupport().getPointerSize()) { + case 4: { + TargetProcessControl::MemoryAccess::UInt32Write PUpdate(PtrAddr, NewAddr); + return MemAccess.writeUInt32s(PUpdate); + } + case 8: { + TargetProcessControl::MemoryAccess::UInt64Write PUpdate(PtrAddr, NewAddr); + return MemAccess.writeUInt64s(PUpdate); + } + default: + return make_error("Unsupported pointer size", + inconvertibleErrorCode()); + } +} + +} // end anonymous namespace. + +namespace llvm { +namespace orc { + +TPCIndirectionUtils::ABISupport::~ABISupport() {} + +Expected> +TPCIndirectionUtils::Create(TargetProcessControl &TPC) { + const auto &TT = TPC.getTargetTriple(); + switch (TT.getArch()) { + default: + return make_error( + std::string("No TPCIndirectionUtils available for ") + TT.str(), + inconvertibleErrorCode()); + case Triple::aarch64: + case Triple::aarch64_32: + return CreateWithABI(TPC); + + case Triple::x86: + return CreateWithABI(TPC); + + case Triple::mips: + return CreateWithABI(TPC); + + case Triple::mipsel: + return CreateWithABI(TPC); + + case Triple::mips64: + case Triple::mips64el: + return CreateWithABI(TPC); + + case Triple::x86_64: + if (TT.getOS() == Triple::OSType::Win32) + return CreateWithABI(TPC); + else + return CreateWithABI(TPC); + } +} + +Error TPCIndirectionUtils::cleanup() { + Error Err = Error::success(); + + for (auto &A : IndirectStubAllocs) + Err = joinErrors(std::move(Err), A->deallocate()); + + if (TP) + Err = joinErrors(std::move(Err), + static_cast(*TP).deallocatePool()); + + if (ResolverBlock) + Err = joinErrors(std::move(Err), ResolverBlock->deallocate()); + + return Err; +} + +Expected +TPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + assert(ABI && "ABI can not be null"); + constexpr auto ResolverBlockPermissions = + static_cast(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + auto ResolverSize = ABI->getResolverCodeSize(); + + auto Alloc = TPC.getMemMgr().allocate( + {{ResolverBlockPermissions, {TPC.getPageSize(), ResolverSize, 0}}}); + if (!Alloc) + return Alloc.takeError(); + + auto WorkingMemory = (*Alloc)->getWorkingMemory(ResolverBlockPermissions); + auto TargetAddress = (*Alloc)->getTargetMemory(ResolverBlockPermissions); + ABI->writeResolverCode(WorkingMemory.data(), TargetAddress, ReentryFnAddr, + ReentryCtxAddr); + + if (auto Err = (*Alloc)->finalize()) + return std::move(Err); + + ResolverBlock = std::move(*Alloc); + ResolverBlockAddr = ResolverBlock->getTargetMemory(ResolverBlockPermissions); + return ResolverBlockAddr; +} + +std::unique_ptr +TPCIndirectionUtils::createIndirectStubsManager() { + return std::make_unique(*this); +} + +TrampolinePool &TPCIndirectionUtils::getTrampolinePool() { + if (!TP) + TP = std::make_unique(*this); + return *TP; +} + +LazyCallThroughManager &TPCIndirectionUtils::createLazyCallThroughManager( + ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { + assert(!LCTM && + "createLazyCallThroughManager can not have been called before"); + LCTM = std::make_unique(ES, ErrorHandlerAddr, + &getTrampolinePool()); + return *LCTM; +} + +TPCIndirectionUtils::TPCIndirectionUtils(TargetProcessControl &TPC, + std::unique_ptr ABI) + : TPC(TPC), ABI(std::move(ABI)) { + assert(this->ABI && "ABI can not be null"); + + assert(TPC.getPageSize() > getABISupport().getStubSize() && + "Stubs larger than one page are not supported"); +} + +Expected +TPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { + + std::lock_guard Lock(TPCUIMutex); + + // If there aren't enough stubs available then allocate some more. + if (NumStubs > AvailableIndirectStubs.size()) { + auto NumStubsToAllocate = NumStubs; + auto PageSize = TPC.getPageSize(); + auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); + NumStubsToAllocate = StubBytes / ABI->getStubSize(); + auto PointerBytes = + alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); + + constexpr auto StubPagePermissions = + static_cast(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + constexpr auto PointerPagePermissions = + static_cast(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + auto Alloc = TPC.getMemMgr().allocate( + {{StubPagePermissions, {PageSize, StubBytes, 0}}, + {PointerPagePermissions, {PageSize, 0, PointerBytes}}}); + + if (!Alloc) + return Alloc.takeError(); + + auto StubTargetAddr = (*Alloc)->getTargetMemory(StubPagePermissions); + auto PointerTargetAddr = (*Alloc)->getTargetMemory(PointerPagePermissions); + + ABI->writeIndirectStubsBlock( + (*Alloc)->getWorkingMemory(StubPagePermissions).data(), StubTargetAddr, + PointerTargetAddr, NumStubsToAllocate); + + if (auto Err = (*Alloc)->finalize()) + return std::move(Err); + + for (unsigned I = 0; I != NumStubsToAllocate; ++I) { + AvailableIndirectStubs.push_back( + IndirectStubInfo(StubTargetAddr, PointerTargetAddr)); + StubTargetAddr += ABI->getStubSize(); + PointerTargetAddr += ABI->getPointerSize(); + } + + IndirectStubAllocs.push_back(std::move(*Alloc)); + } + + assert(NumStubs <= AvailableIndirectStubs.size() && + "Sufficient stubs should have been allocated above"); + + IndirectStubInfoVector Result; + while (NumStubs--) { + Result.push_back(AvailableIndirectStubs.back()); + AvailableIndirectStubs.pop_back(); + } + + return std::move(Result); +} + +} // end namespace orc +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp new file mode 100644 index 000000000000..833b597fe712 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp @@ -0,0 +1,79 @@ +//===------ TargetProcessControl.cpp -- Target process control APIs -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/Process.h" + +#include + +namespace llvm { +namespace orc { + +TargetProcessControl::MemoryAccess::~MemoryAccess() {} + +TargetProcessControl::TargetProcessControl(Triple TT, unsigned PageSize) + : TT(std::move(TT)), PageSize(PageSize) {} + +TargetProcessControl::~TargetProcessControl() {} + +SelfTargetProcessControl::SelfTargetProcessControl(Triple TT, unsigned PageSize) + : TargetProcessControl(std::move(TT), PageSize) { + this->MemMgr = IPMM.get(); + this->MemAccess = this; +} + +Expected> +SelfTargetProcessControl::Create() { + auto PageSize = sys::Process::getPageSize(); + if (!PageSize) + return PageSize.takeError(); + + Triple TT(sys::getProcessTriple()); + + return std::make_unique(std::move(TT), *PageSize); +} + +void SelfTargetProcessControl::writeUInt8s(ArrayRef Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeUInt16s(ArrayRef Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeUInt32s(ArrayRef Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeUInt64s(ArrayRef Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeBuffers(ArrayRef Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + memcpy(jitTargetAddressToPointer(W.Address), W.Buffer.data(), + W.Buffer.size()); + OnWriteComplete(Error::success()); +} + +} // end namespace orc +} // end namespace llvm