forked from OSchip/llvm-project
535 lines
18 KiB
C++
535 lines
18 KiB
C++
//===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
|
|
#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
|
|
|
|
#include "llvm-c/OrcBindings.h"
|
|
#include "llvm-c/TargetMachine.h"
|
|
#include "llvm/ADT/STLExtras.h"
|
|
#include "llvm/ADT/StringRef.h"
|
|
#include "llvm/ExecutionEngine/JITSymbol.h"
|
|
#include "llvm/ExecutionEngine/JITEventListener.h"
|
|
#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
|
|
#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
|
|
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
|
|
#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
|
|
#include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
|
|
#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
|
|
#include "llvm/ExecutionEngine/RuntimeDyld.h"
|
|
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
|
|
#include "llvm/IR/DataLayout.h"
|
|
#include "llvm/IR/Mangler.h"
|
|
#include "llvm/IR/Module.h"
|
|
#include "llvm/Support/CBindingWrapping.h"
|
|
#include "llvm/Support/Error.h"
|
|
#include "llvm/Support/raw_ostream.h"
|
|
#include "llvm/Target/TargetMachine.h"
|
|
#include <algorithm>
|
|
#include <cstdint>
|
|
#include <functional>
|
|
#include <map>
|
|
#include <memory>
|
|
#include <set>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
namespace llvm {
|
|
|
|
class OrcCBindingsStack;
|
|
|
|
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef)
|
|
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
|
|
|
|
namespace detail {
|
|
|
|
// FIXME: Kill this off once the Layer concept becomes an interface.
|
|
class GenericLayer {
|
|
public:
|
|
virtual ~GenericLayer() = default;
|
|
|
|
virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
|
|
bool ExportedSymbolsOnly) = 0;
|
|
virtual Error removeModule(orc::VModuleKey K) = 0;
|
|
};
|
|
|
|
template <typename LayerT> class GenericLayerImpl : public GenericLayer {
|
|
public:
|
|
GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
|
|
|
|
JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
|
|
bool ExportedSymbolsOnly) override {
|
|
return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
|
|
}
|
|
|
|
Error removeModule(orc::VModuleKey K) override {
|
|
return Layer.removeModule(K);
|
|
}
|
|
|
|
private:
|
|
LayerT &Layer;
|
|
};
|
|
|
|
template <>
|
|
class GenericLayerImpl<orc::LegacyRTDyldObjectLinkingLayer> : public GenericLayer {
|
|
private:
|
|
using LayerT = orc::LegacyRTDyldObjectLinkingLayer;
|
|
public:
|
|
GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
|
|
|
|
JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
|
|
bool ExportedSymbolsOnly) override {
|
|
return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
|
|
}
|
|
|
|
Error removeModule(orc::VModuleKey K) override {
|
|
return Layer.removeObject(K);
|
|
}
|
|
|
|
private:
|
|
LayerT &Layer;
|
|
};
|
|
|
|
template <typename LayerT>
|
|
std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) {
|
|
return std::make_unique<GenericLayerImpl<LayerT>>(Layer);
|
|
}
|
|
|
|
} // end namespace detail
|
|
|
|
class OrcCBindingsStack {
|
|
public:
|
|
|
|
using CompileCallbackMgr = orc::JITCompileCallbackManager;
|
|
using ObjLayerT = orc::LegacyRTDyldObjectLinkingLayer;
|
|
using CompileLayerT = orc::LegacyIRCompileLayer<ObjLayerT, orc::SimpleCompiler>;
|
|
using CODLayerT =
|
|
orc::LegacyCompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>;
|
|
|
|
using CallbackManagerBuilder =
|
|
std::function<std::unique_ptr<CompileCallbackMgr>()>;
|
|
|
|
using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT;
|
|
|
|
private:
|
|
|
|
using OwningObject = object::OwningBinary<object::ObjectFile>;
|
|
|
|
class CBindingsResolver : public orc::SymbolResolver {
|
|
public:
|
|
CBindingsResolver(OrcCBindingsStack &Stack,
|
|
LLVMOrcSymbolResolverFn ExternalResolver,
|
|
void *ExternalResolverCtx)
|
|
: Stack(Stack), ExternalResolver(std::move(ExternalResolver)),
|
|
ExternalResolverCtx(std::move(ExternalResolverCtx)) {}
|
|
|
|
orc::SymbolNameSet
|
|
getResponsibilitySet(const orc::SymbolNameSet &Symbols) override {
|
|
orc::SymbolNameSet Result;
|
|
|
|
for (auto &S : Symbols) {
|
|
if (auto Sym = findSymbol(std::string(*S))) {
|
|
if (!Sym.getFlags().isStrong())
|
|
Result.insert(S);
|
|
} else if (auto Err = Sym.takeError()) {
|
|
Stack.reportError(std::move(Err));
|
|
return orc::SymbolNameSet();
|
|
}
|
|
}
|
|
|
|
return Result;
|
|
}
|
|
|
|
orc::SymbolNameSet
|
|
lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,
|
|
orc::SymbolNameSet Symbols) override {
|
|
orc::SymbolNameSet UnresolvedSymbols;
|
|
|
|
for (auto &S : Symbols) {
|
|
if (auto Sym = findSymbol(std::string(*S))) {
|
|
if (auto Addr = Sym.getAddress()) {
|
|
Query->notifySymbolMetRequiredState(
|
|
S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
|
|
} else {
|
|
Stack.ES.legacyFailQuery(*Query, Addr.takeError());
|
|
return orc::SymbolNameSet();
|
|
}
|
|
} else if (auto Err = Sym.takeError()) {
|
|
Stack.ES.legacyFailQuery(*Query, std::move(Err));
|
|
return orc::SymbolNameSet();
|
|
} else
|
|
UnresolvedSymbols.insert(S);
|
|
}
|
|
|
|
if (Query->isComplete())
|
|
Query->handleComplete();
|
|
|
|
return UnresolvedSymbols;
|
|
}
|
|
|
|
private:
|
|
JITSymbol findSymbol(const std::string &Name) {
|
|
// Search order:
|
|
// 1. JIT'd symbols.
|
|
// 2. Runtime overrides.
|
|
// 3. External resolver (if present).
|
|
|
|
if (Stack.CODLayer) {
|
|
if (auto Sym = Stack.CODLayer->findSymbol(Name, true))
|
|
return Sym;
|
|
else if (auto Err = Sym.takeError())
|
|
return Sym.takeError();
|
|
} else {
|
|
if (auto Sym = Stack.CompileLayer.findSymbol(Name, true))
|
|
return Sym;
|
|
else if (auto Err = Sym.takeError())
|
|
return Sym.takeError();
|
|
}
|
|
|
|
if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name))
|
|
return Sym;
|
|
|
|
if (ExternalResolver)
|
|
return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx),
|
|
JITSymbolFlags::Exported);
|
|
|
|
return JITSymbol(nullptr);
|
|
}
|
|
|
|
OrcCBindingsStack &Stack;
|
|
LLVMOrcSymbolResolverFn ExternalResolver;
|
|
void *ExternalResolverCtx = nullptr;
|
|
};
|
|
|
|
public:
|
|
OrcCBindingsStack(TargetMachine &TM,
|
|
IndirectStubsManagerBuilder IndirectStubsMgrBuilder)
|
|
: CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()),
|
|
IndirectStubsMgr(IndirectStubsMgrBuilder()),
|
|
ObjectLayer(
|
|
AcknowledgeORCv1Deprecation, ES,
|
|
[this](orc::VModuleKey K) {
|
|
auto ResolverI = Resolvers.find(K);
|
|
assert(ResolverI != Resolvers.end() &&
|
|
"No resolver for module K");
|
|
auto Resolver = std::move(ResolverI->second);
|
|
Resolvers.erase(ResolverI);
|
|
return ObjLayerT::Resources{
|
|
std::make_shared<SectionMemoryManager>(), Resolver};
|
|
},
|
|
nullptr,
|
|
[this](orc::VModuleKey K, const object::ObjectFile &Obj,
|
|
const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
|
|
this->notifyFinalized(K, Obj, LoadedObjInfo);
|
|
},
|
|
[this](orc::VModuleKey K, const object::ObjectFile &Obj) {
|
|
this->notifyFreed(K, Obj);
|
|
}),
|
|
CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer,
|
|
orc::SimpleCompiler(TM)),
|
|
CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(),
|
|
std::move(IndirectStubsMgrBuilder), Resolvers)),
|
|
CXXRuntimeOverrides(
|
|
AcknowledgeORCv1Deprecation,
|
|
[this](const std::string &S) { return mangle(S); }) {}
|
|
|
|
Error shutdown() {
|
|
// Run any destructors registered with __cxa_atexit.
|
|
CXXRuntimeOverrides.runDestructors();
|
|
// Run any IR destructors.
|
|
for (auto &DtorRunner : IRStaticDestructorRunners)
|
|
if (auto Err = DtorRunner.runViaLayer(*this))
|
|
return Err;
|
|
return Error::success();
|
|
}
|
|
|
|
std::string mangle(StringRef Name) {
|
|
std::string MangledName;
|
|
{
|
|
raw_string_ostream MangledNameStream(MangledName);
|
|
Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
|
|
}
|
|
return MangledName;
|
|
}
|
|
|
|
template <typename PtrTy>
|
|
static PtrTy fromTargetAddress(JITTargetAddress Addr) {
|
|
return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr));
|
|
}
|
|
|
|
Expected<JITTargetAddress>
|
|
createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback,
|
|
void *CallbackCtx) {
|
|
auto WrappedCallback = [=]() -> JITTargetAddress {
|
|
return Callback(wrap(this), CallbackCtx);
|
|
};
|
|
|
|
return CCMgr->getCompileCallback(std::move(WrappedCallback));
|
|
}
|
|
|
|
Error createIndirectStub(StringRef StubName, JITTargetAddress Addr) {
|
|
return IndirectStubsMgr->createStub(StubName, Addr,
|
|
JITSymbolFlags::Exported);
|
|
}
|
|
|
|
Error setIndirectStubPointer(StringRef Name, JITTargetAddress Addr) {
|
|
return IndirectStubsMgr->updatePointer(Name, Addr);
|
|
}
|
|
|
|
template <typename LayerT>
|
|
Expected<orc::VModuleKey>
|
|
addIRModule(LayerT &Layer, std::unique_ptr<Module> M,
|
|
std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,
|
|
LLVMOrcSymbolResolverFn ExternalResolver,
|
|
void *ExternalResolverCtx) {
|
|
|
|
// Attach a data-layout if one isn't already present.
|
|
if (M->getDataLayout().isDefault())
|
|
M->setDataLayout(DL);
|
|
|
|
// Record the static constructors and destructors. We have to do this before
|
|
// we hand over ownership of the module to the JIT.
|
|
std::vector<std::string> CtorNames, DtorNames;
|
|
for (auto Ctor : orc::getConstructors(*M))
|
|
CtorNames.push_back(mangle(Ctor.Func->getName()));
|
|
for (auto Dtor : orc::getDestructors(*M))
|
|
DtorNames.push_back(mangle(Dtor.Func->getName()));
|
|
|
|
// Add the module to the JIT.
|
|
auto K = ES.allocateVModule();
|
|
Resolvers[K] = std::make_shared<CBindingsResolver>(*this, ExternalResolver,
|
|
ExternalResolverCtx);
|
|
if (auto Err = Layer.addModule(K, std::move(M)))
|
|
return std::move(Err);
|
|
|
|
KeyLayers[K] = detail::createGenericLayer(Layer);
|
|
|
|
// Run the static constructors, and save the static destructor runner for
|
|
// execution when the JIT is torn down.
|
|
orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner(
|
|
AcknowledgeORCv1Deprecation, std::move(CtorNames), K);
|
|
if (auto Err = CtorRunner.runViaLayer(*this))
|
|
return std::move(Err);
|
|
|
|
IRStaticDestructorRunners.emplace_back(AcknowledgeORCv1Deprecation,
|
|
std::move(DtorNames), K);
|
|
|
|
return K;
|
|
}
|
|
|
|
Expected<orc::VModuleKey>
|
|
addIRModuleEager(std::unique_ptr<Module> M,
|
|
LLVMOrcSymbolResolverFn ExternalResolver,
|
|
void *ExternalResolverCtx) {
|
|
return addIRModule(CompileLayer, std::move(M),
|
|
std::make_unique<SectionMemoryManager>(),
|
|
std::move(ExternalResolver), ExternalResolverCtx);
|
|
}
|
|
|
|
Expected<orc::VModuleKey>
|
|
addIRModuleLazy(std::unique_ptr<Module> M,
|
|
LLVMOrcSymbolResolverFn ExternalResolver,
|
|
void *ExternalResolverCtx) {
|
|
if (!CODLayer)
|
|
return make_error<StringError>("Can not add lazy module: No compile "
|
|
"callback manager available",
|
|
inconvertibleErrorCode());
|
|
|
|
return addIRModule(*CODLayer, std::move(M),
|
|
std::make_unique<SectionMemoryManager>(),
|
|
std::move(ExternalResolver), ExternalResolverCtx);
|
|
}
|
|
|
|
Error removeModule(orc::VModuleKey K) {
|
|
// FIXME: Should error release the module key?
|
|
if (auto Err = KeyLayers[K]->removeModule(K))
|
|
return Err;
|
|
ES.releaseVModule(K);
|
|
KeyLayers.erase(K);
|
|
return Error::success();
|
|
}
|
|
|
|
Expected<orc::VModuleKey> addObject(std::unique_ptr<MemoryBuffer> ObjBuffer,
|
|
LLVMOrcSymbolResolverFn ExternalResolver,
|
|
void *ExternalResolverCtx) {
|
|
if (auto Obj = object::ObjectFile::createObjectFile(
|
|
ObjBuffer->getMemBufferRef())) {
|
|
|
|
auto K = ES.allocateVModule();
|
|
Resolvers[K] = std::make_shared<CBindingsResolver>(
|
|
*this, ExternalResolver, ExternalResolverCtx);
|
|
|
|
if (auto Err = ObjectLayer.addObject(K, std::move(ObjBuffer)))
|
|
return std::move(Err);
|
|
|
|
KeyLayers[K] = detail::createGenericLayer(ObjectLayer);
|
|
|
|
return K;
|
|
} else
|
|
return Obj.takeError();
|
|
}
|
|
|
|
JITSymbol findSymbol(const std::string &Name,
|
|
bool ExportedSymbolsOnly) {
|
|
if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly))
|
|
return Sym;
|
|
if (CODLayer)
|
|
return CODLayer->findSymbol(mangle(Name), ExportedSymbolsOnly);
|
|
return CompileLayer.findSymbol(mangle(Name), ExportedSymbolsOnly);
|
|
}
|
|
|
|
JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
|
|
bool ExportedSymbolsOnly) {
|
|
assert(KeyLayers.count(K) && "looking up symbol in unknown module");
|
|
return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly);
|
|
}
|
|
|
|
Expected<JITTargetAddress> findSymbolAddress(const std::string &Name,
|
|
bool ExportedSymbolsOnly) {
|
|
if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) {
|
|
// Successful lookup, non-null symbol:
|
|
if (auto AddrOrErr = Sym.getAddress())
|
|
return *AddrOrErr;
|
|
else
|
|
return AddrOrErr.takeError();
|
|
} else if (auto Err = Sym.takeError()) {
|
|
// Lookup failure - report error.
|
|
return std::move(Err);
|
|
}
|
|
|
|
// No symbol not found. Return 0.
|
|
return 0;
|
|
}
|
|
|
|
Expected<JITTargetAddress> findSymbolAddressIn(orc::VModuleKey K,
|
|
const std::string &Name,
|
|
bool ExportedSymbolsOnly) {
|
|
if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) {
|
|
// Successful lookup, non-null symbol:
|
|
if (auto AddrOrErr = Sym.getAddress())
|
|
return *AddrOrErr;
|
|
else
|
|
return AddrOrErr.takeError();
|
|
} else if (auto Err = Sym.takeError()) {
|
|
// Lookup failure - report error.
|
|
return std::move(Err);
|
|
}
|
|
|
|
// Symbol not found. Return 0.
|
|
return 0;
|
|
}
|
|
|
|
const std::string &getErrorMessage() const { return ErrMsg; }
|
|
|
|
void RegisterJITEventListener(JITEventListener *L) {
|
|
if (!L)
|
|
return;
|
|
EventListeners.push_back(L);
|
|
}
|
|
|
|
void UnregisterJITEventListener(JITEventListener *L) {
|
|
if (!L)
|
|
return;
|
|
|
|
auto I = find(reverse(EventListeners), L);
|
|
if (I != EventListeners.rend()) {
|
|
std::swap(*I, EventListeners.back());
|
|
EventListeners.pop_back();
|
|
}
|
|
}
|
|
|
|
private:
|
|
using ResolverMap =
|
|
std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>>;
|
|
|
|
static std::unique_ptr<CompileCallbackMgr>
|
|
createCompileCallbackManager(TargetMachine &TM, orc::ExecutionSession &ES) {
|
|
auto CCMgr = createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0);
|
|
if (!CCMgr) {
|
|
// FIXME: It would be good if we could report this somewhere, but we do
|
|
// have an instance yet.
|
|
logAllUnhandledErrors(CCMgr.takeError(), errs(), "ORC error: ");
|
|
return nullptr;
|
|
}
|
|
return std::move(*CCMgr);
|
|
}
|
|
|
|
static std::unique_ptr<CODLayerT>
|
|
createCODLayer(orc::ExecutionSession &ES, CompileLayerT &CompileLayer,
|
|
CompileCallbackMgr *CCMgr,
|
|
IndirectStubsManagerBuilder IndirectStubsMgrBuilder,
|
|
ResolverMap &Resolvers) {
|
|
// If there is no compile callback manager available we can not create a
|
|
// compile on demand layer.
|
|
if (!CCMgr)
|
|
return nullptr;
|
|
|
|
return std::make_unique<CODLayerT>(
|
|
AcknowledgeORCv1Deprecation, ES, CompileLayer,
|
|
[&Resolvers](orc::VModuleKey K) {
|
|
auto ResolverI = Resolvers.find(K);
|
|
assert(ResolverI != Resolvers.end() && "No resolver for module K");
|
|
return ResolverI->second;
|
|
},
|
|
[&Resolvers](orc::VModuleKey K,
|
|
std::shared_ptr<orc::SymbolResolver> Resolver) {
|
|
assert(!Resolvers.count(K) && "Resolver already present");
|
|
Resolvers[K] = std::move(Resolver);
|
|
},
|
|
[](Function &F) { return std::set<Function *>({&F}); }, *CCMgr,
|
|
std::move(IndirectStubsMgrBuilder), false);
|
|
}
|
|
|
|
void reportError(Error Err) {
|
|
// FIXME: Report errors on the execution session.
|
|
logAllUnhandledErrors(std::move(Err), errs(), "ORC error: ");
|
|
};
|
|
|
|
void notifyFinalized(orc::VModuleKey K,
|
|
const object::ObjectFile &Obj,
|
|
const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
|
|
uint64_t Key = static_cast<uint64_t>(
|
|
reinterpret_cast<uintptr_t>(Obj.getData().data()));
|
|
for (auto &Listener : EventListeners)
|
|
Listener->notifyObjectLoaded(Key, Obj, LoadedObjInfo);
|
|
}
|
|
|
|
void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) {
|
|
uint64_t Key = static_cast<uint64_t>(
|
|
reinterpret_cast<uintptr_t>(Obj.getData().data()));
|
|
for (auto &Listener : EventListeners)
|
|
Listener->notifyFreeingObject(Key);
|
|
}
|
|
|
|
orc::ExecutionSession ES;
|
|
std::unique_ptr<CompileCallbackMgr> CCMgr;
|
|
|
|
std::vector<JITEventListener *> EventListeners;
|
|
|
|
DataLayout DL;
|
|
SectionMemoryManager CCMgrMemMgr;
|
|
|
|
std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr;
|
|
|
|
ObjLayerT ObjectLayer;
|
|
CompileLayerT CompileLayer;
|
|
std::unique_ptr<CODLayerT> CODLayer;
|
|
|
|
std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers;
|
|
|
|
orc::LegacyLocalCXXRuntimeOverrides CXXRuntimeOverrides;
|
|
std::vector<orc::LegacyCtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners;
|
|
std::string ErrMsg;
|
|
|
|
ResolverMap Resolvers;
|
|
};
|
|
|
|
} // end namespace llvm
|
|
|
|
#endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
|