From 1b023a319e335592bebee48c585858793db87c35 Mon Sep 17 00:00:00 2001 From: Rafael Espindola Date: Tue, 3 Nov 2015 16:25:20 +0000 Subject: [PATCH] Revert "[Orc] Directly emit machine code for the x86 resolver block and trampolines." This reverts commit r251933. It broke the build of examples/Kaleidoscope/Orc/fully_lazy/toy.cpp. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@251937 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm-c/OrcBindings.h | 3 +- .../Orc/CompileOnDemandLayer.h | 2 +- .../ExecutionEngine/Orc/IndirectionUtils.h | 148 +++++++------ .../ExecutionEngine/Orc/OrcTargetSupport.h | 35 ++-- lib/ExecutionEngine/Orc/OrcCBindings.cpp | 10 +- lib/ExecutionEngine/Orc/OrcCBindingsStack.cpp | 13 +- lib/ExecutionEngine/Orc/OrcCBindingsStack.h | 21 +- lib/ExecutionEngine/Orc/OrcTargetSupport.cpp | 196 +++++++++++------- tools/lli/OrcLazyJIT.cpp | 22 +- tools/lli/OrcLazyJIT.h | 23 +- .../Orc/CompileOnDemandLayerTest.cpp | 4 +- unittests/ExecutionEngine/Orc/OrcCAPITest.cpp | 6 +- 12 files changed, 289 insertions(+), 194 deletions(-) diff --git a/include/llvm-c/OrcBindings.h b/include/llvm-c/OrcBindings.h index f6aff916999..78a7bc146ee 100644 --- a/include/llvm-c/OrcBindings.h +++ b/include/llvm-c/OrcBindings.h @@ -47,7 +47,8 @@ typedef uint64_t (*LLVMOrcLazyCompileCallbackFn)(LLVMOrcJITStackRef JITStack, * client should not attempt to dispose of the Target Machine, or it will result * in a double-free. */ -LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM); +LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM, + LLVMContextRef Context); /** * Mangle the given symbol. diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 8f4e844032c..9c37aa01681 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -242,7 +242,7 @@ private: // Create a callback, associate it with the stub for the function, // and set the compile action to compile the partition containing the // function. - auto CCInfo = CompileCallbackMgr.getCompileCallback(); + auto CCInfo = CompileCallbackMgr.getCompileCallback(SrcM.getContext()); StubInits[mangle(F.getName(), DL)] = std::make_pair(CCInfo.getAddress(), JITSymbolBase::flagsFromGlobalValue(F)); diff --git a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index cabc95543d8..5ed8935980f 100644 --- a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -27,7 +27,8 @@ namespace llvm { namespace orc { -/// @brief Target-independent base class JITCompileCallbackManager. +/// @brief Base class for JITLayer independent aspects of +/// JITCompileCallbackManager. class JITCompileCallbackManagerBase { public: @@ -53,8 +54,13 @@ public: /// @brief Construct a JITCompileCallbackManagerBase. /// @param ErrorHandlerAddress The address of an error handler in the target /// process to be used if a compile callback fails. - JITCompileCallbackManagerBase(TargetAddress ErrorHandlerAddress) - : ErrorHandlerAddress(ErrorHandlerAddress) {} + /// @param NumTrampolinesPerBlock Number of trampolines to emit if there is no + /// available trampoline when getCompileCallback is + /// called. + JITCompileCallbackManagerBase(TargetAddress ErrorHandlerAddress, + unsigned NumTrampolinesPerBlock) + : ErrorHandlerAddress(ErrorHandlerAddress), + NumTrampolinesPerBlock(NumTrampolinesPerBlock) {} virtual ~JITCompileCallbackManagerBase() {} @@ -84,7 +90,7 @@ public: } /// @brief Reserve a compile callback. - virtual CompileCallbackInfo getCompileCallback() = 0; + virtual CompileCallbackInfo getCompileCallback(LLVMContext &Context) = 0; /// @brief Get a CompileCallbackInfo for an existing callback. CompileCallbackInfo getCompileCallbackInfo(TargetAddress TrampolineAddr) { @@ -107,6 +113,7 @@ public: protected: TargetAddress ErrorHandlerAddress; + unsigned NumTrampolinesPerBlock; typedef std::map TrampolineMapT; TrampolineMapT ActiveTrampolines; @@ -117,54 +124,69 @@ private: }; /// @brief Manage compile callbacks. -template +template class JITCompileCallbackManager : public JITCompileCallbackManagerBase { public: /// @brief Construct a JITCompileCallbackManager. + /// @param JIT JIT layer to emit callback trampolines, etc. into. + /// @param Context LLVMContext to use for trampoline & resolve block modules. /// @param ErrorHandlerAddress The address of an error handler in the target /// process to be used if a compile callback fails. - JITCompileCallbackManager(TargetAddress ErrorHandlerAddress) - : JITCompileCallbackManagerBase(ErrorHandlerAddress) { - - /// Set up the resolver block. - std::error_code EC; - ResolverBlock = - sys::OwningMemoryBlock( - sys::Memory::allocateMappedMemory(TargetT::ResolverCodeSize, nullptr, - sys::Memory::MF_READ | - sys::Memory::MF_WRITE, EC)); - assert(!EC && "Failed to allocate resolver block"); - - TargetT::writeResolverCode(static_cast(ResolverBlock.base()), - &reenter, this); - - EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(), - sys::Memory::MF_READ | - sys::Memory::MF_EXEC); - assert(!EC && "Failed to mprotect resolver block"); + /// @param NumTrampolinesPerBlock Number of trampolines to allocate whenever + /// there is no existing callback trampoline. + /// (Trampolines are allocated in blocks for + /// efficiency.) + JITCompileCallbackManager(JITLayerT &JIT, RuntimeDyld::MemoryManager &MemMgr, + LLVMContext &Context, + TargetAddress ErrorHandlerAddress, + unsigned NumTrampolinesPerBlock) + : JITCompileCallbackManagerBase(ErrorHandlerAddress, + NumTrampolinesPerBlock), + JIT(JIT), MemMgr(MemMgr) { + emitResolverBlock(Context); } /// @brief Get/create a compile callback with the given signature. - CompileCallbackInfo getCompileCallback() final { - TargetAddress TrampolineAddr = getAvailableTrampolineAddr(); + CompileCallbackInfo getCompileCallback(LLVMContext &Context) final { + TargetAddress TrampolineAddr = getAvailableTrampolineAddr(Context); auto &Compile = this->ActiveTrampolines[TrampolineAddr]; return CompileCallbackInfo(TrampolineAddr, Compile); } private: - static TargetAddress reenter(void *CCMgr, void *TrampolineId) { - JITCompileCallbackManager *Mgr = - static_cast(CCMgr); - return Mgr->executeCompileCallback( - static_cast( - reinterpret_cast(TrampolineId))); + std::vector> + SingletonSet(std::unique_ptr M) { + std::vector> Ms; + Ms.push_back(std::move(M)); + return Ms; + } + + void emitResolverBlock(LLVMContext &Context) { + std::unique_ptr M(new Module("resolver_block_module", + Context)); + TargetT::insertResolverBlock(*M, *this); + auto NonResolver = + createLambdaResolver( + [](const std::string &Name) -> RuntimeDyld::SymbolInfo { + llvm_unreachable("External symbols in resolver block?"); + }, + [](const std::string &Name) -> RuntimeDyld::SymbolInfo { + llvm_unreachable("Dylib symbols in resolver block?"); + }); + auto H = JIT.addModuleSet(SingletonSet(std::move(M)), &MemMgr, + std::move(NonResolver)); + JIT.emitAndFinalize(H); + auto ResolverBlockSymbol = + JIT.findSymbolIn(H, TargetT::ResolverBlockName, false); + assert(ResolverBlockSymbol && "Failed to insert resolver block"); + ResolverBlockAddr = ResolverBlockSymbol.getAddress(); } - TargetAddress getAvailableTrampolineAddr() { + TargetAddress getAvailableTrampolineAddr(LLVMContext &Context) { if (this->AvailableTrampolines.empty()) - grow(); + grow(Context); assert(!this->AvailableTrampolines.empty() && "Failed to grow available trampolines."); TargetAddress TrampolineAddr = this->AvailableTrampolines.back(); @@ -172,41 +194,35 @@ private: return TrampolineAddr; } - void grow() { + void grow(LLVMContext &Context) { assert(this->AvailableTrampolines.empty() && "Growing prematurely?"); - - std::error_code EC; - auto TrampolineBlock = - sys::OwningMemoryBlock( - sys::Memory::allocateMappedMemory(TargetT::PageSize, nullptr, - sys::Memory::MF_READ | - sys::Memory::MF_WRITE, EC)); - assert(!EC && "Failed to allocate trampoline block"); - - - unsigned NumTrampolines = - (TargetT::PageSize - TargetT::PointerSize) / TargetT::TrampolineSize; - - uint8_t *TrampolineMem = static_cast(TrampolineBlock.base()); - TargetT::writeTrampolines(TrampolineMem, ResolverBlock.base(), - NumTrampolines); - - for (unsigned I = 0; I < NumTrampolines; ++I) - this->AvailableTrampolines.push_back( - static_cast( - reinterpret_cast( - TrampolineMem + (I * TargetT::TrampolineSize)))); - - EC = sys::Memory::protectMappedMemory(TrampolineBlock.getMemoryBlock(), - sys::Memory::MF_READ | - sys::Memory::MF_EXEC); - assert(!EC && "Failed to mprotect trampoline block"); - - TrampolineBlocks.push_back(std::move(TrampolineBlock)); + std::unique_ptr M(new Module("trampoline_block", Context)); + auto GetLabelName = + TargetT::insertCompileCallbackTrampolines(*M, ResolverBlockAddr, + this->NumTrampolinesPerBlock, + this->ActiveTrampolines.size()); + auto NonResolver = + createLambdaResolver( + [](const std::string &Name) -> RuntimeDyld::SymbolInfo { + llvm_unreachable("External symbols in trampoline block?"); + }, + [](const std::string &Name) -> RuntimeDyld::SymbolInfo { + llvm_unreachable("Dylib symbols in trampoline block?"); + }); + auto H = JIT.addModuleSet(SingletonSet(std::move(M)), &MemMgr, + std::move(NonResolver)); + JIT.emitAndFinalize(H); + for (unsigned I = 0; I < this->NumTrampolinesPerBlock; ++I) { + std::string Name = GetLabelName(I); + auto TrampolineSymbol = JIT.findSymbolIn(H, Name, false); + assert(TrampolineSymbol && "Failed to emit trampoline."); + this->AvailableTrampolines.push_back(TrampolineSymbol.getAddress()); + } } - sys::OwningMemoryBlock ResolverBlock; - std::vector TrampolineBlocks; + JITLayerT &JIT; + RuntimeDyld::MemoryManager &MemMgr; + TargetAddress ResolverBlockAddr; }; /// @brief Base class for managing collections of named indirect stubs. diff --git a/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h b/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h index 3e57314ba2d..58273ae4616 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h +++ b/include/llvm/ExecutionEngine/Orc/OrcTargetSupport.h @@ -26,24 +26,29 @@ namespace orc { class OrcX86_64 { public: - static const unsigned PageSize = 4096; - static const unsigned PointerSize = 8; - static const unsigned TrampolineSize = 8; - static const unsigned ResolverCodeSize = 0x78; + static const char *ResolverBlockName; - typedef TargetAddress (*JITReentryFn)(void *CallbackMgr, - void *TrampolineId); + /// @brief Insert module-level inline callback asm into module M for the + /// symbols managed by JITResolveCallbackHandler J. + static void insertResolverBlock(Module &M, + JITCompileCallbackManagerBase &JCBM); - /// @brief Write the resolver code into the given memory. The user is be - /// responsible for allocating the memory and setting permissions. - static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, - void *CallbackMgr); + /// @brief Get a label name from the given index. + typedef std::function LabelNameFtor; - /// @brief Write the requsted number of trampolines into the given memory, - /// which must be big enough to hold 1 pointer, plus NumTrampolines - /// trampolines. - static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, - unsigned NumTrampolines); + /// @brief Insert the requested number of trampolines into the given module. + /// @param M Module to insert the call block into. + /// @param NumCalls Number of calls to create in the call block. + /// @param StartIndex Optional argument specifying the index suffix to start + /// with. + /// @return A functor that provides the symbol name for each entry in the call + /// block. + /// + static LabelNameFtor insertCompileCallbackTrampolines( + Module &M, + TargetAddress TrampolineAddr, + unsigned NumCalls, + unsigned StartIndex = 0); /// @brief Provide information about stub blocks generated by the /// makeIndirectStubsBlock function. diff --git a/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/lib/ExecutionEngine/Orc/OrcCBindings.cpp index d2379cd441d..d3effeec97e 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindings.cpp +++ b/lib/ExecutionEngine/Orc/OrcCBindings.cpp @@ -12,18 +12,20 @@ using namespace llvm; -LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM) { +LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM, + LLVMContextRef Context) { TargetMachine *TM2(unwrap(TM)); + LLVMContext &Ctx = *unwrap(Context); Triple T(TM2->getTargetTriple()); - auto CompileCallbackMgr = OrcCBindingsStack::createCompileCallbackMgr(T); + auto CallbackMgrBuilder = OrcCBindingsStack::createCallbackManagerBuilder(T); auto IndirectStubsMgrBuilder = OrcCBindingsStack::createIndirectStubsMgrBuilder(T); OrcCBindingsStack *JITStack = - new OrcCBindingsStack(*TM2, std::move(CompileCallbackMgr), - IndirectStubsMgrBuilder); + new OrcCBindingsStack(*TM2, Ctx, CallbackMgrBuilder, + IndirectStubsMgrBuilder); return wrap(JITStack); } diff --git a/lib/ExecutionEngine/Orc/OrcCBindingsStack.cpp b/lib/ExecutionEngine/Orc/OrcCBindingsStack.cpp index 7326fa7e2f8..93c698da104 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindingsStack.cpp +++ b/lib/ExecutionEngine/Orc/OrcCBindingsStack.cpp @@ -17,14 +17,19 @@ using namespace llvm; -std::unique_ptr -OrcCBindingsStack::createCompileCallbackMgr(Triple T) { +OrcCBindingsStack::CallbackManagerBuilder +OrcCBindingsStack::createCallbackManagerBuilder(Triple T) { switch (T.getArch()) { default: return nullptr; case Triple::x86_64: { - typedef orc::JITCompileCallbackManager CCMgrT; - return llvm::make_unique(0); + typedef orc::JITCompileCallbackManager CCMgrT; + return [](CompileLayerT &CompileLayer, RuntimeDyld::MemoryManager &MemMgr, + LLVMContext &Context) { + return llvm::make_unique(CompileLayer, MemMgr, Context, 0, + 64); + }; } } } diff --git a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h index c62210112c3..6188b29befc 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -34,7 +34,10 @@ public: typedef orc::IRCompileLayer CompileLayerT; typedef orc::CompileOnDemandLayer CODLayerT; - typedef std::function()> + typedef std::function< + std::unique_ptr(CompileLayerT&, + RuntimeDyld::MemoryManager&, + LLVMContext&)> CallbackManagerBuilder; typedef CODLayerT::IndirectStubsManagerBuilderT IndirectStubsManagerBuilder; @@ -83,18 +86,19 @@ public: typedef unsigned ModuleHandleT; - static std::unique_ptr createCompileCallbackMgr(Triple T); + static CallbackManagerBuilder createCallbackManagerBuilder(Triple T); static IndirectStubsManagerBuilder createIndirectStubsMgrBuilder(Triple T); - OrcCBindingsStack(TargetMachine &TM, - std::unique_ptr CCMgr, + OrcCBindingsStack(TargetMachine &TM, LLVMContext &Context, + CallbackManagerBuilder &BuildCallbackMgr, IndirectStubsManagerBuilder IndirectStubsMgrBuilder) - : DL(TM.createDataLayout()), CCMgr(std::move(CCMgr)), + : Context(Context), DL(TM.createDataLayout()), ObjectLayer(), CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)), + CCMgr(BuildCallbackMgr(CompileLayer, CCMgrMemMgr, Context)), CODLayer(CompileLayer, [](Function &F) { std::set S; S.insert(&F); return S; }, - *this->CCMgr, std::move(IndirectStubsMgrBuilder), false), + *CCMgr, std::move(IndirectStubsMgrBuilder), false), IndirectStubsMgr(IndirectStubsMgrBuilder()), CXXRuntimeOverrides([this](const std::string &S) { return mangle(S); }) {} @@ -123,7 +127,7 @@ public: orc::TargetAddress createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx) { - auto CCInfo = CCMgr->getCompileCallback(); + auto CCInfo = CCMgr->getCompileCallback(Context); CCInfo.setCompileAction( [=]() -> orc::TargetAddress { return Callback(wrap(this), CallbackCtx); @@ -260,12 +264,13 @@ private: return NewHandle; } + LLVMContext &Context; DataLayout DL; SectionMemoryManager CCMgrMemMgr; - std::unique_ptr CCMgr; ObjLayerT ObjectLayer; CompileLayerT CompileLayer; + std::unique_ptr CCMgr; CODLayerT CODLayer; std::unique_ptr IndirectStubsMgr; diff --git a/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp b/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp index b931f10b9d7..c03b935baeb 100644 --- a/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp +++ b/lib/ExecutionEngine/Orc/OrcTargetSupport.cpp @@ -12,88 +12,136 @@ #include "llvm/Support/Process.h" #include -namespace llvm { -namespace orc { +using namespace llvm::orc; -void OrcX86_64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, - void *CallbackMgr) { - - const uint8_t ResolverCode[] = { - // resolver_entry: - 0x55, // 0x00: pushq %rbp - 0x48, 0x89, 0xe5, // 0x01: movq %rsp, %rbp - 0x50, // 0x04: pushq %rax - 0x53, // 0x05: pushq %rbx - 0x51, // 0x06: pushq %rcx - 0x52, // 0x07: pushq %rdx - 0x56, // 0x08: pushq %rsi - 0x57, // 0x09: pushq %rdi - 0x41, 0x50, // 0x0a: pushq %r8 - 0x41, 0x51, // 0x0c: pushq %r9 - 0x41, 0x52, // 0x0e: pushq %r10 - 0x41, 0x53, // 0x10: pushq %r11 - 0x41, 0x54, // 0x12: pushq %r12 - 0x41, 0x55, // 0x14: pushq %r13 - 0x41, 0x56, // 0x16: pushq %r14 - 0x41, 0x57, // 0x18: pushq %r15 - 0x48, 0x81, 0xec, 0x08, 0x02, 0x00, 0x00, // 0x1a: subq 20, %rsp - 0x48, 0x0f, 0xae, 0x04, 0x24, // 0x21: fxsave64 (%rsp) - 0x48, 0x8d, 0x3d, 0x43, 0x00, 0x00, 0x00, // 0x26: leaq 67(%rip), %rdi - 0x48, 0x8b, 0x3f, // 0x2d: movq (%rdi), %rdi - 0x48, 0x8b, 0x75, 0x08, // 0x30: movq 8(%rbp), %rsi - 0x48, 0x83, 0xee, 0x06, // 0x34: subq $6, %rsi - 0x48, 0xb8, // 0x38: movabsq $0, %rax - - // 0x3a: JIT re-entry fn addr: - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - 0xff, 0xd0, // 0x42: callq *%rax - 0x48, 0x89, 0x45, 0x08, // 0x44: movq %rax, 8(%rbp) - 0x48, 0x0f, 0xae, 0x0c, 0x24, // 0x48: fxrstor64 (%rsp) - 0x48, 0x81, 0xc4, 0x08, 0x02, 0x00, 0x00, // 0x4d: addq 20, %rsp - 0x41, 0x5f, // 0x54: popq %r15 - 0x41, 0x5e, // 0x56: popq %r14 - 0x41, 0x5d, // 0x58: popq %r13 - 0x41, 0x5c, // 0x5a: popq %r12 - 0x41, 0x5b, // 0x5c: popq %r11 - 0x41, 0x5a, // 0x5e: popq %r10 - 0x41, 0x59, // 0x60: popq %r9 - 0x41, 0x58, // 0x62: popq %r8 - 0x5f, // 0x64: popq %rdi - 0x5e, // 0x65: popq %rsi - 0x5a, // 0x66: popq %rdx - 0x59, // 0x67: popq %rcx - 0x5b, // 0x68: popq %rbx - 0x58, // 0x69: popq %rax - 0x5d, // 0x6a: popq %rbp - 0xc3, // 0x6b: retq - 0x00, 0x00, 0x00, 0x00, // 0x6c: - - // 0x70: Callback mgr address. - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - }; +namespace { + +uint64_t executeCompileCallback(JITCompileCallbackManagerBase *JCBM, + TargetAddress CallbackID) { + return JCBM->executeCompileCallback(CallbackID); +} - const unsigned ReentryFnAddrOffset = 0x3a; - const unsigned CallbackMgrAddrOffset = 0x70; - - memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); - memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); - memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, - sizeof(CallbackMgr)); } -void OrcX86_64::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, - unsigned NumTrampolines) { +namespace llvm { +namespace orc { - unsigned OffsetToPtr = NumTrampolines * TrampolineSize; +const char* OrcX86_64::ResolverBlockName = "orc_resolver_block"; + +void OrcX86_64::insertResolverBlock( + Module &M, JITCompileCallbackManagerBase &JCBM) { + + // Trampoline code-sequence length, used to get trampoline address from return + // address. + const unsigned X86_64_TrampolineLength = 6; + + // List of x86-64 GPRs to save. Note - RBP saved separately below. + std::array GPRs = {{ + "rax", "rbx", "rcx", "rdx", + "rsi", "rdi", "r8", "r9", + "r10", "r11", "r12", "r13", + "r14", "r15" + }}; + + // Address of the executeCompileCallback function. + uint64_t CallbackAddr = + static_cast( + reinterpret_cast(executeCompileCallback)); + + std::ostringstream AsmStream; + Triple TT(M.getTargetTriple()); + + // Switch to text section. + if (TT.getOS() == Triple::Darwin) + AsmStream << ".section __TEXT,__text,regular,pure_instructions\n" + << ".align 4, 0x90\n"; + else + AsmStream << ".text\n" + << ".align 16, 0x90\n"; + + // Bake in a pointer to the callback manager immediately before the + // start of the resolver function. + AsmStream << "jit_callback_manager_addr:\n" + << " .quad " << &JCBM << "\n"; + + // Start the resolver function. + AsmStream << ResolverBlockName << ":\n" + << " pushq %rbp\n" + << " movq %rsp, %rbp\n"; + + // Store the GPRs. + for (const auto &GPR : GPRs) + AsmStream << " pushq %" << GPR << "\n"; + + // Store floating-point state with FXSAVE. + // Note: We need to keep the stack 16-byte aligned, so if we've emitted an odd + // number of 64-bit pushes so far (GPRs.size() plus 1 for RBP) then add + // an extra 64 bits of padding to the FXSave area. + unsigned Padding = (GPRs.size() + 1) % 2 ? 8 : 0; + unsigned FXSaveSize = 512 + Padding; + AsmStream << " subq $" << FXSaveSize << ", %rsp\n" + << " fxsave64 (%rsp)\n" + + // Load callback manager address, compute trampoline address, call JIT. + << " lea jit_callback_manager_addr(%rip), %rdi\n" + << " movq (%rdi), %rdi\n" + << " movq 0x8(%rbp), %rsi\n" + << " subq $" << X86_64_TrampolineLength << ", %rsi\n" + << " movabsq $" << CallbackAddr << ", %rax\n" + << " callq *%rax\n" + + // Replace the return to the trampoline with the return address of the + // compiled function body. + << " movq %rax, 0x8(%rbp)\n" + + // Restore the floating point state. + << " fxrstor64 (%rsp)\n" + << " addq $" << FXSaveSize << ", %rsp\n"; + + for (const auto &GPR : make_range(GPRs.rbegin(), GPRs.rend())) + AsmStream << " popq %" << GPR << "\n"; + + // Restore original RBP and return to compiled function body. + AsmStream << " popq %rbp\n" + << " retq\n"; + + M.appendModuleInlineAsm(AsmStream.str()); +} + +OrcX86_64::LabelNameFtor +OrcX86_64::insertCompileCallbackTrampolines(Module &M, + TargetAddress ResolverBlockAddr, + unsigned NumCalls, + unsigned StartIndex) { + const char *ResolverBlockPtrName = "Lorc_resolve_block_addr"; + + std::ostringstream AsmStream; + Triple TT(M.getTargetTriple()); + + if (TT.getOS() == Triple::Darwin) + AsmStream << ".section __TEXT,__text,regular,pure_instructions\n" + << ".align 4, 0x90\n"; + else + AsmStream << ".text\n" + << ".align 16, 0x90\n"; + + AsmStream << ResolverBlockPtrName << ":\n" + << " .quad " << ResolverBlockAddr << "\n"; + + auto GetLabelName = + [=](unsigned I) { + std::ostringstream LabelStream; + LabelStream << "orc_jcc_" << (StartIndex + I); + return LabelStream.str(); + }; - memcpy(TrampolineMem + OffsetToPtr, &ResolverAddr, sizeof(void*)); + for (unsigned I = 0; I < NumCalls; ++I) + AsmStream << GetLabelName(I) << ":\n" + << " callq *" << ResolverBlockPtrName << "(%rip)\n"; - uint64_t *Trampolines = reinterpret_cast(TrampolineMem); - uint64_t CallIndirPCRel = 0xf1c40000000015ff; + M.appendModuleInlineAsm(AsmStream.str()); - for (unsigned I = 0; I < NumTrampolines; ++I, OffsetToPtr -= TrampolineSize) - Trampolines[I] = CallIndirPCRel | ((OffsetToPtr - 6) << 16); + return GetLabelName; } std::error_code OrcX86_64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, diff --git a/tools/lli/OrcLazyJIT.cpp b/tools/lli/OrcLazyJIT.cpp index 7d79c48559e..aec6e1a7297 100644 --- a/tools/lli/OrcLazyJIT.cpp +++ b/tools/lli/OrcLazyJIT.cpp @@ -46,14 +46,19 @@ namespace { cl::init(true), cl::Hidden); } -std::unique_ptr -OrcLazyJIT::createCompileCallbackMgr(Triple T) { +OrcLazyJIT::CallbackManagerBuilder +OrcLazyJIT::createCallbackMgrBuilder(Triple T) { switch (T.getArch()) { default: return nullptr; case Triple::x86_64: { - typedef orc::JITCompileCallbackManager CCMgrT; - return llvm::make_unique(0); + typedef orc::JITCompileCallbackManager CCMgrT; + return [](IRDumpLayerT &IRDumpLayer, RuntimeDyld::MemoryManager &MemMgr, + LLVMContext &Context) { + return llvm::make_unique(IRDumpLayer, MemMgr, Context, 0, + 64); + }; } } } @@ -141,12 +146,13 @@ int llvm::runOrcLazyJIT(std::unique_ptr M, int ArgC, char* ArgV[]) { EngineBuilder EB; EB.setOptLevel(getOptLevel()); auto TM = std::unique_ptr(EB.selectTarget()); - auto CompileCallbackMgr = - OrcLazyJIT::createCompileCallbackMgr(Triple(TM->getTargetTriple())); + auto &Context = getGlobalContext(); + auto CallbackMgrBuilder = + OrcLazyJIT::createCallbackMgrBuilder(Triple(TM->getTargetTriple())); // If we couldn't build the factory function then there must not be a callback // manager for this target. Bail out. - if (!CompileCallbackMgr) { + if (!CallbackMgrBuilder) { errs() << "No callback manager available for target '" << TM->getTargetTriple().str() << "'.\n"; return 1; @@ -163,7 +169,7 @@ int llvm::runOrcLazyJIT(std::unique_ptr M, int ArgC, char* ArgV[]) { } // Everything looks good. Build the JIT. - OrcLazyJIT J(std::move(TM), std::move(CompileCallbackMgr), + OrcLazyJIT J(std::move(TM), Context, CallbackMgrBuilder, std::move(IndirectStubsMgrBuilder), OrcInlineStubs); diff --git a/tools/lli/OrcLazyJIT.h b/tools/lli/OrcLazyJIT.h index ec86a72efaa..389380e8dd4 100644 --- a/tools/lli/OrcLazyJIT.h +++ b/tools/lli/OrcLazyJIT.h @@ -23,6 +23,7 @@ #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" +#include "llvm/IR/LLVMContext.h" namespace llvm { @@ -40,16 +41,21 @@ public: IndirectStubsManagerBuilder; typedef CODLayerT::ModuleSetHandleT ModuleHandleT; - OrcLazyJIT(std::unique_ptr TM, - std::unique_ptr CCMgr, + typedef std::function< + std::unique_ptr(IRDumpLayerT&, + RuntimeDyld::MemoryManager&, + LLVMContext&)> + CallbackManagerBuilder; + + OrcLazyJIT(std::unique_ptr TM, LLVMContext &Context, + CallbackManagerBuilder &BuildCallbackMgr, IndirectStubsManagerBuilder IndirectStubsMgrBuilder, bool InlineStubs) - : TM(std::move(TM)), DL(this->TM->createDataLayout()), - CCMgr(std::move(CCMgr)), - ObjectLayer(), + : TM(std::move(TM)), DL(this->TM->createDataLayout()), ObjectLayer(), CompileLayer(ObjectLayer, orc::SimpleCompiler(*this->TM)), IRDumpLayer(CompileLayer, createDebugDumper()), - CODLayer(IRDumpLayer, extractSingleFunction, *this->CCMgr, + CCMgr(BuildCallbackMgr(IRDumpLayer, CCMgrMemMgr, Context)), + CODLayer(IRDumpLayer, extractSingleFunction, *CCMgr, std::move(IndirectStubsMgrBuilder), InlineStubs), CXXRuntimeOverrides( [this](const std::string &S) { return mangle(S); }) {} @@ -62,7 +68,8 @@ public: DtorRunner.runViaLayer(CODLayer); } - static std::unique_ptr createCompileCallbackMgr(Triple T); + static CallbackManagerBuilder createCallbackMgrBuilder(Triple T); + static IndirectStubsManagerBuilder createIndirectStubsMgrBuilder(Triple T); ModuleHandleT addModule(std::unique_ptr M) { @@ -148,10 +155,10 @@ private: DataLayout DL; SectionMemoryManager CCMgrMemMgr; - std::unique_ptr CCMgr; ObjLayerT ObjectLayer; CompileLayerT CompileLayer; IRDumpLayerT IRDumpLayer; + std::unique_ptr CCMgr; CODLayerT CODLayer; orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides; diff --git a/unittests/ExecutionEngine/Orc/CompileOnDemandLayerTest.cpp b/unittests/ExecutionEngine/Orc/CompileOnDemandLayerTest.cpp index 078cbdaa300..ecfefe47c36 100644 --- a/unittests/ExecutionEngine/Orc/CompileOnDemandLayerTest.cpp +++ b/unittests/ExecutionEngine/Orc/CompileOnDemandLayerTest.cpp @@ -19,11 +19,11 @@ namespace { class DummyCallbackManager : public orc::JITCompileCallbackManagerBase { public: DummyCallbackManager() - : JITCompileCallbackManagerBase(0), NextStubAddress(0), + : JITCompileCallbackManagerBase(0, 0), NextStubAddress(0), UniversalCompile([]() { return 0; }) { } - CompileCallbackInfo getCompileCallback() override { + CompileCallbackInfo getCompileCallback(LLVMContext &Context) override { return CompileCallbackInfo(++NextStubAddress, UniversalCompile); } public: diff --git a/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp b/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp index 2957a3c1570..095d42599bf 100644 --- a/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp +++ b/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp @@ -82,7 +82,7 @@ TEST_F(OrcCAPIExecutionTest, TestEagerIRCompilation) { return; LLVMOrcJITStackRef JIT = - LLVMOrcCreateInstance(wrap(TM.get())); + LLVMOrcCreateInstance(wrap(TM.get()), LLVMGetGlobalContext()); std::unique_ptr M = createTestModule(TM->getTargetTriple()); @@ -106,7 +106,7 @@ TEST_F(OrcCAPIExecutionTest, TestLazyIRCompilation) { return; LLVMOrcJITStackRef JIT = - LLVMOrcCreateInstance(wrap(TM.get())); + LLVMOrcCreateInstance(wrap(TM.get()), LLVMGetGlobalContext()); std::unique_ptr M = createTestModule(TM->getTargetTriple()); @@ -130,7 +130,7 @@ TEST_F(OrcCAPIExecutionTest, TestDirectCallbacksAPI) { return; LLVMOrcJITStackRef JIT = - LLVMOrcCreateInstance(wrap(TM.get())); + LLVMOrcCreateInstance(wrap(TM.get()), LLVMGetGlobalContext()); LLVMOrcGetMangledSymbol(JIT, &testFuncName, "testFunc"); -- 2.34.1