1 //===-- IndirectionUtils.h - Utilities for adding indirections --*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Contains utilities for adding indirections and breaking up modules.
12 //===----------------------------------------------------------------------===//
14 #ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
15 #define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
17 #include "JITSymbol.h"
18 #include "LambdaResolver.h"
19 #include "llvm/ADT/DenseSet.h"
20 #include "llvm/ExecutionEngine/RuntimeDyld.h"
21 #include "llvm/IR/IRBuilder.h"
22 #include "llvm/IR/Mangler.h"
23 #include "llvm/IR/Module.h"
24 #include "llvm/Transforms/Utils/ValueMapper.h"
30 /// @brief Target-independent base class JITCompileCallbackManager.
31 class JITCompileCallbackManagerBase {
34 typedef std::function<TargetAddress()> CompileFtor;
36 /// @brief Handle to a newly created compile callback. Can be used to get an
37 /// IR constant representing the address of the trampoline, and to set
38 /// the compile action for the callback.
39 class CompileCallbackInfo {
41 CompileCallbackInfo(TargetAddress Addr, CompileFtor &Compile)
42 : Addr(Addr), Compile(Compile) {}
44 TargetAddress getAddress() const { return Addr; }
45 void setCompileAction(CompileFtor Compile) {
46 this->Compile = std::move(Compile);
53 /// @brief Construct a JITCompileCallbackManagerBase.
54 /// @param ErrorHandlerAddress The address of an error handler in the target
55 /// process to be used if a compile callback fails.
56 JITCompileCallbackManagerBase(TargetAddress ErrorHandlerAddress)
57 : ErrorHandlerAddress(ErrorHandlerAddress) {}
59 virtual ~JITCompileCallbackManagerBase() {}
61 /// @brief Execute the callback for the given trampoline id. Called by the JIT
62 /// to compile functions on demand.
63 TargetAddress executeCompileCallback(TargetAddress TrampolineAddr) {
64 auto I = ActiveTrampolines.find(TrampolineAddr);
65 // FIXME: Also raise an error in the Orc error-handler when we finally have
67 if (I == ActiveTrampolines.end())
68 return ErrorHandlerAddress;
70 // Found a callback handler. Yank this trampoline out of the active list and
71 // put it back in the available trampolines list, then try to run the
72 // handler's compile and update actions.
73 // Moving the trampoline ID back to the available list first means there's at
74 // least one available trampoline if the compile action triggers a request for
76 auto Compile = std::move(I->second);
77 ActiveTrampolines.erase(I);
78 AvailableTrampolines.push_back(TrampolineAddr);
80 if (auto Addr = Compile())
83 return ErrorHandlerAddress;
86 /// @brief Reserve a compile callback.
87 virtual CompileCallbackInfo getCompileCallback() = 0;
89 /// @brief Get a CompileCallbackInfo for an existing callback.
90 CompileCallbackInfo getCompileCallbackInfo(TargetAddress TrampolineAddr) {
91 auto I = ActiveTrampolines.find(TrampolineAddr);
92 assert(I != ActiveTrampolines.end() && "Not an active trampoline.");
93 return CompileCallbackInfo(I->first, I->second);
96 /// @brief Release a compile callback.
98 /// Note: Callbacks are auto-released after they execute. This method should
99 /// only be called to manually release a callback that is not going to
101 void releaseCompileCallback(TargetAddress TrampolineAddr) {
102 auto I = ActiveTrampolines.find(TrampolineAddr);
103 assert(I != ActiveTrampolines.end() && "Not an active trampoline.");
104 ActiveTrampolines.erase(I);
105 AvailableTrampolines.push_back(TrampolineAddr);
109 TargetAddress ErrorHandlerAddress;
111 typedef std::map<TargetAddress, CompileFtor> TrampolineMapT;
112 TrampolineMapT ActiveTrampolines;
113 std::vector<TargetAddress> AvailableTrampolines;
116 virtual void anchor();
119 /// @brief Manage compile callbacks.
120 template <typename TargetT>
121 class JITCompileCallbackManager : public JITCompileCallbackManagerBase {
124 /// @brief Construct a JITCompileCallbackManager.
125 /// @param ErrorHandlerAddress The address of an error handler in the target
126 /// process to be used if a compile callback fails.
127 JITCompileCallbackManager(TargetAddress ErrorHandlerAddress)
128 : JITCompileCallbackManagerBase(ErrorHandlerAddress) {
130 /// Set up the resolver block.
133 sys::OwningMemoryBlock(
134 sys::Memory::allocateMappedMemory(TargetT::ResolverCodeSize, nullptr,
135 sys::Memory::MF_READ |
136 sys::Memory::MF_WRITE, EC));
137 assert(!EC && "Failed to allocate resolver block");
139 TargetT::writeResolverCode(static_cast<uint8_t*>(ResolverBlock.base()),
142 EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),
143 sys::Memory::MF_READ |
144 sys::Memory::MF_EXEC);
145 assert(!EC && "Failed to mprotect resolver block");
148 /// @brief Get/create a compile callback with the given signature.
149 CompileCallbackInfo getCompileCallback() final {
150 TargetAddress TrampolineAddr = getAvailableTrampolineAddr();
151 auto &Compile = this->ActiveTrampolines[TrampolineAddr];
152 return CompileCallbackInfo(TrampolineAddr, Compile);
157 static TargetAddress reenter(void *CCMgr, void *TrampolineId) {
158 JITCompileCallbackManager *Mgr =
159 static_cast<JITCompileCallbackManager*>(CCMgr);
160 return Mgr->executeCompileCallback(
161 static_cast<TargetAddress>(
162 reinterpret_cast<uintptr_t>(TrampolineId)));
165 TargetAddress getAvailableTrampolineAddr() {
166 if (this->AvailableTrampolines.empty())
168 assert(!this->AvailableTrampolines.empty() &&
169 "Failed to grow available trampolines.");
170 TargetAddress TrampolineAddr = this->AvailableTrampolines.back();
171 this->AvailableTrampolines.pop_back();
172 return TrampolineAddr;
176 assert(this->AvailableTrampolines.empty() && "Growing prematurely?");
179 auto TrampolineBlock =
180 sys::OwningMemoryBlock(
181 sys::Memory::allocateMappedMemory(TargetT::PageSize, nullptr,
182 sys::Memory::MF_READ |
183 sys::Memory::MF_WRITE, EC));
184 assert(!EC && "Failed to allocate trampoline block");
187 unsigned NumTrampolines =
188 (TargetT::PageSize - TargetT::PointerSize) / TargetT::TrampolineSize;
190 uint8_t *TrampolineMem = static_cast<uint8_t*>(TrampolineBlock.base());
191 TargetT::writeTrampolines(TrampolineMem, ResolverBlock.base(),
194 for (unsigned I = 0; I < NumTrampolines; ++I)
195 this->AvailableTrampolines.push_back(
196 static_cast<TargetAddress>(
197 reinterpret_cast<uintptr_t>(
198 TrampolineMem + (I * TargetT::TrampolineSize))));
200 EC = sys::Memory::protectMappedMemory(TrampolineBlock.getMemoryBlock(),
201 sys::Memory::MF_READ |
202 sys::Memory::MF_EXEC);
203 assert(!EC && "Failed to mprotect trampoline block");
205 TrampolineBlocks.push_back(std::move(TrampolineBlock));
208 sys::OwningMemoryBlock ResolverBlock;
209 std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
212 /// @brief Base class for managing collections of named indirect stubs.
213 class IndirectStubsManagerBase {
216 /// @brief Map type for initializing the manager. See init.
217 typedef StringMap<std::pair<TargetAddress, JITSymbolFlags>> StubInitsMap;
219 virtual ~IndirectStubsManagerBase() {}
221 /// @brief Create a single stub with the given name, target address and flags.
222 virtual std::error_code createStub(StringRef StubName, TargetAddress StubAddr,
223 JITSymbolFlags StubFlags) = 0;
225 /// @brief Create StubInits.size() stubs with the given names, target
226 /// addresses, and flags.
227 virtual std::error_code createStubs(const StubInitsMap &StubInits) = 0;
229 /// @brief Find the stub with the given name. If ExportedStubsOnly is true,
230 /// this will only return a result if the stub's flags indicate that it
232 virtual JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0;
234 /// @brief Find the implementation-pointer for the stub.
235 virtual JITSymbol findPointer(StringRef Name) = 0;
237 /// @brief Change the value of the implementation pointer for the stub.
238 virtual std::error_code updatePointer(StringRef Name, TargetAddress NewAddr) = 0;
240 virtual void anchor();
243 /// @brief IndirectStubsManager implementation for a concrete target, e.g.
244 /// OrcX86_64. (See OrcTargetSupport.h).
245 template <typename TargetT>
246 class IndirectStubsManager : public IndirectStubsManagerBase {
249 std::error_code createStub(StringRef StubName, TargetAddress StubAddr,
250 JITSymbolFlags StubFlags) override {
251 if (auto EC = reserveStubs(1))
254 createStubInternal(StubName, StubAddr, StubFlags);
256 return std::error_code();
259 std::error_code createStubs(const StubInitsMap &StubInits) override {
260 if (auto EC = reserveStubs(StubInits.size()))
263 for (auto &Entry : StubInits)
264 createStubInternal(Entry.first(), Entry.second.first,
265 Entry.second.second);
267 return std::error_code();
270 JITSymbol findStub(StringRef Name, bool ExportedStubsOnly) override {
271 auto I = StubIndexes.find(Name);
272 if (I == StubIndexes.end())
274 auto Key = I->second.first;
275 void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);
276 assert(StubAddr && "Missing stub address");
277 auto StubTargetAddr =
278 static_cast<TargetAddress>(reinterpret_cast<uintptr_t>(StubAddr));
279 auto StubSymbol = JITSymbol(StubTargetAddr, I->second.second);
280 if (ExportedStubsOnly && !StubSymbol.isExported())
285 JITSymbol findPointer(StringRef Name) override {
286 auto I = StubIndexes.find(Name);
287 if (I == StubIndexes.end())
289 auto Key = I->second.first;
290 void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);
291 assert(PtrAddr && "Missing pointer address");
293 static_cast<TargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr));
294 return JITSymbol(PtrTargetAddr, I->second.second);
297 std::error_code updatePointer(StringRef Name, TargetAddress NewAddr) override {
298 auto I = StubIndexes.find(Name);
299 assert(I != StubIndexes.end() && "No stub pointer for symbol");
300 auto Key = I->second.first;
301 *IndirectStubsInfos[Key.first].getPtr(Key.second) =
302 reinterpret_cast<void*>(static_cast<uintptr_t>(NewAddr));
303 return std::error_code();
308 std::error_code reserveStubs(unsigned NumStubs) {
309 if (NumStubs <= FreeStubs.size())
310 return std::error_code();
312 unsigned NewStubsRequired = NumStubs - FreeStubs.size();
313 unsigned NewBlockId = IndirectStubsInfos.size();
314 typename TargetT::IndirectStubsInfo ISI;
315 if (auto EC = TargetT::emitIndirectStubsBlock(ISI, NewStubsRequired,
318 for (unsigned I = 0; I < ISI.getNumStubs(); ++I)
319 FreeStubs.push_back(std::make_pair(NewBlockId, I));
320 IndirectStubsInfos.push_back(std::move(ISI));
321 return std::error_code();
324 void createStubInternal(StringRef StubName, TargetAddress InitAddr,
325 JITSymbolFlags StubFlags) {
326 auto Key = FreeStubs.back();
327 FreeStubs.pop_back();
328 *IndirectStubsInfos[Key.first].getPtr(Key.second) =
329 reinterpret_cast<void*>(static_cast<uintptr_t>(InitAddr));
330 StubIndexes[StubName] = std::make_pair(Key, StubFlags);
333 std::vector<typename TargetT::IndirectStubsInfo> IndirectStubsInfos;
334 typedef std::pair<uint16_t, uint16_t> StubKey;
335 std::vector<StubKey> FreeStubs;
336 StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;
339 /// @brief Build a function pointer of FunctionType with the given constant
342 /// Usage example: Turn a trampoline address into a function pointer constant
343 /// for use in a stub.
344 Constant* createIRTypedAddress(FunctionType &FT, TargetAddress Addr);
346 /// @brief Create a function pointer with the given type, name, and initializer
347 /// in the given Module.
348 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
349 const Twine &Name, Constant *Initializer);
351 /// @brief Turn a function declaration into a stub function that makes an
352 /// indirect call using the given function pointer.
353 void makeStub(Function &F, Value &ImplPointer);
355 /// @brief Raise linkage types and rename as necessary to ensure that all
356 /// symbols are accessible for other modules.
358 /// This should be called before partitioning a module to ensure that the
359 /// partitions retain access to each other's symbols.
360 void makeAllSymbolsExternallyAccessible(Module &M);
362 /// @brief Clone a function declaration into a new module.
364 /// This function can be used as the first step towards creating a callback
365 /// stub (see makeStub), or moving a function body (see moveFunctionBody).
367 /// If the VMap argument is non-null, a mapping will be added between F and
368 /// the new declaration, and between each of F's arguments and the new
369 /// declaration's arguments. This map can then be passed in to moveFunction to
370 /// move the function body if required. Note: When moving functions between
371 /// modules with these utilities, all decls should be cloned (and added to a
372 /// single VMap) before any bodies are moved. This will ensure that references
373 /// between functions all refer to the versions in the new module.
374 Function* cloneFunctionDecl(Module &Dst, const Function &F,
375 ValueToValueMapTy *VMap = nullptr);
377 /// @brief Move the body of function 'F' to a cloned function declaration in a
378 /// different module (See related cloneFunctionDecl).
380 /// If the target function declaration is not supplied via the NewF parameter
381 /// then it will be looked up via the VMap.
383 /// This will delete the body of function 'F' from its original parent module,
384 /// but leave its declaration.
385 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
386 ValueMaterializer *Materializer = nullptr,
387 Function *NewF = nullptr);
389 /// @brief Clone a global variable declaration into a new module.
390 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
391 ValueToValueMapTy *VMap = nullptr);
393 /// @brief Move global variable GV from its parent module to cloned global
394 /// declaration in a different module.
396 /// If the target global declaration is not supplied via the NewGV parameter
397 /// then it will be looked up via the VMap.
399 /// This will delete the initializer of GV from its original parent module,
400 /// but leave its declaration.
401 void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
402 ValueToValueMapTy &VMap,
403 ValueMaterializer *Materializer = nullptr,
404 GlobalVariable *NewGV = nullptr);
407 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
408 ValueToValueMapTy &VMap);
410 } // End namespace orc.
411 } // End namespace llvm.
413 #endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H