X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FExecutionEngine%2FJIT%2FJITEmitter.cpp;h=61eb7f4c3eccc843fdd521b72295924fcb64dcb1;hb=4437658aff84fa25563740c806a091de1395e5f4;hp=ea08de79f3e1ba6be7050d90af87e0f521d7644d;hpb=6f348e458660063a40052b208bab96895c822877;p=oota-llvm.git diff --git a/lib/ExecutionEngine/JIT/JITEmitter.cpp b/lib/ExecutionEngine/JIT/JITEmitter.cpp index ea08de79f3e..61eb7f4c3ec 100644 --- a/lib/ExecutionEngine/JIT/JITEmitter.cpp +++ b/lib/ExecutionEngine/JIT/JITEmitter.cpp @@ -12,53 +12,58 @@ // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "jit" #include "JIT.h" -#include "JITDebugRegisterer.h" -#include "JITDwarfEmitter.h" -#include "llvm/ADT/OwningPtr.h" -#include "llvm/Constants.h" -#include "llvm/Module.h" -#include "llvm/DerivedTypes.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/JITCodeEmitter.h" -#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineCodeInfo.h" #include "llvm/CodeGen/MachineConstantPool.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRelocation.h" #include "llvm/ExecutionEngine/GenericValue.h" #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/JITMemoryManager.h" -#include "llvm/CodeGen/MachineCodeInfo.h" -#include "llvm/Target/TargetData.h" -#include "llvm/Target/TargetJITInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Operator.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/IR/ValueMap.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Memory.h" #include "llvm/Support/MutexGuard.h" -#include "llvm/Support/ValueHandle.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/System/Disassembler.h" -#include "llvm/System/Memory.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/ValueMap.h" +#include "llvm/Target/TargetJITInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" #include #ifndef NDEBUG #include #endif using namespace llvm; +#define DEBUG_TYPE "jit" + STATISTIC(NumBytes, "Number of bytes of machine code compiled"); STATISTIC(NumRelos, "Number of relocations applied"); STATISTIC(NumRetries, "Number of retries with more memory"); -static JIT *TheJIT = 0; +// A declaration may stop being a declaration once it's fully read from bitcode. +// This function returns true if F is fully read and is still a declaration. +static bool isNonGhostDeclaration(const Function *F) { + return F->isDeclaration() && !F->isMaterializable(); +} + //===----------------------------------------------------------------------===// // JIT lazy compilation code. // @@ -70,8 +75,8 @@ namespace { struct NoRAUWValueMapConfig : public ValueMapConfig { typedef JITResolverState *ExtraData; static void onRAUW(JITResolverState *, Value *Old, Value *New) { - assert(false && "The JIT doesn't know how to handle a" - " RAUW on a value it has emitted."); + llvm_unreachable("The JIT doesn't know how to handle a" + " RAUW on a value it has emitted."); } }; @@ -83,15 +88,15 @@ namespace { class JITResolverState { public: typedef ValueMap > - FunctionToStubMapTy; + FunctionToLazyStubMapTy; typedef std::map > CallSiteToFunctionMapTy; typedef ValueMap, CallSiteValueMapConfig> FunctionToCallSitesMapTy; typedef std::map, void*> GlobalToIndirectSymMapTy; private: - /// FunctionToStubMap - Keep track of the stub created for a particular - /// function so that we can reuse them if necessary. - FunctionToStubMapTy FunctionToStubMap; + /// FunctionToLazyStubMap - Keep track of the lazy stub created for a + /// particular function so that we can reuse them if necessary. + FunctionToLazyStubMapTy FunctionToLazyStubMap; /// CallSiteToFunctionMap - Keep track of the function that each lazy call /// site corresponds to, and vice versa. @@ -102,26 +107,32 @@ namespace { /// particular GlobalVariable so that we can reuse them if necessary. GlobalToIndirectSymMapTy GlobalToIndirectSymMap; +#ifndef NDEBUG + /// Instance of the JIT this ResolverState serves. + JIT *TheJIT; +#endif + public: - JITResolverState() : FunctionToStubMap(this), - FunctionToCallSitesMap(this) {} + JITResolverState(JIT *jit) : FunctionToLazyStubMap(this), + FunctionToCallSitesMap(this) { +#ifndef NDEBUG + TheJIT = jit; +#endif + } - FunctionToStubMapTy& getFunctionToStubMap(const MutexGuard& locked) { - assert(locked.holds(TheJIT->lock)); - return FunctionToStubMap; + FunctionToLazyStubMapTy& getFunctionToLazyStubMap() { + return FunctionToLazyStubMap; } - GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap(const MutexGuard& locked) { - assert(locked.holds(TheJIT->lock)); + GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap() { return GlobalToIndirectSymMap; } - pair LookupFunctionFromCallSite( - const MutexGuard &locked, void *CallSite) const { - assert(locked.holds(TheJIT->lock)); - - // The address given to us for the stub may not be exactly right, it might be - // a little bit after the stub. As such, use upper_bound to find it. + std::pair LookupFunctionFromCallSite( + void *CallSite) const { + // The address given to us for the stub may not be exactly right, it + // might be a little bit after the stub. As such, use upper_bound to + // find it. CallSiteToFunctionMapTy::const_iterator I = CallSiteToFunctionMap.upper_bound(CallSite); assert(I != CallSiteToFunctionMap.begin() && @@ -130,9 +141,7 @@ namespace { return *I; } - void AddCallSite(const MutexGuard &locked, void *CallSite, Function *F) { - assert(locked.holds(TheJIT->lock)); - + void AddCallSite(void *CallSite, Function *F) { bool Inserted = CallSiteToFunctionMap.insert( std::make_pair(CallSite, F)).second; (void)Inserted; @@ -140,63 +149,18 @@ namespace { FunctionToCallSitesMap[F].insert(CallSite); } - // Returns the Function of the stub if a stub was erased, or NULL if there - // was no stub. This function uses the call-site->function map to find a - // relevant function, but asserts that only stubs and not other call sites - // will be passed in. - Function *EraseStub(const MutexGuard &locked, void *Stub) { - CallSiteToFunctionMapTy::iterator C2F_I = - CallSiteToFunctionMap.find(Stub); - if (C2F_I == CallSiteToFunctionMap.end()) { - // Not a stub. - return NULL; - } - - Function *const F = C2F_I->second; -#ifndef NDEBUG - void *RealStub = FunctionToStubMap.lookup(F); - assert(RealStub == Stub && - "Call-site that wasn't a stub pass in to EraseStub"); -#endif - FunctionToStubMap.erase(F); - CallSiteToFunctionMap.erase(C2F_I); - - // Remove the stub from the function->call-sites map, and remove the whole - // entry from the map if that was the last call site. - FunctionToCallSitesMapTy::iterator F2C_I = FunctionToCallSitesMap.find(F); - assert(F2C_I != FunctionToCallSitesMap.end() && - "FunctionToCallSitesMap broken"); - bool Erased = F2C_I->second.erase(Stub); - (void)Erased; - assert(Erased && "FunctionToCallSitesMap broken"); - if (F2C_I->second.empty()) - FunctionToCallSitesMap.erase(F2C_I); - - return F; - } + void EraseAllCallSitesForPrelocked(Function *F); - void EraseAllCallSites(const MutexGuard &locked, Function *F) { - assert(locked.holds(TheJIT->lock)); - EraseAllCallSitesPrelocked(F); - } - void EraseAllCallSitesPrelocked(Function *F) { - FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F); - if (F2C == FunctionToCallSitesMap.end()) - return; - for (SmallPtrSet::const_iterator I = F2C->second.begin(), - E = F2C->second.end(); I != E; ++I) { - bool Erased = CallSiteToFunctionMap.erase(*I); - (void)Erased; - assert(Erased && "Missing call site->function mapping"); - } - FunctionToCallSitesMap.erase(F2C); - } + // Erases _all_ call sites regardless of their function. This is used to + // unregister the stub addresses from the StubToResolverMap in + // ~JITResolver(). + void EraseAllCallSitesPrelocked(); }; /// JITResolver - Keep track of, and resolve, call sites for functions that /// have not yet been compiled. class JITResolver { - typedef JITResolverState::FunctionToStubMapTy FunctionToStubMapTy; + typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy; typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy; typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy; @@ -206,8 +170,11 @@ namespace { JITResolverState state; - /// ExternalFnToStubMap - This is the equivalent of FunctionToStubMap for - /// external functions. + /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap + /// for external functions. TODO: Of course, external functions don't need + /// a lazy stub. It's actually here to make it more likely that far calls + /// succeed, but no single stub can guarantee that. I'll remove this in a + /// subsequent checkin when I actually fix far calls. std::map ExternalFnToStubMap; /// revGOTMap - map addresses to indexes in the GOT @@ -216,28 +183,24 @@ namespace { JITEmitter &JE; - static JITResolver *TheJITResolver; - public: - explicit JITResolver(JIT &jit, JITEmitter &je) : nextGOTIndex(0), JE(je) { - TheJIT = &jit; + /// Instance of JIT corresponding to this Resolver. + JIT *TheJIT; + public: + explicit JITResolver(JIT &jit, JITEmitter &je) + : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) { LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); - assert(TheJITResolver == 0 && "Multiple JIT resolvers?"); - TheJITResolver = this; - } - - ~JITResolver() { - TheJITResolver = 0; } - /// getFunctionStubIfAvailable - This returns a pointer to a function stub - /// if it has already been created. - void *getFunctionStubIfAvailable(Function *F); + ~JITResolver(); + + /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's + /// lazy-compilation stub if it has already been created. + void *getLazyFunctionStubIfAvailable(Function *F); - /// getFunctionStub - This returns a pointer to a function stub, creating - /// one on demand as needed. If empty is true, create a function stub - /// pointing at address 0, to be filled in later. - void *getFunctionStub(Function *F); + /// getLazyFunctionStub - This returns a pointer to a function's + /// lazy-compilation stub, creating one on demand as needed. + void *getLazyFunctionStub(Function *F); /// getExternalFunctionStub - Return a stub for the function at the /// specified address, created lazily on demand. @@ -247,21 +210,6 @@ namespace { /// specified GV address. void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress); - /// AddCallbackAtLocation - If the target is capable of rewriting an - /// instruction without the use of a stub, record the location of the use so - /// we know which function is being used at the location. - void *AddCallbackAtLocation(Function *F, void *Location) { - MutexGuard locked(TheJIT->lock); - /// Get the target-specific JIT resolver function. - state.AddCallSite(locked, Location, F); - return (void*)(intptr_t)LazyResolverFn; - } - - void getRelocatableGVs(SmallVectorImpl &GVs, - SmallVectorImpl &Ptrs); - - GlobalValue *invalidateStub(void *Stub); - /// getGOTIndexForAddress - Return a new or existing index in the GOT for /// an address. This function only manages slots, it does not manage the /// contents of the slots or the memory associated with the GOT. @@ -273,6 +221,55 @@ namespace { static void *JITCompilerFn(void *Stub); }; + class StubToResolverMapTy { + /// Map a stub address to a specific instance of a JITResolver so that + /// lazily-compiled functions can find the right resolver to use. + /// + /// Guarded by Lock. + std::map Map; + + /// Guards Map from concurrent accesses. + mutable sys::Mutex Lock; + + public: + /// Registers a Stub to be resolved by Resolver. + void RegisterStubResolver(void *Stub, JITResolver *Resolver) { + MutexGuard guard(Lock); + Map.insert(std::make_pair(Stub, Resolver)); + } + /// Unregisters the Stub when it's invalidated. + void UnregisterStubResolver(void *Stub) { + MutexGuard guard(Lock); + Map.erase(Stub); + } + /// Returns the JITResolver instance that owns the Stub. + JITResolver *getResolverFromStub(void *Stub) const { + MutexGuard guard(Lock); + // The address given to us for the stub may not be exactly right, it might + // be a little bit after the stub. As such, use upper_bound to find it. + // This is the same trick as in LookupFunctionFromCallSite from + // JITResolverState. + std::map::const_iterator I = Map.upper_bound(Stub); + assert(I != Map.begin() && "This is not a known stub!"); + --I; + return I->second; + } + /// True if any stubs refer to the given resolver. Only used in an assert(). + /// O(N) + bool ResolverHasStubs(JITResolver* Resolver) const { + MutexGuard guard(Lock); + for (std::map::const_iterator I = Map.begin(), + E = Map.end(); I != E; ++I) { + if (I->second == Resolver) + return true; + } + return false; + } + }; + /// This needs to be static so that a lazy call stub can access it with no + /// context except the address of the stub. + ManagedStatic StubToResolverMap; + /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is /// used to output functions to memory for execution. class JITEmitter : public JITCodeEmitter { @@ -291,7 +288,7 @@ namespace { /// Relocations - These are the relocations that the function needs, as /// emitted. std::vector Relocations; - + /// MBBLocations - This vector is a mapping from MBB ID's to their address. /// It is filled in by the StartMachineBasicBlock callback and queried by /// the getMachineBasicBlockAddress callback. @@ -312,7 +309,7 @@ namespace { /// JumpTable - The jump tables for the current function. /// MachineJumpTableInfo *JumpTable; - + /// JumpTableBase - A pointer to the first entry in the jump table. /// void *JumpTableBase; @@ -320,23 +317,14 @@ namespace { /// Resolver - This contains info about the currently resolved functions. JITResolver Resolver; - /// DE - The dwarf emitter for the jit. - OwningPtr DE; - - /// DR - The debug registerer for the jit. - OwningPtr DR; - - /// LabelLocations - This vector is a mapping from Label ID's to their + /// LabelLocations - This vector is a mapping from Label ID's to their /// address. - std::vector LabelLocations; + DenseMap LabelLocations; /// MMI - Machine module info for exception informations MachineModuleInfo* MMI; - // GVSet - a set to keep track of which globals have been seen - SmallPtrSet GVSet; - - // CurFn - The llvm function being emitted. Only valid during + // CurFn - The llvm function being emitted. Only valid during // finishFunction(). const Function *CurFn; @@ -349,7 +337,8 @@ namespace { void *FunctionBody; // Beginning of the function's allocation. void *Code; // The address the function's code actually starts at. void *ExceptionTable; - EmittedCode() : FunctionBody(0), Code(0), ExceptionTable(0) {} + EmittedCode() : FunctionBody(nullptr), Code(nullptr), + ExceptionTable(nullptr) {} }; struct EmittedFunctionConfig : public ValueMapConfig { typedef JITEmitter *ExtraData; @@ -359,86 +348,72 @@ namespace { ValueMap EmittedFunctions; - // CurFnStubUses - For a given Function, a vector of stubs that it - // references. This facilitates the JIT detecting that a stub is no - // longer used, so that it may be deallocated. - DenseMap, SmallVector > CurFnStubUses; + DebugLoc PrevDL; - // StubFnRefs - For a given pointer to a stub, a set of Functions which - // reference the stub. When the count of a stub's references drops to zero, - // the stub is unused. - DenseMap > StubFnRefs; - - DebugLocTuple PrevDLT; + /// Instance of the JIT + JIT *TheJIT; public: JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM) - : SizeEstimate(0), Resolver(jit, *this), MMI(0), CurFn(0), - EmittedFunctions(this) { + : SizeEstimate(0), Resolver(jit, *this), MMI(nullptr), CurFn(nullptr), + EmittedFunctions(this), TheJIT(&jit) { MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); if (jit.getJITInfo().needsGOT()) { MemMgr->AllocateGOT(); - DEBUG(errs() << "JIT is managing a GOT\n"); + DEBUG(dbgs() << "JIT is managing a GOT\n"); } - if (DwarfExceptionHandling || JITEmitDebugInfo) { - DE.reset(new JITDwarfEmitter(jit)); - } - if (JITEmitDebugInfo) { - DR.reset(new JITDebugRegisterer(TM)); - } } - ~JITEmitter() { + ~JITEmitter() { delete MemMgr; } - /// classof - Methods for support type inquiry through isa, cast, and - /// dyn_cast: - /// - static inline bool classof(const JITEmitter*) { return true; } - static inline bool classof(const MachineCodeEmitter*) { return true; } - JITResolver &getJITResolver() { return Resolver; } - virtual void startFunction(MachineFunction &F); - virtual bool finishFunction(MachineFunction &F); - + void startFunction(MachineFunction &F) override; + bool finishFunction(MachineFunction &F) override; + void emitConstantPool(MachineConstantPool *MCP); void initJumpTableInfo(MachineJumpTableInfo *MJTI); void emitJumpTableInfo(MachineJumpTableInfo *MJTI); - - virtual void startGVStub(const GlobalValue* GV, unsigned StubSize, - unsigned Alignment = 1); - virtual void startGVStub(const GlobalValue* GV, void *Buffer, - unsigned StubSize); - virtual void* finishGVStub(const GlobalValue *GV); + + void startGVStub(const GlobalValue* GV, + unsigned StubSize, unsigned Alignment = 1); + void startGVStub(void *Buffer, unsigned StubSize); + void finishGVStub(); + void *allocIndirectGV(const GlobalValue *GV, const uint8_t *Buffer, + size_t Size, unsigned Alignment) override; /// allocateSpace - Reserves space in the current block if any, or /// allocate a new one of the given size. - virtual void *allocateSpace(uintptr_t Size, unsigned Alignment); + void *allocateSpace(uintptr_t Size, unsigned Alignment) override; /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, /// this method does not allocate memory in the current output buffer, /// because a global may live longer than the current function. - virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment); + void *allocateGlobal(uintptr_t Size, unsigned Alignment) override; - virtual void addRelocation(const MachineRelocation &MR) { + void addRelocation(const MachineRelocation &MR) override { Relocations.push_back(MR); } - - virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) { + + void StartMachineBasicBlock(MachineBasicBlock *MBB) override { if (MBBLocations.size() <= (unsigned)MBB->getNumber()) MBBLocations.resize((MBB->getNumber()+1)*2); MBBLocations[MBB->getNumber()] = getCurrentPCValue(); - DEBUG(errs() << "JIT: Emitting BB" << MBB->getNumber() << " at [" + if (MBB->hasAddressTaken()) + TheJIT->addPointerToBasicBlock(MBB->getBasicBlock(), + (void*)getCurrentPCValue()); + DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at [" << (void*) getCurrentPCValue() << "]\n"); } - virtual uintptr_t getConstantPoolEntryAddress(unsigned Entry) const; - virtual uintptr_t getJumpTableEntryAddress(unsigned Entry) const; + uintptr_t getConstantPoolEntryAddress(unsigned Entry) const override; + uintptr_t getJumpTableEntryAddress(unsigned Entry) const override; - virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const { - assert(MBBLocations.size() > (unsigned)MBB->getNumber() && + uintptr_t + getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override { + assert(MBBLocations.size() > (unsigned)MBB->getNumber() && MBBLocations[MBB->getNumber()] && "MBB not emitted!"); return MBBLocations[MBB->getNumber()]; } @@ -452,89 +427,107 @@ namespace { /// function body. void deallocateMemForFunction(const Function *F); - /// AddStubToCurrentFunction - Mark the current function being JIT'd as - /// using the stub at the specified address. Allows - /// deallocateMemForFunction to also remove stubs no longer referenced. - void AddStubToCurrentFunction(void *Stub); - - virtual void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn); - - virtual void emitLabel(uint64_t LabelID) { - if (LabelLocations.size() <= LabelID) - LabelLocations.resize((LabelID+1)*2); - LabelLocations[LabelID] = getCurrentPCValue(); + void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) override; + + void emitLabel(MCSymbol *Label) override { + LabelLocations[Label] = getCurrentPCValue(); } - virtual uintptr_t getLabelAddress(uint64_t LabelID) const { - assert(LabelLocations.size() > (unsigned)LabelID && - LabelLocations[LabelID] && "Label not emitted!"); - return LabelLocations[LabelID]; + DenseMap *getLabelLocations() override { + return &LabelLocations; } - - virtual void setModuleInfo(MachineModuleInfo* Info) { - MMI = Info; - if (DE.get()) DE->setModuleInfo(Info); + + uintptr_t getLabelAddress(MCSymbol *Label) const override { + assert(LabelLocations.count(Label) && "Label not emitted!"); + return LabelLocations.find(Label)->second; } - void setMemoryExecutable() { - MemMgr->setMemoryExecutable(); + void setModuleInfo(MachineModuleInfo* Info) override { + MMI = Info; } - - JITMemoryManager *getMemMgr() const { return MemMgr; } private: void *getPointerToGlobal(GlobalValue *GV, void *Reference, bool MayNeedFarStub); void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference); - unsigned addSizeOfGlobal(const GlobalVariable *GV, unsigned Size); - unsigned addSizeOfGlobalsInConstantVal(const Constant *C, unsigned Size); - unsigned addSizeOfGlobalsInInitializer(const Constant *Init, unsigned Size); - unsigned GetSizeOfGlobalsInBytes(MachineFunction &MF); }; } -JITResolver *JITResolver::TheJITResolver = 0; - void CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) { - JRS->EraseAllCallSitesPrelocked(F); + JRS->EraseAllCallSitesForPrelocked(F); } -/// getFunctionStubIfAvailable - This returns a pointer to a function stub +void JITResolverState::EraseAllCallSitesForPrelocked(Function *F) { + FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F); + if (F2C == FunctionToCallSitesMap.end()) + return; + StubToResolverMapTy &S2RMap = *StubToResolverMap; + for (void *C : F2C->second) { + S2RMap.UnregisterStubResolver(C); + bool Erased = CallSiteToFunctionMap.erase(C); + (void)Erased; + assert(Erased && "Missing call site->function mapping"); + } + FunctionToCallSitesMap.erase(F2C); +} + +void JITResolverState::EraseAllCallSitesPrelocked() { + StubToResolverMapTy &S2RMap = *StubToResolverMap; + for (CallSiteToFunctionMapTy::const_iterator + I = CallSiteToFunctionMap.begin(), + E = CallSiteToFunctionMap.end(); I != E; ++I) { + S2RMap.UnregisterStubResolver(I->first); + } + CallSiteToFunctionMap.clear(); + FunctionToCallSitesMap.clear(); +} + +JITResolver::~JITResolver() { + // No need to lock because we're in the destructor, and state isn't shared. + state.EraseAllCallSitesPrelocked(); + assert(!StubToResolverMap->ResolverHasStubs(this) && + "Resolver destroyed with stubs still alive."); +} + +/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub /// if it has already been created. -void *JITResolver::getFunctionStubIfAvailable(Function *F) { +void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) { MutexGuard locked(TheJIT->lock); // If we already have a stub for this function, recycle it. - return state.getFunctionToStubMap(locked).lookup(F); + return state.getFunctionToLazyStubMap().lookup(F); } /// getFunctionStub - This returns a pointer to a function stub, creating /// one on demand as needed. -void *JITResolver::getFunctionStub(Function *F) { +void *JITResolver::getLazyFunctionStub(Function *F) { MutexGuard locked(TheJIT->lock); - // If we already have a stub for this function, recycle it. - void *&Stub = state.getFunctionToStubMap(locked)[F]; + // If we already have a lazy stub for this function, recycle it. + void *&Stub = state.getFunctionToLazyStubMap()[F]; if (Stub) return Stub; // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we // must resolve the symbol now. void *Actual = TheJIT->isCompilingLazily() - ? (void *)(intptr_t)LazyResolverFn : (void *)0; + ? (void *)(intptr_t)LazyResolverFn : (void *)nullptr; // If this is an external declaration, attempt to resolve the address now // to place in the stub. - if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode()) { + if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) { Actual = TheJIT->getPointerToFunction(F); // If we resolved the symbol to a null address (eg. a weak external) // don't emit a stub. Return a null pointer to the application. - if (!Actual) return 0; + if (!Actual) return nullptr; } + TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); + JE.startGVStub(F, SL.Size, SL.Alignment); // Codegen a new stub, calling the lazy resolver or the actual address of the // external function, if it was resolved. Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE); + JE.finishGVStub(); if (Actual != (void*)(intptr_t)LazyResolverFn) { // If we are getting the stub for an external function, we really want the @@ -543,19 +536,25 @@ void *JITResolver::getFunctionStub(Function *F) { TheJIT->updateGlobalMapping(F, Stub); } - DEBUG(errs() << "JIT: Stub emitted at [" << Stub << "] for function '" + DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '" << F->getName() << "'\n"); - // Finally, keep track of the stub-to-Function mapping so that the - // JITCompilerFn knows which function to compile! - state.AddCallSite(locked, Stub, F); - - // If we are JIT'ing non-lazily but need to call a function that does not - // exist yet, add it to the JIT's work list so that we can fill in the stub - // address later. - if (!Actual && !TheJIT->isCompilingLazily()) - if (!F->isDeclaration() || F->hasNotBeenReadFromBitcode()) - TheJIT->addPendingFunction(F); + if (TheJIT->isCompilingLazily()) { + // Register this JITResolver as the one corresponding to this call site so + // JITCompilerFn will be able to find it. + StubToResolverMap->RegisterStubResolver(Stub, this); + + // Finally, keep track of the stub-to-Function mapping so that the + // JITCompilerFn knows which function to compile! + state.AddCallSite(Stub, F); + } else if (!Actual) { + // If we are JIT'ing non-lazily but need to call a function that does not + // exist yet, add it to the JIT's work list so that we can fill in the + // stub address later. + assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() && + "'Actual' should have been set above."); + TheJIT->addPendingFunction(F); + } return Stub; } @@ -566,14 +565,14 @@ void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) { MutexGuard locked(TheJIT->lock); // If we already have a stub for this global variable, recycle it. - void *&IndirectSym = state.getGlobalToIndirectSymMap(locked)[GV]; + void *&IndirectSym = state.getGlobalToIndirectSymMap()[GV]; if (IndirectSym) return IndirectSym; // Otherwise, codegen a new indirect symbol. IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress, JE); - DEBUG(errs() << "JIT: Indirect symbol emitted at [" << IndirectSym + DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym << "] for GV '" << GV->getName() << "'\n"); return IndirectSym; @@ -586,9 +585,12 @@ void *JITResolver::getExternalFunctionStub(void *FnAddr) { void *&Stub = ExternalFnToStubMap[FnAddr]; if (Stub) return Stub; - Stub = TheJIT->getJITInfo().emitFunctionStub(0, FnAddr, JE); + TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); + JE.startGVStub(nullptr, SL.Size, SL.Alignment); + Stub = TheJIT->getJITInfo().emitFunctionStub(nullptr, FnAddr, JE); + JE.finishGVStub(); - DEBUG(errs() << "JIT: Stub emitted at [" << Stub + DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub << "] for external function at '" << FnAddr << "'\n"); return Stub; } @@ -598,111 +600,59 @@ unsigned JITResolver::getGOTIndexForAddr(void* addr) { if (!idx) { idx = ++nextGOTIndex; revGOTMap[addr] = idx; - DEBUG(errs() << "JIT: Adding GOT entry " << idx << " for addr [" + DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr [" << addr << "]\n"); } return idx; } -void JITResolver::getRelocatableGVs(SmallVectorImpl &GVs, - SmallVectorImpl &Ptrs) { - MutexGuard locked(TheJIT->lock); - - const FunctionToStubMapTy &FM = state.getFunctionToStubMap(locked); - GlobalToIndirectSymMapTy &GM = state.getGlobalToIndirectSymMap(locked); - - for (FunctionToStubMapTy::const_iterator i = FM.begin(), e = FM.end(); - i != e; ++i){ - Function *F = i->first; - if (F->isDeclaration() && F->hasExternalLinkage()) { - GVs.push_back(i->first); - Ptrs.push_back(i->second); - } - } - for (GlobalToIndirectSymMapTy::iterator i = GM.begin(), e = GM.end(); - i != e; ++i) { - GVs.push_back(i->first); - Ptrs.push_back(i->second); - } -} - -GlobalValue *JITResolver::invalidateStub(void *Stub) { - MutexGuard locked(TheJIT->lock); - - GlobalToIndirectSymMapTy &GM = state.getGlobalToIndirectSymMap(locked); - - // Look up the cheap way first, to see if it's a function stub we are - // invalidating. If so, remove it from both the forward and reverse maps. - if (Function *F = state.EraseStub(locked, Stub)) { - return F; - } - - // Otherwise, it might be an indirect symbol stub. Find it and remove it. - for (GlobalToIndirectSymMapTy::iterator i = GM.begin(), e = GM.end(); - i != e; ++i) { - if (i->second != Stub) - continue; - GlobalValue *GV = i->first; - GM.erase(i); - return GV; - } - - // Lastly, check to see if it's in the ExternalFnToStubMap. - for (std::map::iterator i = ExternalFnToStubMap.begin(), - e = ExternalFnToStubMap.end(); i != e; ++i) { - if (i->second != Stub) - continue; - ExternalFnToStubMap.erase(i); - break; - } - - return 0; -} - /// JITCompilerFn - This function is called when a lazy compilation stub has /// been entered. It looks up which function this stub corresponds to, compiles /// it if necessary, then returns the resultant function pointer. void *JITResolver::JITCompilerFn(void *Stub) { - JITResolver &JR = *TheJITResolver; - - Function* F = 0; - void* ActualPtr = 0; + JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub); + assert(JR && "Unable to find the corresponding JITResolver to the call site"); + + Function* F = nullptr; + void* ActualPtr = nullptr; { // Only lock for getting the Function. The call getPointerToFunction made // in this function might trigger function materializing, which requires // JIT lock to be unlocked. - MutexGuard locked(TheJIT->lock); + MutexGuard locked(JR->TheJIT->lock); // The address given to us for the stub may not be exactly right, it might // be a little bit after the stub. As such, use upper_bound to find it. - pair I = - JR.state.LookupFunctionFromCallSite(locked, Stub); + std::pair I = + JR->state.LookupFunctionFromCallSite(Stub); F = I.second; ActualPtr = I.first; } // If we have already code generated the function, just return the address. - void *Result = TheJIT->getPointerToGlobalIfAvailable(F); - + void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F); + if (!Result) { // Otherwise we don't have it, do lazy compilation now. - + // If lazy compilation is disabled, emit a useful error message and abort. - if (!TheJIT->isCompilingLazily()) { - llvm_report_error("LLVM JIT requested to do lazy compilation of function '" + if (!JR->TheJIT->isCompilingLazily()) { + report_fatal_error("LLVM JIT requested to do lazy compilation of" + " function '" + F->getName() + "' when lazy compiles are disabled!"); } - - DEBUG(errs() << "JIT: Lazily resolving function '" << F->getName() + + DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName() << "' In stub ptr = " << Stub << " actual ptr = " << ActualPtr << "\n"); + (void)ActualPtr; - Result = TheJIT->getPointerToFunction(F); + Result = JR->TheJIT->getPointerToFunction(F); } // Reacquire the lock to update the GOT map. - MutexGuard locked(TheJIT->lock); + MutexGuard locked(JR->TheJIT->lock); // We might like to remove the call site from the CallSiteToFunction map, but // we can't do that! Multiple threads could be stuck, waiting to acquire the @@ -717,8 +667,8 @@ void *JITResolver::JITCompilerFn(void *Stub) { // if they see it still using the stub address. // Note: this is done so the Resolver doesn't have to manage GOT memory // Do this without allocating map space if the target isn't using a GOT - if(JR.revGOTMap.find(Stub) != JR.revGOTMap.end()) - JR.revGOTMap[Result] = JR.revGOTMap[Stub]; + if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end()) + JR->revGOTMap[Result] = JR->revGOTMap[Stub]; return Result; } @@ -726,51 +676,53 @@ void *JITResolver::JITCompilerFn(void *Stub) { //===----------------------------------------------------------------------===// // JITEmitter code. // + +static GlobalObject *getSimpleAliasee(Constant *C) { + C = C->stripPointerCasts(); + return dyn_cast(C); +} + void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, bool MayNeedFarStub) { if (GlobalVariable *GV = dyn_cast(V)) return TheJIT->getOrEmitGlobalVariable(GV); - if (GlobalAlias *GA = dyn_cast(V)) - return TheJIT->getPointerToGlobal(GA->resolveAliasedGlobal(false)); + if (GlobalAlias *GA = dyn_cast(V)) { + // We can only handle simple cases. + if (GlobalValue *GV = getSimpleAliasee(GA->getAliasee())) + return TheJIT->getPointerToGlobal(GV); + return nullptr; + } // If we have already compiled the function, return a pointer to its body. Function *F = cast(V); - void *ResultPtr; - if (MayNeedFarStub) { - // Return the function stub if it's already created. - ResultPtr = Resolver.getFunctionStubIfAvailable(F); - if (ResultPtr) - AddStubToCurrentFunction(ResultPtr); - } else { - ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); + + void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F); + if (FnStub) { + // Return the function stub if it's already created. We do this first so + // that we're returning the same address for the function as any previous + // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be + // close enough to call. + return FnStub; } - if (ResultPtr) return ResultPtr; - - // If this is an external function pointer, we can force the JIT to - // 'compile' it, which really just adds it to the map. - if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode() && - !MayNeedFarStub) - return TheJIT->getPointerToFunction(F); - - // Okay, the function has not been compiled yet, if the target callback - // mechanism is capable of rewriting the instruction directly, prefer to do - // that instead of emitting a stub. This uses the lazy resolver, so is not - // legal if lazy compilation is disabled. - if (!MayNeedFarStub && TheJIT->isCompilingLazily()) - return Resolver.AddCallbackAtLocation(F, Reference); - - // Otherwise, we have to emit a stub. - void *StubAddr = Resolver.getFunctionStub(F); - - // Add the stub to the current function's list of referenced stubs, so we can - // deallocate them if the current function is ever freed. It's possible to - // return null from getFunctionStub in the case of a weak extern that fails - // to resolve. - if (StubAddr) - AddStubToCurrentFunction(StubAddr); - return StubAddr; + // If we know the target can handle arbitrary-distance calls, try to + // return a direct pointer. + if (!MayNeedFarStub) { + // If we have code, go ahead and return that. + void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); + if (ResultPtr) return ResultPtr; + + // If this is an external function pointer, we can force the JIT to + // 'compile' it, which really just adds it to the map. + if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) + return TheJIT->getPointerToFunction(F); + } + + // Otherwise, we may need a to emit a stub, and, conservatively, we always do + // so. Note that it's possible to return null from getLazyFunctionStub in the + // case of a weak extern that fails to resolve. + return Resolver.getLazyFunctionStub(F); } void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) { @@ -778,43 +730,27 @@ void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) { // resolved address. void *GVAddress = getPointerToGlobal(V, Reference, false); void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress); - - // Add the stub to the current function's list of referenced stubs, so we can - // deallocate them if the current function is ever freed. - AddStubToCurrentFunction(StubAddr); - return StubAddr; } -void JITEmitter::AddStubToCurrentFunction(void *StubAddr) { - assert(CurFn && "Stub added to current function, but current function is 0!"); - - SmallVectorImpl &StubsUsed = CurFnStubUses[CurFn]; - StubsUsed.push_back(StubAddr); +void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) { + if (DL.isUnknown()) return; + if (!BeforePrintingInsn) return; - SmallPtrSet &FnRefs = StubFnRefs[StubAddr]; - FnRefs.insert(CurFn); -} + const LLVMContext &Context = EmissionDetails.MF->getFunction()->getContext(); -void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) { - if (!DL.isUnknown()) { - DebugLocTuple CurDLT = EmissionDetails.MF->getDebugLocTuple(DL); - - if (BeforePrintingInsn) { - if (CurDLT.Scope != 0 && PrevDLT != CurDLT) { - JITEvent_EmittedFunctionDetails::LineStart NextLine; - NextLine.Address = getCurrentPCValue(); - NextLine.Loc = DL; - EmissionDetails.LineStarts.push_back(NextLine); - } - - PrevDLT = CurDLT; - } + if (DL.getScope(Context) != nullptr && PrevDL != DL) { + JITEvent_EmittedFunctionDetails::LineStart NextLine; + NextLine.Address = getCurrentPCValue(); + NextLine.Loc = DL; + EmissionDetails.LineStarts.push_back(NextLine); } + + PrevDL = DL; } static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, - const TargetData *TD) { + const DataLayout *TD) { const std::vector &Constants = MCP->getConstants(); if (Constants.empty()) return 0; @@ -823,222 +759,21 @@ static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, MachineConstantPoolEntry CPE = Constants[i]; unsigned AlignMask = CPE.getAlignment() - 1; Size = (Size + AlignMask) & ~AlignMask; - const Type *Ty = CPE.getType(); + Type *Ty = CPE.getType(); Size += TD->getTypeAllocSize(Ty); } return Size; } -static unsigned GetJumpTableSizeInBytes(MachineJumpTableInfo *MJTI) { - const std::vector &JT = MJTI->getJumpTables(); - if (JT.empty()) return 0; - - unsigned NumEntries = 0; - for (unsigned i = 0, e = JT.size(); i != e; ++i) - NumEntries += JT[i].MBBs.size(); - - unsigned EntrySize = MJTI->getEntrySize(); - - return NumEntries * EntrySize; -} - -static uintptr_t RoundUpToAlign(uintptr_t Size, unsigned Alignment) { - if (Alignment == 0) Alignment = 1; - // Since we do not know where the buffer will be allocated, be pessimistic. - return Size + Alignment; -} - -/// addSizeOfGlobal - add the size of the global (plus any alignment padding) -/// into the running total Size. - -unsigned JITEmitter::addSizeOfGlobal(const GlobalVariable *GV, unsigned Size) { - const Type *ElTy = GV->getType()->getElementType(); - size_t GVSize = (size_t)TheJIT->getTargetData()->getTypeAllocSize(ElTy); - size_t GVAlign = - (size_t)TheJIT->getTargetData()->getPreferredAlignment(GV); - DEBUG(errs() << "JIT: Adding in size " << GVSize << " alignment " << GVAlign); - DEBUG(GV->dump()); - // Assume code section ends with worst possible alignment, so first - // variable needs maximal padding. - if (Size==0) - Size = 1; - Size = ((Size+GVAlign-1)/GVAlign)*GVAlign; - Size += GVSize; - return Size; -} - -/// addSizeOfGlobalsInConstantVal - find any globals that we haven't seen yet -/// but are referenced from the constant; put them in GVSet and add their -/// size into the running total Size. - -unsigned JITEmitter::addSizeOfGlobalsInConstantVal(const Constant *C, - unsigned Size) { - // If its undefined, return the garbage. - if (isa(C)) - return Size; - - // If the value is a ConstantExpr - if (const ConstantExpr *CE = dyn_cast(C)) { - Constant *Op0 = CE->getOperand(0); - switch (CE->getOpcode()) { - case Instruction::GetElementPtr: - case Instruction::Trunc: - case Instruction::ZExt: - case Instruction::SExt: - case Instruction::FPTrunc: - case Instruction::FPExt: - case Instruction::UIToFP: - case Instruction::SIToFP: - case Instruction::FPToUI: - case Instruction::FPToSI: - case Instruction::PtrToInt: - case Instruction::IntToPtr: - case Instruction::BitCast: { - Size = addSizeOfGlobalsInConstantVal(Op0, Size); - break; - } - case Instruction::Add: - case Instruction::FAdd: - case Instruction::Sub: - case Instruction::FSub: - case Instruction::Mul: - case Instruction::FMul: - case Instruction::UDiv: - case Instruction::SDiv: - case Instruction::URem: - case Instruction::SRem: - case Instruction::And: - case Instruction::Or: - case Instruction::Xor: { - Size = addSizeOfGlobalsInConstantVal(Op0, Size); - Size = addSizeOfGlobalsInConstantVal(CE->getOperand(1), Size); - break; - } - default: { - std::string msg; - raw_string_ostream Msg(msg); - Msg << "ConstantExpr not handled: " << *CE; - llvm_report_error(Msg.str()); - } - } - } - - if (C->getType()->getTypeID() == Type::PointerTyID) - if (const GlobalVariable* GV = dyn_cast(C)) - if (GVSet.insert(GV)) - Size = addSizeOfGlobal(GV, Size); - - return Size; -} - -/// addSizeOfGLobalsInInitializer - handle any globals that we haven't seen yet -/// but are referenced from the given initializer. - -unsigned JITEmitter::addSizeOfGlobalsInInitializer(const Constant *Init, - unsigned Size) { - if (!isa(Init) && - !isa(Init) && - !isa(Init) && - !isa(Init) && - !isa(Init) && - Init->getType()->isFirstClassType()) - Size = addSizeOfGlobalsInConstantVal(Init, Size); - return Size; -} - -/// GetSizeOfGlobalsInBytes - walk the code for the function, looking for -/// globals; then walk the initializers of those globals looking for more. -/// If their size has not been considered yet, add it into the running total -/// Size. - -unsigned JITEmitter::GetSizeOfGlobalsInBytes(MachineFunction &MF) { - unsigned Size = 0; - GVSet.clear(); - - for (MachineFunction::iterator MBB = MF.begin(), E = MF.end(); - MBB != E; ++MBB) { - for (MachineBasicBlock::const_iterator I = MBB->begin(), E = MBB->end(); - I != E; ++I) { - const TargetInstrDesc &Desc = I->getDesc(); - const MachineInstr &MI = *I; - unsigned NumOps = Desc.getNumOperands(); - for (unsigned CurOp = 0; CurOp < NumOps; CurOp++) { - const MachineOperand &MO = MI.getOperand(CurOp); - if (MO.isGlobal()) { - GlobalValue* V = MO.getGlobal(); - const GlobalVariable *GV = dyn_cast(V); - if (!GV) - continue; - // If seen in previous function, it will have an entry here. - if (TheJIT->getPointerToGlobalIfAvailable(GV)) - continue; - // If seen earlier in this function, it will have an entry here. - // FIXME: it should be possible to combine these tables, by - // assuming the addresses of the new globals in this module - // start at 0 (or something) and adjusting them after codegen - // complete. Another possibility is to grab a marker bit in GV. - if (GVSet.insert(GV)) - // A variable as yet unseen. Add in its size. - Size = addSizeOfGlobal(GV, Size); - } - } - } - } - DEBUG(errs() << "JIT: About to look through initializers\n"); - // Look for more globals that are referenced only from initializers. - // GVSet.end is computed each time because the set can grow as we go. - for (SmallPtrSet::iterator I = GVSet.begin(); - I != GVSet.end(); I++) { - const GlobalVariable* GV = *I; - if (GV->hasInitializer()) - Size = addSizeOfGlobalsInInitializer(GV->getInitializer(), Size); - } - - return Size; -} - void JITEmitter::startFunction(MachineFunction &F) { - DEBUG(errs() << "JIT: Starting CodeGen of Function " - << F.getFunction()->getName() << "\n"); + DEBUG(dbgs() << "JIT: Starting CodeGen of Function " + << F.getName() << "\n"); uintptr_t ActualSize = 0; // Set the memory writable, if it's not already MemMgr->setMemoryWritable(); - if (MemMgr->NeedsExactSize()) { - DEBUG(errs() << "JIT: ExactSize\n"); - const TargetInstrInfo* TII = F.getTarget().getInstrInfo(); - MachineJumpTableInfo *MJTI = F.getJumpTableInfo(); - MachineConstantPool *MCP = F.getConstantPool(); - - // Ensure the constant pool/jump table info is at least 4-byte aligned. - ActualSize = RoundUpToAlign(ActualSize, 16); - - // Add the alignment of the constant pool - ActualSize = RoundUpToAlign(ActualSize, MCP->getConstantPoolAlignment()); - - // Add the constant pool size - ActualSize += GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); - - // Add the aligment of the jump table info - ActualSize = RoundUpToAlign(ActualSize, MJTI->getAlignment()); - - // Add the jump table size - ActualSize += GetJumpTableSizeInBytes(MJTI); - - // Add the alignment for the function - ActualSize = RoundUpToAlign(ActualSize, - std::max(F.getFunction()->getAlignment(), 8U)); - - // Add the function size - ActualSize += TII->GetFunctionSizeInBytes(F); - - DEBUG(errs() << "JIT: ActualSize before globals " << ActualSize << "\n"); - // Add the size of the globals that will be allocated after this function. - // These are all the ones referenced from this function that were not - // previously allocated. - ActualSize += GetSizeOfGlobalsInBytes(F); - DEBUG(errs() << "JIT: ActualSize after globals " << ActualSize << "\n"); - } else if (SizeEstimate > 0) { + + if (SizeEstimate > 0) { // SizeEstimate will be non-zero on reallocation attempts. ActualSize = SizeEstimate; } @@ -1052,7 +787,8 @@ void JITEmitter::startFunction(MachineFunction &F) { emitAlignment(16); emitConstantPool(F.getConstantPool()); - initJumpTableInfo(F.getJumpTableInfo()); + if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) + initJumpTableInfo(MJTI); // About to start emitting the machine code for the function. emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); @@ -1074,7 +810,8 @@ bool JITEmitter::finishFunction(MachineFunction &F) { return true; } - emitJumpTableInfo(F.getJumpTableInfo()); + if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) + emitJumpTableInfo(MJTI); // FnStart is the start of the text, not the start of the constant pool and // other per-function data. @@ -1091,13 +828,13 @@ bool JITEmitter::finishFunction(MachineFunction &F) { // Resolve the relocations to concrete pointers. for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { MachineRelocation &MR = Relocations[i]; - void *ResultPtr = 0; + void *ResultPtr = nullptr; if (!MR.letTargetResolve()) { if (MR.isExternalSymbol()) { ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(), false); - DEBUG(errs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" - << ResultPtr << "]\n"); + DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" + << ResultPtr << "]\n"); // If the target REALLY wants a stub for this function, emit it now. if (MR.mayNeedFarStub()) { @@ -1113,7 +850,8 @@ bool JITEmitter::finishFunction(MachineFunction &F) { } else if (MR.isBasicBlock()) { ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); } else if (MR.isConstantPoolIndex()) { - ResultPtr = (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); + ResultPtr = + (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); } else { assert(MR.isJumpTableIndex()); ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); @@ -1128,7 +866,7 @@ bool JITEmitter::finishFunction(MachineFunction &F) { unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); MR.setGOTIndex(idx); if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { - DEBUG(errs() << "JIT: GOT was out of date for " << ResultPtr + DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] << "\n"); ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; @@ -1136,7 +874,7 @@ bool JITEmitter::finishFunction(MachineFunction &F) { } } - CurFn = 0; + CurFn = nullptr; TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], Relocations.size(), MemMgr->getGOTBase()); } @@ -1145,7 +883,7 @@ bool JITEmitter::finishFunction(MachineFunction &F) { if (MemMgr->isManagingGOT()) { unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { - DEBUG(errs() << "JIT: GOT was out of date for " << (void*)BufferBegin + DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] << "\n"); ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; @@ -1165,7 +903,7 @@ bool JITEmitter::finishFunction(MachineFunction &F) { SizeEstimate = 0; } - BufferBegin = CurBufferPtr = 0; + BufferBegin = CurBufferPtr = nullptr; NumBytes += FnEnd-FnStart; // Invalidate the icache if necessary. @@ -1174,8 +912,11 @@ bool JITEmitter::finishFunction(MachineFunction &F) { TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart, EmissionDetails); - DEBUG(errs() << "JIT: Finished CodeGen of [" << (void*)FnStart - << "] Function: " << F.getFunction()->getName() + // Reset the previous debug location. + PrevDL = DebugLoc(); + + DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart + << "] Function: " << F.getName() << ": " << (FnEnd-FnStart) << " bytes of text, " << Relocations.size() << " relocations\n"); @@ -1185,88 +926,49 @@ bool JITEmitter::finishFunction(MachineFunction &F) { // Mark code region readable and executable if it's not so already. MemMgr->setMemoryExecutable(); - DEBUG( - if (sys::hasDisassembler()) { - errs() << "JIT: Disassembled code:\n"; - errs() << sys::disassembleBuffer(FnStart, FnEnd-FnStart, - (uintptr_t)FnStart); - } else { - errs() << "JIT: Binary code:\n"; - uint8_t* q = FnStart; - for (int i = 0; q < FnEnd; q += 4, ++i) { - if (i == 4) - i = 0; - if (i == 0) - errs() << "JIT: " << (long)(q - FnStart) << ": "; - bool Done = false; - for (int j = 3; j >= 0; --j) { - if (q + j >= FnEnd) - Done = true; - else - errs() << (unsigned short)q[j]; + DEBUG({ + dbgs() << "JIT: Binary code:\n"; + uint8_t* q = FnStart; + for (int i = 0; q < FnEnd; q += 4, ++i) { + if (i == 4) + i = 0; + if (i == 0) + dbgs() << "JIT: " << (long)(q - FnStart) << ": "; + bool Done = false; + for (int j = 3; j >= 0; --j) { + if (q + j >= FnEnd) + Done = true; + else + dbgs() << (unsigned short)q[j]; + } + if (Done) + break; + dbgs() << ' '; + if (i == 3) + dbgs() << '\n'; } - if (Done) - break; - errs() << ' '; - if (i == 3) - errs() << '\n'; - } - errs()<< '\n'; - } - ); - - if (DwarfExceptionHandling || JITEmitDebugInfo) { - uintptr_t ActualSize = 0; - SavedBufferBegin = BufferBegin; - SavedBufferEnd = BufferEnd; - SavedCurBufferPtr = CurBufferPtr; - - if (MemMgr->NeedsExactSize()) { - ActualSize = DE->GetDwarfTableSizeInBytes(F, *this, FnStart, FnEnd); - } - - BufferBegin = CurBufferPtr = MemMgr->startExceptionTable(F.getFunction(), - ActualSize); - BufferEnd = BufferBegin+ActualSize; - EmittedFunctions[F.getFunction()].ExceptionTable = BufferBegin; - uint8_t *EhStart; - uint8_t *FrameRegister = DE->EmitDwarfTable(F, *this, FnStart, FnEnd, - EhStart); - MemMgr->endExceptionTable(F.getFunction(), BufferBegin, CurBufferPtr, - FrameRegister); - uint8_t *EhEnd = CurBufferPtr; - BufferBegin = SavedBufferBegin; - BufferEnd = SavedBufferEnd; - CurBufferPtr = SavedCurBufferPtr; - - if (DwarfExceptionHandling) { - TheJIT->RegisterTable(FrameRegister); - } - - if (JITEmitDebugInfo) { - DebugInfo I; - I.FnStart = FnStart; - I.FnEnd = FnEnd; - I.EhStart = EhStart; - I.EhEnd = EhEnd; - DR->RegisterFunction(F.getFunction(), I); - } - } + dbgs()<< '\n'; + }); if (MMI) MMI->EndFunction(); - + return false; } void JITEmitter::retryWithMoreMemory(MachineFunction &F) { - DEBUG(errs() << "JIT: Ran out of space for native code. Reattempting.\n"); + DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n"); Relocations.clear(); // Clear the old relocations or we'll reapply them. ConstPoolAddresses.clear(); ++NumRetries; deallocateMemForFunction(F.getFunction()); // Try again with at least twice as much free space. SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin)); + + for (MachineFunction::iterator MBB = F.begin(), E = F.end(); MBB != E; ++MBB){ + if (MBB->hasAddressTaken()) + TheJIT->clearPointerToBasicBlock(MBB->getBasicBlock()); + } } /// deallocateMemForFunction - Deallocate all memory for the specified @@ -1277,56 +979,14 @@ void JITEmitter::deallocateMemForFunction(const Function *F) { Emitted = EmittedFunctions.find(F); if (Emitted != EmittedFunctions.end()) { MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody); - MemMgr->deallocateExceptionTable(Emitted->second.ExceptionTable); TheJIT->NotifyFreeingMachineCode(Emitted->second.Code); EmittedFunctions.erase(Emitted); } - - // TODO: Do we need to unregister exception handling information from libgcc - // here? - - if (JITEmitDebugInfo) { - DR->UnregisterFunction(F); - } - - // If the function did not reference any stubs, return. - if (CurFnStubUses.find(F) == CurFnStubUses.end()) - return; - - // For each referenced stub, erase the reference to this function, and then - // erase the list of referenced stubs. - SmallVectorImpl &StubList = CurFnStubUses[F]; - for (unsigned i = 0, e = StubList.size(); i != e; ++i) { - void *Stub = StubList[i]; - - // If we already invalidated this stub for this function, continue. - if (StubFnRefs.count(Stub) == 0) - continue; - - SmallPtrSet &FnRefs = StubFnRefs[Stub]; - FnRefs.erase(F); - - // If this function was the last reference to the stub, invalidate the stub - // in the JITResolver. Were there a memory manager deallocateStub routine, - // we could call that at this point too. - if (FnRefs.empty()) { - DEBUG(errs() << "\nJIT: Invalidated Stub at [" << Stub << "]\n"); - StubFnRefs.erase(Stub); - - // Invalidate the stub. If it is a GV stub, update the JIT's global - // mapping for that GV to zero. - GlobalValue *GV = Resolver.invalidateStub(Stub); - if (GV) { - TheJIT->updateGlobalMapping(GV, 0); - } - } - } - CurFnStubUses.erase(F); } -void* JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { +void *JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { if (BufferBegin) return JITCodeEmitter::allocateSpace(Size, Alignment); @@ -1338,7 +998,7 @@ void* JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { return CurBufferPtr; } -void* JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { +void *JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { // Delegate this call through the memory manager. return MemMgr->allocateGlobal(Size, Alignment); } @@ -1350,14 +1010,14 @@ void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { const std::vector &Constants = MCP->getConstants(); if (Constants.empty()) return; - unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); + unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getDataLayout()); unsigned Align = MCP->getConstantPoolAlignment(); ConstantPoolBase = allocateSpace(Size, Align); ConstantPool = MCP; - if (ConstantPoolBase == 0) return; // Buffer overflow. + if (!ConstantPoolBase) return; // Buffer overflow. - DEBUG(errs() << "JIT: Emitted constant pool at [" << ConstantPoolBase + DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase << "] (size: " << Size << ", alignment: " << Align << ")\n"); // Initialize the memory for all of the constant pool entries. @@ -1371,36 +1031,39 @@ void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { ConstPoolAddresses.push_back(CAddr); if (CPE.isMachineConstantPoolEntry()) { // FIXME: add support to lower machine constant pool values into bytes! - llvm_report_error("Initialize memory with machine specific constant pool" + report_fatal_error("Initialize memory with machine specific constant pool" "entry has not been implemented!"); } TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr); - DEBUG(errs() << "JIT: CP" << i << " at [0x"; - errs().write_hex(CAddr) << "]\n"); + DEBUG(dbgs() << "JIT: CP" << i << " at [0x"; + dbgs().write_hex(CAddr) << "]\n"); - const Type *Ty = CPE.Val.ConstVal->getType(); - Offset += TheJIT->getTargetData()->getTypeAllocSize(Ty); + Type *Ty = CPE.Val.ConstVal->getType(); + Offset += TheJIT->getDataLayout()->getTypeAllocSize(Ty); } } void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { if (TheJIT->getJITInfo().hasCustomJumpTables()) return; + if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) + return; const std::vector &JT = MJTI->getJumpTables(); if (JT.empty()) return; - + unsigned NumEntries = 0; for (unsigned i = 0, e = JT.size(); i != e; ++i) NumEntries += JT[i].MBBs.size(); - unsigned EntrySize = MJTI->getEntrySize(); + unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getDataLayout()); // Just allocate space for all the jump tables now. We will fix up the actual // MBB entries in the tables after we emit the code for each block, since then // we will know the final locations of the MBBs in memory. JumpTable = MJTI; - JumpTableBase = allocateSpace(NumEntries * EntrySize, MJTI->getAlignment()); + JumpTableBase = allocateSpace(NumEntries * EntrySize, + MJTI->getEntryAlignment(*TheJIT->getDataLayout())); } void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { @@ -1408,10 +1071,36 @@ void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { return; const std::vector &JT = MJTI->getJumpTables(); - if (JT.empty() || JumpTableBase == 0) return; - - if (TargetMachine::getRelocationModel() == Reloc::PIC_) { - assert(MJTI->getEntrySize() == 4 && "Cross JIT'ing?"); + if (JT.empty() || !JumpTableBase) return; + + + switch (MJTI->getEntryKind()) { + case MachineJumpTableInfo::EK_Inline: + return; + case MachineJumpTableInfo::EK_BlockAddress: { + // EK_BlockAddress - Each entry is a plain address of block, e.g.: + // .word LBB123 + assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == sizeof(void*) && + "Cross JIT'ing?"); + + // For each jump table, map each target in the jump table to the address of + // an emitted MachineBasicBlock. + intptr_t *SlotPtr = (intptr_t*)JumpTableBase; + + for (unsigned i = 0, e = JT.size(); i != e; ++i) { + const std::vector &MBBs = JT[i].MBBs; + // Store the address of the basic block for this jump table slot in the + // memory we allocated for the jump table in 'initJumpTableInfo' + for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) + *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); + } + break; + } + + case MachineJumpTableInfo::EK_Custom32: + case MachineJumpTableInfo::EK_GPRel32BlockAddress: + case MachineJumpTableInfo::EK_LabelDifference32: { + assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == 4&&"Cross JIT'ing?"); // For each jump table, place the offset from the beginning of the table // to the target address. int *SlotPtr = (int*)JumpTableBase; @@ -1423,52 +1112,51 @@ void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { uintptr_t Base = (uintptr_t)SlotPtr; for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); + /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook. *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); } } - } else { - assert(MJTI->getEntrySize() == sizeof(void*) && "Cross JIT'ing?"); - - // For each jump table, map each target in the jump table to the address of - // an emitted MachineBasicBlock. - intptr_t *SlotPtr = (intptr_t*)JumpTableBase; - - for (unsigned i = 0, e = JT.size(); i != e; ++i) { - const std::vector &MBBs = JT[i].MBBs; - // Store the address of the basic block for this jump table slot in the - // memory we allocated for the jump table in 'initJumpTableInfo' - for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) - *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); - } + break; + } + case MachineJumpTableInfo::EK_GPRel64BlockAddress: + llvm_unreachable( + "JT Info emission not implemented for GPRel64BlockAddress yet."); } } -void JITEmitter::startGVStub(const GlobalValue* GV, unsigned StubSize, - unsigned Alignment) { +void JITEmitter::startGVStub(const GlobalValue* GV, + unsigned StubSize, unsigned Alignment) { SavedBufferBegin = BufferBegin; SavedBufferEnd = BufferEnd; SavedCurBufferPtr = CurBufferPtr; - + BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); BufferEnd = BufferBegin+StubSize+1; } -void JITEmitter::startGVStub(const GlobalValue* GV, void *Buffer, - unsigned StubSize) { +void JITEmitter::startGVStub(void *Buffer, unsigned StubSize) { SavedBufferBegin = BufferBegin; SavedBufferEnd = BufferEnd; SavedCurBufferPtr = CurBufferPtr; - + BufferBegin = CurBufferPtr = (uint8_t *)Buffer; BufferEnd = BufferBegin+StubSize+1; } -void *JITEmitter::finishGVStub(const GlobalValue* GV) { +void JITEmitter::finishGVStub() { + assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space."); NumBytes += getCurrentPCOffset(); - std::swap(SavedBufferBegin, BufferBegin); + BufferBegin = SavedBufferBegin; BufferEnd = SavedBufferEnd; CurBufferPtr = SavedCurBufferPtr; - return SavedBufferBegin; +} + +void *JITEmitter::allocIndirectGV(const GlobalValue *GV, + const uint8_t *Buffer, size_t Size, + unsigned Alignment) { + uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment); + memcpy(IndGV, Buffer, Size); + return IndGV; } // getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry @@ -1487,15 +1175,15 @@ uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { const std::vector &JT = JumpTable->getJumpTables(); assert(Index < JT.size() && "Invalid jump table index!"); - + + unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getDataLayout()); + unsigned Offset = 0; - unsigned EntrySize = JumpTable->getEntrySize(); - for (unsigned i = 0; i < Index; ++i) Offset += JT[i].MBBs.size(); - + Offset *= EntrySize; - + return (uintptr_t)((char *)JumpTableBase + Offset); } @@ -1519,19 +1207,6 @@ JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM, return new JITEmitter(jit, JMM, tm); } -// getPointerToNamedFunction - This function is used as a global wrapper to -// JIT::getPointerToNamedFunction for the purpose of resolving symbols when -// bugpoint is debugging the JIT. In that scenario, we are loading an .so and -// need to resolve function(s) that are being mis-codegenerated, so we need to -// resolve their addresses at runtime, and this is the way to do it. -extern "C" { - void *getPointerToNamedFunction(const char *Name) { - if (Function *F = TheJIT->FindFunctionNamed(Name)) - return TheJIT->getPointerToFunction(F); - return TheJIT->getPointerToNamedFunction(Name); - } -} - // getPointerToFunctionOrStub - If the specified function has been // code-gen'd, return a pointer to the function. If not, compile it, or use // a stub to implement lazy compilation if available. @@ -1540,23 +1215,25 @@ void *JIT::getPointerToFunctionOrStub(Function *F) { // If we have already code generated the function, just return the address. if (void *Addr = getPointerToGlobalIfAvailable(F)) return Addr; - + // Get a stub if the target supports it. - assert(isa(JCE) && "Unexpected MCE?"); - JITEmitter *JE = cast(getCodeEmitter()); - return JE->getJITResolver().getFunctionStub(F); + JITEmitter *JE = static_cast(getCodeEmitter()); + return JE->getJITResolver().getLazyFunctionStub(F); } -void JIT::updateFunctionStub(Function *F) { +void JIT::updateFunctionStubUnlocked(Function *F) { // Get the empty stub we generated earlier. - assert(isa(JCE) && "Unexpected MCE?"); - JITEmitter *JE = cast(getCodeEmitter()); - void *Stub = JE->getJITResolver().getFunctionStub(F); + JITEmitter *JE = static_cast(getCodeEmitter()); + void *Stub = JE->getJITResolver().getLazyFunctionStub(F); + void *Addr = getPointerToGlobalIfAvailable(F); + assert(Addr != Stub && "Function must have non-stub address to be updated."); // Tell the target jit info to rewrite the stub at the specified address, // rather than creating a new one. - void *Addr = getPointerToGlobalIfAvailable(F); - getJITInfo().emitFunctionStubAtAddr(F, Addr, Stub, *getCodeEmitter()); + TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout(); + JE->startGVStub(Stub, layout.Size); + getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter()); + JE->finishGVStub(); } /// freeMachineCodeForFunction - release machine code memory for given Function. @@ -1564,9 +1241,8 @@ void JIT::updateFunctionStub(Function *F) { void JIT::freeMachineCodeForFunction(Function *F) { // Delete translation for this from the ExecutionEngine, so it will get // retranslated next time it is used. - updateGlobalMapping(F, 0); + updateGlobalMapping(F, nullptr); // Free the actual memory for the function body and related stuff. - assert(isa(JCE) && "Unexpected MCE?"); - cast(JCE)->deallocateMemForFunction(F); + static_cast(JCE)->deallocateMemForFunction(F); }