From 0261d795f83a45dd53d82e511ae672d6d1f4e298 Mon Sep 17 00:00:00 2001 From: Jeffrey Yasskin Date: Mon, 23 Nov 2009 22:49:00 +0000 Subject: [PATCH] Allow more than one stub to be being generated at the same time. It's probably better in the long run to replace the indirect-GlobalVariable system. That'll be done after a subsequent patch. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@89708 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/CodeGen/JITCodeEmitter.h | 28 ++++++++------ include/llvm/CodeGen/MachineCodeEmitter.h | 44 +++++++++++++++++----- lib/ExecutionEngine/JIT/JITEmitter.cpp | 45 ++++++++--------------- lib/Target/ARM/ARMJITInfo.cpp | 14 ++++--- lib/Target/Alpha/AlphaJITInfo.cpp | 5 ++- lib/Target/PowerPC/PPCJITInfo.cpp | 9 +++-- lib/Target/X86/X86JITInfo.cpp | 25 +++++++------ 7 files changed, 97 insertions(+), 73 deletions(-) diff --git a/include/llvm/CodeGen/JITCodeEmitter.h b/include/llvm/CodeGen/JITCodeEmitter.h index 792fb599233..ea3e59beab0 100644 --- a/include/llvm/CodeGen/JITCodeEmitter.h +++ b/include/llvm/CodeGen/JITCodeEmitter.h @@ -68,23 +68,29 @@ public: /// virtual bool finishFunction(MachineFunction &F) = 0; - /// startGVStub - This callback is invoked when the JIT needs the - /// address of a GV (e.g. function) that has not been code generated yet. - /// The StubSize specifies the total size required by the stub. + /// startGVStub - This callback is invoked when the JIT needs the address of a + /// GV (e.g. function) that has not been code generated yet. The StubSize + /// specifies the total size required by the stub. The BufferState must be + /// passed to finishGVStub, and start/finish pairs with the same BufferState + /// must be properly nested. /// - virtual void startGVStub(const GlobalValue* GV, unsigned StubSize, - unsigned Alignment = 1) = 0; + virtual void startGVStub(BufferState &BS, const GlobalValue* GV, + unsigned StubSize, unsigned Alignment = 1) = 0; - /// startGVStub - This callback is invoked when the JIT needs the address of a + /// startGVStub - This callback is invoked when the JIT needs the address of a /// GV (e.g. function) that has not been code generated yet. Buffer points to - /// memory already allocated for this stub. + /// memory already allocated for this stub. The BufferState must be passed to + /// finishGVStub, and start/finish pairs with the same BufferState must be + /// properly nested. /// - virtual void startGVStub(const GlobalValue* GV, void *Buffer, + virtual void startGVStub(BufferState &BS, void *Buffer, unsigned StubSize) = 0; - - /// finishGVStub - This callback is invoked to terminate a GV stub. + + /// finishGVStub - This callback is invoked to terminate a GV stub and returns + /// the start address of the stub. The BufferState must first have been + /// passed to startGVStub. /// - virtual void *finishGVStub(const GlobalValue* F) = 0; + virtual void *finishGVStub(BufferState &BS) = 0; /// emitByte - This callback is invoked when a byte needs to be written to the /// output stream. diff --git a/include/llvm/CodeGen/MachineCodeEmitter.h b/include/llvm/CodeGen/MachineCodeEmitter.h index c55a9e65e45..791db003ead 100644 --- a/include/llvm/CodeGen/MachineCodeEmitter.h +++ b/include/llvm/CodeGen/MachineCodeEmitter.h @@ -48,17 +48,41 @@ class Function; /// occurred, more memory is allocated, and we reemit the code into it. /// class MachineCodeEmitter { +public: + class BufferState { + friend class MachineCodeEmitter; + /// BufferBegin/BufferEnd - Pointers to the start and end of the memory + /// allocated for this code buffer. + uint8_t *BufferBegin, *BufferEnd; + + /// CurBufferPtr - Pointer to the next byte of memory to fill when emitting + /// code. This is guranteed to be in the range [BufferBegin,BufferEnd]. If + /// this pointer is at BufferEnd, it will never move due to code emission, + /// and all code emission requests will be ignored (this is the buffer + /// overflow condition). + uint8_t *CurBufferPtr; + public: + BufferState() : BufferBegin(NULL), BufferEnd(NULL), CurBufferPtr(NULL) {} + }; + protected: - /// BufferBegin/BufferEnd - Pointers to the start and end of the memory - /// allocated for this code buffer. - uint8_t *BufferBegin, *BufferEnd; - - /// CurBufferPtr - Pointer to the next byte of memory to fill when emitting - /// code. This is guranteed to be in the range [BufferBegin,BufferEnd]. If - /// this pointer is at BufferEnd, it will never move due to code emission, and - /// all code emission requests will be ignored (this is the buffer overflow - /// condition). - uint8_t *CurBufferPtr; + /// These have the same meanings as the fields in BufferState + uint8_t *BufferBegin, *BufferEnd, *CurBufferPtr; + + /// Save or restore the current buffer state. The BufferState objects must be + /// used as a stack. + void SaveStateTo(BufferState &BS) { + assert(BS.BufferBegin == NULL && + "Can't save state into the same BufferState twice."); + BS.BufferBegin = BufferBegin; + BS.BufferEnd = BufferEnd; + BS.CurBufferPtr = CurBufferPtr; + } + void RestoreStateFrom(BufferState &BS) { + BufferBegin = BS.BufferBegin; + BufferEnd = BS.BufferEnd; + CurBufferPtr = BS.CurBufferPtr; + } public: virtual ~MachineCodeEmitter() {} diff --git a/lib/ExecutionEngine/JIT/JITEmitter.cpp b/lib/ExecutionEngine/JIT/JITEmitter.cpp index aabab98dcca..f34ae00f8bc 100644 --- a/lib/ExecutionEngine/JIT/JITEmitter.cpp +++ b/lib/ExecutionEngine/JIT/JITEmitter.cpp @@ -268,10 +268,6 @@ namespace { class JITEmitter : public JITCodeEmitter { JITMemoryManager *MemMgr; - // When outputting a function stub in the context of some other function, we - // save BufferBegin/BufferEnd/CurBufferPtr here. - uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; - // When reattempting to JIT a function after running out of space, we store // the estimated size of the function we're trying to JIT here, so we can // ask the memory manager for at least this much space. When we @@ -397,11 +393,11 @@ namespace { void initJumpTableInfo(MachineJumpTableInfo *MJTI); void emitJumpTableInfo(MachineJumpTableInfo *MJTI); - virtual void startGVStub(const GlobalValue* GV, unsigned StubSize, - unsigned Alignment = 1); - virtual void startGVStub(const GlobalValue* GV, void *Buffer, + virtual void startGVStub(BufferState &BS, const GlobalValue* GV, + unsigned StubSize, unsigned Alignment = 1); + virtual void startGVStub(BufferState &BS, void *Buffer, unsigned StubSize); - virtual void* finishGVStub(const GlobalValue *GV); + virtual void* finishGVStub(BufferState &BS); /// allocateSpace - Reserves space in the current block if any, or /// allocate a new one of the given size. @@ -1207,9 +1203,8 @@ bool JITEmitter::finishFunction(MachineFunction &F) { if (DwarfExceptionHandling || JITEmitDebugInfo) { uintptr_t ActualSize = 0; - SavedBufferBegin = BufferBegin; - SavedBufferEnd = BufferEnd; - SavedCurBufferPtr = CurBufferPtr; + BufferState BS; + SaveStateTo(BS); if (MemMgr->NeedsExactSize()) { ActualSize = DE->GetDwarfTableSizeInBytes(F, *this, FnStart, FnEnd); @@ -1225,9 +1220,7 @@ bool JITEmitter::finishFunction(MachineFunction &F) { MemMgr->endExceptionTable(F.getFunction(), BufferBegin, CurBufferPtr, FrameRegister); uint8_t *EhEnd = CurBufferPtr; - BufferBegin = SavedBufferBegin; - BufferEnd = SavedBufferEnd; - CurBufferPtr = SavedCurBufferPtr; + RestoreStateFrom(BS); if (DwarfExceptionHandling) { TheJIT->RegisterTable(FrameRegister); @@ -1433,32 +1426,26 @@ void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { } } -void JITEmitter::startGVStub(const GlobalValue* GV, unsigned StubSize, - unsigned Alignment) { - SavedBufferBegin = BufferBegin; - SavedBufferEnd = BufferEnd; - SavedCurBufferPtr = CurBufferPtr; +void JITEmitter::startGVStub(BufferState &BS, const GlobalValue* GV, + unsigned StubSize, unsigned Alignment) { + SaveStateTo(BS); BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); BufferEnd = BufferBegin+StubSize+1; } -void JITEmitter::startGVStub(const GlobalValue* GV, void *Buffer, - unsigned StubSize) { - SavedBufferBegin = BufferBegin; - SavedBufferEnd = BufferEnd; - SavedCurBufferPtr = CurBufferPtr; +void JITEmitter::startGVStub(BufferState &BS, void *Buffer, unsigned StubSize) { + SaveStateTo(BS); BufferBegin = CurBufferPtr = (uint8_t *)Buffer; BufferEnd = BufferBegin+StubSize+1; } -void *JITEmitter::finishGVStub(const GlobalValue* GV) { +void *JITEmitter::finishGVStub(BufferState &BS) { NumBytes += getCurrentPCOffset(); - std::swap(SavedBufferBegin, BufferBegin); - BufferEnd = SavedBufferEnd; - CurBufferPtr = SavedCurBufferPtr; - return SavedBufferBegin; + void *Result = BufferBegin; + RestoreStateFrom(BS); + return Result; } // getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry diff --git a/lib/Target/ARM/ARMJITInfo.cpp b/lib/Target/ARM/ARMJITInfo.cpp index bdddcd67fab..7031640471f 100644 --- a/lib/Target/ARM/ARMJITInfo.cpp +++ b/lib/Target/ARM/ARMJITInfo.cpp @@ -139,7 +139,8 @@ ARMJITInfo::getLazyResolverFunction(JITCompilerFn F) { void *ARMJITInfo::emitGlobalValueIndirectSym(const GlobalValue *GV, void *Ptr, JITCodeEmitter &JCE) { - JCE.startGVStub(GV, 4, 4); + MachineCodeEmitter::BufferState BS; + JCE.startGVStub(BS, GV, 4, 4); intptr_t Addr = (intptr_t)JCE.getCurrentPCValue(); if (!sys::Memory::setRangeWritable((void*)Addr, 4)) { llvm_unreachable("ERROR: Unable to mark indirect symbol writable"); @@ -148,13 +149,14 @@ void *ARMJITInfo::emitGlobalValueIndirectSym(const GlobalValue *GV, void *Ptr, if (!sys::Memory::setRangeExecutable((void*)Addr, 4)) { llvm_unreachable("ERROR: Unable to mark indirect symbol executable"); } - void *PtrAddr = JCE.finishGVStub(GV); + void *PtrAddr = JCE.finishGVStub(BS); addIndirectSymAddr(Ptr, (intptr_t)PtrAddr); return PtrAddr; } void *ARMJITInfo::emitFunctionStub(const Function* F, void *Fn, JITCodeEmitter &JCE) { + MachineCodeEmitter::BufferState BS; // If this is just a call to an external function, emit a branch instead of a // call. The code is the same except for one bit of the last instruction. if (Fn != (void*)(intptr_t)ARMCompilationCallback) { @@ -172,7 +174,7 @@ void *ARMJITInfo::emitFunctionStub(const Function* F, void *Fn, errs() << "JIT: Stub emitted at [" << LazyPtr << "] for external function at '" << Fn << "'\n"); } - JCE.startGVStub(F, 16, 4); + JCE.startGVStub(BS, F, 16, 4); intptr_t Addr = (intptr_t)JCE.getCurrentPCValue(); if (!sys::Memory::setRangeWritable((void*)Addr, 16)) { llvm_unreachable("ERROR: Unable to mark stub writable"); @@ -187,7 +189,7 @@ void *ARMJITInfo::emitFunctionStub(const Function* F, void *Fn, } } else { // The stub is 8-byte size and 4-aligned. - JCE.startGVStub(F, 8, 4); + JCE.startGVStub(BS, F, 8, 4); intptr_t Addr = (intptr_t)JCE.getCurrentPCValue(); if (!sys::Memory::setRangeWritable((void*)Addr, 8)) { llvm_unreachable("ERROR: Unable to mark stub writable"); @@ -207,7 +209,7 @@ void *ARMJITInfo::emitFunctionStub(const Function* F, void *Fn, // // Branch and link to the compilation callback. // The stub is 16-byte size and 4-byte aligned. - JCE.startGVStub(F, 16, 4); + JCE.startGVStub(BS, F, 16, 4); intptr_t Addr = (intptr_t)JCE.getCurrentPCValue(); if (!sys::Memory::setRangeWritable((void*)Addr, 16)) { llvm_unreachable("ERROR: Unable to mark stub writable"); @@ -228,7 +230,7 @@ void *ARMJITInfo::emitFunctionStub(const Function* F, void *Fn, } } - return JCE.finishGVStub(F); + return JCE.finishGVStub(BS); } intptr_t ARMJITInfo::resolveRelocDestAddr(MachineRelocation *MR) const { diff --git a/lib/Target/Alpha/AlphaJITInfo.cpp b/lib/Target/Alpha/AlphaJITInfo.cpp index d32813552f0..4e59833953b 100644 --- a/lib/Target/Alpha/AlphaJITInfo.cpp +++ b/lib/Target/Alpha/AlphaJITInfo.cpp @@ -192,15 +192,16 @@ extern "C" { void *AlphaJITInfo::emitFunctionStub(const Function* F, void *Fn, JITCodeEmitter &JCE) { + MachineCodeEmitter::BufferState BS; //assert(Fn == AlphaCompilationCallback && "Where are you going?\n"); //Do things in a stupid slow way! - JCE.startGVStub(F, 19*4); + JCE.startGVStub(BS, F, 19*4); void* Addr = (void*)(intptr_t)JCE.getCurrentPCValue(); for (int x = 0; x < 19; ++ x) JCE.emitWordLE(0); EmitBranchToAt(Addr, Fn); DEBUG(errs() << "Emitting Stub to " << Fn << " at [" << Addr << "]\n"); - return JCE.finishGVStub(F); + return JCE.finishGVStub(BS); } TargetJITInfo::LazyResolverFn diff --git a/lib/Target/PowerPC/PPCJITInfo.cpp b/lib/Target/PowerPC/PPCJITInfo.cpp index ef25d92f719..ddbb3265700 100644 --- a/lib/Target/PowerPC/PPCJITInfo.cpp +++ b/lib/Target/PowerPC/PPCJITInfo.cpp @@ -330,11 +330,12 @@ extern "C" void sys_icache_invalidate(const void *Addr, size_t len); void *PPCJITInfo::emitFunctionStub(const Function* F, void *Fn, JITCodeEmitter &JCE) { + MachineCodeEmitter::BufferState BS; // If this is just a call to an external function, emit a branch instead of a // call. The code is the same except for one bit of the last instruction. if (Fn != (void*)(intptr_t)PPC32CompilationCallback && Fn != (void*)(intptr_t)PPC64CompilationCallback) { - JCE.startGVStub(F, 7*4); + JCE.startGVStub(BS, F, 7*4); intptr_t Addr = (intptr_t)JCE.getCurrentPCValue(); JCE.emitWordBE(0); JCE.emitWordBE(0); @@ -345,10 +346,10 @@ void *PPCJITInfo::emitFunctionStub(const Function* F, void *Fn, JCE.emitWordBE(0); EmitBranchToAt(Addr, (intptr_t)Fn, false, is64Bit); sys::Memory::InvalidateInstructionCache((void*)Addr, 7*4); - return JCE.finishGVStub(F); + return JCE.finishGVStub(BS); } - JCE.startGVStub(F, 10*4); + JCE.startGVStub(BS, F, 10*4); intptr_t Addr = (intptr_t)JCE.getCurrentPCValue(); if (is64Bit) { JCE.emitWordBE(0xf821ffb1); // stdu r1,-80(r1) @@ -373,7 +374,7 @@ void *PPCJITInfo::emitFunctionStub(const Function* F, void *Fn, JCE.emitWordBE(0); EmitBranchToAt(BranchAddr, (intptr_t)Fn, true, is64Bit); sys::Memory::InvalidateInstructionCache((void*)Addr, 10*4); - return JCE.finishGVStub(F); + return JCE.finishGVStub(BS); } diff --git a/lib/Target/X86/X86JITInfo.cpp b/lib/Target/X86/X86JITInfo.cpp index 0792bdd4dd2..a14c155f174 100644 --- a/lib/Target/X86/X86JITInfo.cpp +++ b/lib/Target/X86/X86JITInfo.cpp @@ -426,19 +426,21 @@ X86JITInfo::X86JITInfo(X86TargetMachine &tm) : TM(tm) { void *X86JITInfo::emitGlobalValueIndirectSym(const GlobalValue* GV, void *ptr, JITCodeEmitter &JCE) { + MachineCodeEmitter::BufferState BS; #if defined (X86_64_JIT) - JCE.startGVStub(GV, 8, 8); + JCE.startGVStub(BS, GV, 8, 8); JCE.emitWordLE((unsigned)(intptr_t)ptr); JCE.emitWordLE((unsigned)(((intptr_t)ptr) >> 32)); #else - JCE.startGVStub(GV, 4, 4); + JCE.startGVStub(BS, GV, 4, 4); JCE.emitWordLE((intptr_t)ptr); #endif - return JCE.finishGVStub(GV); + return JCE.finishGVStub(BS); } void *X86JITInfo::emitFunctionStub(const Function* F, void *Fn, JITCodeEmitter &JCE) { + MachineCodeEmitter::BufferState BS; // Note, we cast to intptr_t here to silence a -pedantic warning that // complains about casting a function pointer to a normal pointer. #if defined (X86_32_JIT) && !defined (_MSC_VER) @@ -449,7 +451,7 @@ void *X86JITInfo::emitFunctionStub(const Function* F, void *Fn, #endif if (NotCC) { #if defined (X86_64_JIT) - JCE.startGVStub(F, 13, 4); + JCE.startGVStub(BS, F, 13, 4); JCE.emitByte(0x49); // REX prefix JCE.emitByte(0xB8+2); // movabsq r10 JCE.emitWordLE((unsigned)(intptr_t)Fn); @@ -458,15 +460,15 @@ void *X86JITInfo::emitFunctionStub(const Function* F, void *Fn, JCE.emitByte(0xFF); // jmpq *r10 JCE.emitByte(2 | (4 << 3) | (3 << 6)); #else - JCE.startGVStub(F, 5, 4); + JCE.startGVStub(BS, F, 5, 4); JCE.emitByte(0xE9); JCE.emitWordLE((intptr_t)Fn-JCE.getCurrentPCValue()-4); #endif - return JCE.finishGVStub(F); + return JCE.finishGVStub(BS); } #if defined (X86_64_JIT) - JCE.startGVStub(F, 14, 4); + JCE.startGVStub(BS, F, 14, 4); JCE.emitByte(0x49); // REX prefix JCE.emitByte(0xB8+2); // movabsq r10 JCE.emitWordLE((unsigned)(intptr_t)Fn); @@ -475,7 +477,7 @@ void *X86JITInfo::emitFunctionStub(const Function* F, void *Fn, JCE.emitByte(0xFF); // callq *r10 JCE.emitByte(2 | (2 << 3) | (3 << 6)); #else - JCE.startGVStub(F, 6, 4); + JCE.startGVStub(BS, F, 6, 4); JCE.emitByte(0xE8); // Call with 32 bit pc-rel destination... JCE.emitWordLE((intptr_t)Fn-JCE.getCurrentPCValue()-4); @@ -485,14 +487,15 @@ void *X86JITInfo::emitFunctionStub(const Function* F, void *Fn, // initialize the buffer with garbage, which means it may follow a // noreturn function call, confusing X86CompilationCallback2. PR 4929. JCE.emitByte(0xCE); // Interrupt - Just a marker identifying the stub! - return JCE.finishGVStub(F); + return JCE.finishGVStub(BS); } void X86JITInfo::emitFunctionStubAtAddr(const Function* F, void *Fn, void *Stub, JITCodeEmitter &JCE) { + MachineCodeEmitter::BufferState BS; // Note, we cast to intptr_t here to silence a -pedantic warning that // complains about casting a function pointer to a normal pointer. - JCE.startGVStub(F, Stub, 5); + JCE.startGVStub(BS, Stub, 5); JCE.emitByte(0xE9); #if defined (X86_64_JIT) && !defined (NDEBUG) // Yes, we need both of these casts, or some broken versions of GCC (4.2.4) @@ -502,7 +505,7 @@ void X86JITInfo::emitFunctionStubAtAddr(const Function* F, void *Fn, void *Stub, && "PIC displacement does not fit in displacement field!"); #endif JCE.emitWordLE((intptr_t)Fn-JCE.getCurrentPCValue()-4); - JCE.finishGVStub(F); + JCE.finishGVStub(BS); } /// getPICJumpTableEntry - Returns the value of the jumptable entry for the -- 2.34.1