-namespace {
- class JITResolver {
- SparcV9CodeEmitter &SparcV9;
- MachineCodeEmitter &MCE;
-
- /// LazyCodeGenMap - Keep track of call sites for functions that are to be
- /// lazily resolved.
- ///
- std::map<uint64_t, Function*> LazyCodeGenMap;
-
- /// LazyResolverMap - Keep track of the lazy resolver created for a
- /// particular function so that we can reuse them if necessary.
- ///
- std::map<Function*, uint64_t> LazyResolverMap;
-
- public:
- enum CallType { ShortCall, FarCall };
-
- private:
- /// We need to keep track of whether we used a simple call or a far call
- /// (many instructions) in sequence. This means we need to keep track of
- /// what type of stub we generate.
- static std::map<uint64_t, CallType> LazyCallFlavor;
-
- public:
- JITResolver(SparcV9CodeEmitter &V9,
- MachineCodeEmitter &mce) : SparcV9(V9), MCE(mce) {}
- uint64_t getLazyResolver(Function *F);
- uint64_t addFunctionReference(uint64_t Address, Function *F);
- void deleteFunctionReference(uint64_t Address);
- void addCallFlavor(uint64_t Address, CallType Flavor) {
- LazyCallFlavor[Address] = Flavor;
- }
-
- // Utility functions for accessing data from static callback
- uint64_t getCurrentPCValue() {
- return MCE.getCurrentPCValue();
- }
- unsigned getBinaryCodeForInstr(MachineInstr &MI) {
- return SparcV9.getBinaryCodeForInstr(MI);
- }
-
- inline void insertFarJumpAtAddr(int64_t Value, uint64_t Addr);
- void insertJumpAtAddr(int64_t Value, uint64_t &Addr);
-
- private:
- uint64_t emitStubForFunction(Function *F);
- static void SaveRegisters(uint64_t DoubleFP[], uint64_t CC[],
- uint64_t Globals[]);
- static void RestoreRegisters(uint64_t DoubleFP[], uint64_t CC[],
- uint64_t Globals[]);
- static void CompilationCallback();
- uint64_t resolveFunctionReference(uint64_t RetAddr);
-
- };
-
- JITResolver *TheJITResolver;
- std::map<uint64_t, JITResolver::CallType> JITResolver::LazyCallFlavor;
-}
-
-/// addFunctionReference - This method is called when we need to emit the
-/// address of a function that has not yet been emitted, so we don't know the
-/// address. Instead, we emit a call to the CompilationCallback method, and
-/// keep track of where we are.
-///
-uint64_t JITResolver::addFunctionReference(uint64_t Address, Function *F) {
- LazyCodeGenMap[Address] = F;
- return (intptr_t)&JITResolver::CompilationCallback;
-}
-
-/// deleteFunctionReference - If we are emitting a far call, we already added a
-/// reference to the function, but it is now incorrect, since the address to the
-/// JIT resolver is too far away to be a simple call instruction. This is used
-/// to remove the address from the map.
-///
-void JITResolver::deleteFunctionReference(uint64_t Address) {
- std::map<uint64_t, Function*>::iterator I = LazyCodeGenMap.find(Address);
- assert(I != LazyCodeGenMap.end() && "Not in map!");
- LazyCodeGenMap.erase(I);
-}
-
-uint64_t JITResolver::resolveFunctionReference(uint64_t RetAddr) {
- std::map<uint64_t, Function*>::iterator I = LazyCodeGenMap.find(RetAddr);
- assert(I != LazyCodeGenMap.end() && "Not in map!");
- Function *F = I->second;
- LazyCodeGenMap.erase(I);
- return MCE.forceCompilationOf(F);
-}
-
-uint64_t JITResolver::getLazyResolver(Function *F) {
- std::map<Function*, uint64_t>::iterator I = LazyResolverMap.lower_bound(F);
- if (I != LazyResolverMap.end() && I->first == F) return I->second;
-
- uint64_t Stub = emitStubForFunction(F);
- LazyResolverMap.insert(I, std::make_pair(F, Stub));
- return Stub;
-}
-
-void JITResolver::insertJumpAtAddr(int64_t JumpTarget, uint64_t &Addr) {
- DEBUG(std::cerr << "Emitting a jump to 0x" << std::hex << JumpTarget << "\n");
-
- // If the target function is close enough to fit into the 19bit disp of
- // BA, we should use this version, as it's much cheaper to generate.
- int64_t BranchTarget = (JumpTarget-Addr) >> 2;
- if (BranchTarget >= (1 << 19) || BranchTarget <= -(1 << 19)) {
- TheJITResolver->insertFarJumpAtAddr(JumpTarget, Addr);
- } else {
- // ba <target>
- MachineInstr *I = BuildMI(V9::BA, 1).addSImm(BranchTarget);
- *((unsigned*)(intptr_t)Addr) = getBinaryCodeForInstr(*I);
- Addr += 4;
- delete I;
-
- // nop
- I = BuildMI(V9::NOP, 0);
- *((unsigned*)(intptr_t)Addr) = getBinaryCodeForInstr(*I);
- delete I;
- }
-}
-
-void JITResolver::insertFarJumpAtAddr(int64_t Target, uint64_t Addr) {
- static const unsigned
- o6 = SparcIntRegClass::o6, g0 = SparcIntRegClass::g0,
- g1 = SparcIntRegClass::g1, g5 = SparcIntRegClass::g5;
-
- MachineInstr* BinaryCode[] = {
- //
- // Get address to branch into %g1, using %g5 as a temporary
- //
- // sethi %uhi(Target), %g5 ;; get upper 22 bits of Target into %g5
- BuildMI(V9::SETHI, 2).addSImm(Target >> 42).addReg(g5),
- // or %g5, %ulo(Target), %g5 ;; get 10 lower bits of upper word into %g5
- BuildMI(V9::ORi, 3).addReg(g5).addSImm((Target >> 32) & 0x03ff).addReg(g5),
- // sllx %g5, 32, %g5 ;; shift those 10 bits to the upper word
- BuildMI(V9::SLLXi6, 3).addReg(g5).addSImm(32).addReg(g5),
- // sethi %hi(Target), %g1 ;; extract bits 10-31 into the dest reg
- BuildMI(V9::SETHI, 2).addSImm((Target >> 10) & 0x03fffff).addReg(g1),
- // or %g5, %g1, %g1 ;; get upper word (in %g5) into %g1
- BuildMI(V9::ORr, 3).addReg(g5).addReg(g1).addReg(g1),
- // or %g1, %lo(Target), %g1 ;; get lowest 10 bits of Target into %g1
- BuildMI(V9::ORi, 3).addReg(g1).addSImm(Target & 0x03ff).addReg(g1),
- // jmpl %g1, %g0, %g0 ;; indirect branch on %g1
- BuildMI(V9::JMPLRETr, 3).addReg(g1).addReg(g0).addReg(g0),
- // nop ;; delay slot
- BuildMI(V9::NOP, 0)
- };
-
- for (unsigned i=0, e=sizeof(BinaryCode)/sizeof(BinaryCode[0]); i!=e; ++i) {
- *((unsigned*)(intptr_t)Addr) = getBinaryCodeForInstr(*BinaryCode[i]);
- delete BinaryCode[i];
- Addr += 4;
- }
-}
-
-void JITResolver::SaveRegisters(uint64_t DoubleFP[], uint64_t CC[],
- uint64_t Globals[]) {
-#if defined(sparc) || defined(__sparc__) || defined(__sparcv9)
-
- __asm__ __volatile__ (// Save condition-code registers
- "stx %%fsr, %0;\n\t"
- "rd %%fprs, %1;\n\t"
- "rd %%ccr, %2;\n\t"
- : "=m"(CC[0]), "=r"(CC[1]), "=r"(CC[2]));
-
- __asm__ __volatile__ (// Save globals g1 and g5
- "stx %%g1, %0;\n\t"
- "stx %%g5, %0;\n\t"
- : "=m"(Globals[0]), "=m"(Globals[1]));
-
- // GCC says: `asm' only allows up to thirty parameters!
- __asm__ __volatile__ (// Save Single/Double FP registers, part 1
- "std %%f0, %0;\n\t" "std %%f2, %1;\n\t"
- "std %%f4, %2;\n\t" "std %%f6, %3;\n\t"
- "std %%f8, %4;\n\t" "std %%f10, %5;\n\t"
- "std %%f12, %6;\n\t" "std %%f14, %7;\n\t"
- "std %%f16, %8;\n\t" "std %%f18, %9;\n\t"
- "std %%f20, %10;\n\t" "std %%f22, %11;\n\t"
- "std %%f24, %12;\n\t" "std %%f26, %13;\n\t"
- "std %%f28, %14;\n\t" "std %%f30, %15;\n\t"
- : "=m"(DoubleFP[ 0]), "=m"(DoubleFP[ 1]),
- "=m"(DoubleFP[ 2]), "=m"(DoubleFP[ 3]),
- "=m"(DoubleFP[ 4]), "=m"(DoubleFP[ 5]),
- "=m"(DoubleFP[ 6]), "=m"(DoubleFP[ 7]),
- "=m"(DoubleFP[ 8]), "=m"(DoubleFP[ 9]),
- "=m"(DoubleFP[10]), "=m"(DoubleFP[11]),
- "=m"(DoubleFP[12]), "=m"(DoubleFP[13]),
- "=m"(DoubleFP[14]), "=m"(DoubleFP[15]));
-
- __asm__ __volatile__ (// Save Double FP registers, part 2
- "std %%f32, %0;\n\t" "std %%f34, %1;\n\t"
- "std %%f36, %2;\n\t" "std %%f38, %3;\n\t"
- "std %%f40, %4;\n\t" "std %%f42, %5;\n\t"
- "std %%f44, %6;\n\t" "std %%f46, %7;\n\t"
- "std %%f48, %8;\n\t" "std %%f50, %9;\n\t"
- "std %%f52, %10;\n\t" "std %%f54, %11;\n\t"
- "std %%f56, %12;\n\t" "std %%f58, %13;\n\t"
- "std %%f60, %14;\n\t" "std %%f62, %15;\n\t"
- : "=m"(DoubleFP[16]), "=m"(DoubleFP[17]),
- "=m"(DoubleFP[18]), "=m"(DoubleFP[19]),
- "=m"(DoubleFP[20]), "=m"(DoubleFP[21]),
- "=m"(DoubleFP[22]), "=m"(DoubleFP[23]),
- "=m"(DoubleFP[24]), "=m"(DoubleFP[25]),
- "=m"(DoubleFP[26]), "=m"(DoubleFP[27]),
- "=m"(DoubleFP[28]), "=m"(DoubleFP[29]),
- "=m"(DoubleFP[30]), "=m"(DoubleFP[31]));
-#endif
-}
-
-
-void JITResolver::RestoreRegisters(uint64_t DoubleFP[], uint64_t CC[],
- uint64_t Globals[])
-{
-#if defined(sparc) || defined(__sparc__) || defined(__sparcv9)
-
- __asm__ __volatile__ (// Restore condition-code registers
- "ldx %0, %%fsr;\n\t"
- "wr %1, 0, %%fprs;\n\t"
- "wr %2, 0, %%ccr;\n\t"
- :: "m"(CC[0]), "r"(CC[1]), "r"(CC[2]));
-
- __asm__ __volatile__ (// Restore globals g1 and g5
- "ldx %0, %%g1;\n\t"
- "ldx %0, %%g5;\n\t"
- :: "m"(Globals[0]), "m"(Globals[1]));
-
- // GCC says: `asm' only allows up to thirty parameters!
- __asm__ __volatile__ (// Restore Single/Double FP registers, part 1
- "ldd %0, %%f0;\n\t" "ldd %1, %%f2;\n\t"
- "ldd %2, %%f4;\n\t" "ldd %3, %%f6;\n\t"
- "ldd %4, %%f8;\n\t" "ldd %5, %%f10;\n\t"
- "ldd %6, %%f12;\n\t" "ldd %7, %%f14;\n\t"
- "ldd %8, %%f16;\n\t" "ldd %9, %%f18;\n\t"
- "ldd %10, %%f20;\n\t" "ldd %11, %%f22;\n\t"
- "ldd %12, %%f24;\n\t" "ldd %13, %%f26;\n\t"
- "ldd %14, %%f28;\n\t" "ldd %15, %%f30;\n\t"
- :: "m"(DoubleFP[0]), "m"(DoubleFP[1]),
- "m"(DoubleFP[2]), "m"(DoubleFP[3]),
- "m"(DoubleFP[4]), "m"(DoubleFP[5]),
- "m"(DoubleFP[6]), "m"(DoubleFP[7]),
- "m"(DoubleFP[8]), "m"(DoubleFP[9]),
- "m"(DoubleFP[10]), "m"(DoubleFP[11]),
- "m"(DoubleFP[12]), "m"(DoubleFP[13]),
- "m"(DoubleFP[14]), "m"(DoubleFP[15]));
-
- __asm__ __volatile__ (// Restore Double FP registers, part 2
- "ldd %0, %%f32;\n\t" "ldd %1, %%f34;\n\t"
- "ldd %2, %%f36;\n\t" "ldd %3, %%f38;\n\t"
- "ldd %4, %%f40;\n\t" "ldd %5, %%f42;\n\t"
- "ldd %6, %%f44;\n\t" "ldd %7, %%f46;\n\t"
- "ldd %8, %%f48;\n\t" "ldd %9, %%f50;\n\t"
- "ldd %10, %%f52;\n\t" "ldd %11, %%f54;\n\t"
- "ldd %12, %%f56;\n\t" "ldd %13, %%f58;\n\t"
- "ldd %14, %%f60;\n\t" "ldd %15, %%f62;\n\t"
- :: "m"(DoubleFP[16]), "m"(DoubleFP[17]),
- "m"(DoubleFP[18]), "m"(DoubleFP[19]),
- "m"(DoubleFP[20]), "m"(DoubleFP[21]),
- "m"(DoubleFP[22]), "m"(DoubleFP[23]),
- "m"(DoubleFP[24]), "m"(DoubleFP[25]),
- "m"(DoubleFP[26]), "m"(DoubleFP[27]),
- "m"(DoubleFP[28]), "m"(DoubleFP[29]),
- "m"(DoubleFP[30]), "m"(DoubleFP[31]));
-#endif
-}
-
-void JITResolver::CompilationCallback() {
- // Local space to save the registers
- uint64_t DoubleFP[32];
- uint64_t CC[3];
- uint64_t Globals[2];
-
- SaveRegisters(DoubleFP, CC, Globals);
- ++CallbackCalls;
-
- uint64_t CameFrom = (uint64_t)(intptr_t)__builtin_return_address(0);
- uint64_t CameFrom1 = (uint64_t)(intptr_t)__builtin_return_address(1);
- int64_t Target = (int64_t)TheJITResolver->resolveFunctionReference(CameFrom);
- DEBUG(std::cerr << "In callback! Addr=0x" << std::hex << CameFrom << "\n");
- register int64_t returnAddr = 0;
-#if defined(sparc) || defined(__sparc__) || defined(__sparcv9)
- __asm__ __volatile__ ("add %%i7, %%g0, %0" : "=r" (returnAddr) : );
- DEBUG(std::cerr << "Read i7 (return addr) = "
- << std::hex << returnAddr << ", value: "
- << std::hex << *(unsigned*)returnAddr << "\n");
-#endif
-
- // If we can rewrite the ORIGINAL caller, we eliminate the whole need for a
- // trampoline function stub!!
- unsigned OrigCallInst = *((unsigned*)(intptr_t)CameFrom1);
- int64_t OrigTarget = (Target-CameFrom1) >> 2;
- if ((OrigCallInst & (1 << 30)) &&
- (OrigTarget <= (1 << 30) && OrigTarget >= -(1 << 30)))
- {
- // The original call instruction was CALL <immed>, which means we can
- // overwrite it directly, since the offset will fit into 30 bits
- MachineInstr *C = BuildMI(V9::CALL, 1).addSImm(OrigTarget);
- *((unsigned*)(intptr_t)CameFrom1)=TheJITResolver->getBinaryCodeForInstr(*C);
- delete C;
- ++OverwrittenCalls;
- } else {
- ++UnmodifiedCalls;
- }
-
- // Rewrite the call target so that we don't fault every time we execute it.
- //
-
- static const unsigned o6 = SparcIntRegClass::o6;
-
- // Subtract enough to overwrite up to the 'save' instruction
- // This depends on whether we made a short call (1 instruction) or the
- // farCall (7 instructions)
- uint64_t Offset = (LazyCallFlavor[CameFrom] == ShortCall) ? 4 : 28;
- uint64_t CodeBegin = CameFrom - Offset;
-
- // FIXME FIXME FIXME FIXME: __builtin_frame_address doesn't work if frame
- // pointer elimination has been performed. Having a variable sized alloca
- // disables frame pointer elimination currently, even if it's dead. This is
- // a gross hack.
- alloca(42+Offset);
- // FIXME FIXME FIXME FIXME
-
- // Make sure that what we're about to overwrite is indeed "save"
- MachineInstr *SV =BuildMI(V9::SAVEi, 3).addReg(o6).addSImm(-192).addReg(o6);
- unsigned SaveInst = TheJITResolver->getBinaryCodeForInstr(*SV);
- delete SV;
- unsigned CodeInMem = *(unsigned*)(intptr_t)CodeBegin;
- if (CodeInMem != SaveInst) {
- std::cerr << "About to overwrite smthg not a save instr!";
- abort();
- }
- // Overwrite it
- TheJITResolver->insertJumpAtAddr(Target, CodeBegin);
-
- // Flush the I-Cache: FLUSH clears out a doubleword at a given address
- // Self-modifying code MUST clear out the I-Cache to be portable
-#if defined(sparc) || defined(__sparc__) || defined(__sparcv9)
- for (int i = -Offset, e = 32-((int64_t)Offset); i < e; i += 8)
- __asm__ __volatile__ ("flush %%i7 + %0" : : "r" (i));
-#endif
-
- // Change the return address to re-execute the restore, then the jump.
- DEBUG(std::cerr << "Callback returning to: 0x"
- << std::hex << (CameFrom-Offset-12) << "\n");
-#if defined(sparc) || defined(__sparc__) || defined(__sparcv9)
- __asm__ __volatile__ ("sub %%i7, %0, %%i7" : : "r" (Offset+12));
-#endif
-
- RestoreRegisters(DoubleFP, CC, Globals);
-}
-
-/// emitStubForFunction - This method is used by the JIT when it needs to emit
-/// the address of a function for a function whose code has not yet been
-/// generated. In order to do this, it generates a stub which jumps to the lazy
-/// function compiler, which will eventually get fixed to call the function
-/// directly.
-///
-uint64_t JITResolver::emitStubForFunction(Function *F) {
- MCE.startFunctionStub(*F, 44);
-
- DEBUG(std::cerr << "Emitting stub at addr: 0x"
- << std::hex << MCE.getCurrentPCValue() << "\n");
-
- unsigned o6 = SparcIntRegClass::o6, g0 = SparcIntRegClass::g0;
-
- // restore %g0, 0, %g0
- MachineInstr *R = BuildMI(V9::RESTOREi, 3).addMReg(g0).addSImm(0)
- .addMReg(g0, MOTy::Def);
- SparcV9.emitWord(SparcV9.getBinaryCodeForInstr(*R));
- delete R;
-
- // save %sp, -192, %sp
- MachineInstr *SV = BuildMI(V9::SAVEi, 3).addReg(o6).addSImm(-192).addReg(o6);
- SparcV9.emitWord(SparcV9.getBinaryCodeForInstr(*SV));
- delete SV;
-
- int64_t CurrPC = MCE.getCurrentPCValue();
- int64_t Addr = (int64_t)addFunctionReference(CurrPC, F);
- int64_t CallTarget = (Addr-CurrPC) >> 2;
- if (CallTarget >= (1 << 29) || CallTarget <= -(1 << 29)) {
- // Since this is a far call, the actual address of the call is shifted
- // by the number of instructions it takes to calculate the exact address
- deleteFunctionReference(CurrPC);
- SparcV9.emitFarCall(Addr, F);
- } else {
- // call CallTarget ;; invoke the callback
- MachineInstr *Call = BuildMI(V9::CALL, 1).addSImm(CallTarget);
- SparcV9.emitWord(SparcV9.getBinaryCodeForInstr(*Call));
- delete Call;
-
- // nop ;; call delay slot
- MachineInstr *Nop = BuildMI(V9::NOP, 0);
- SparcV9.emitWord(SparcV9.getBinaryCodeForInstr(*Nop));
- delete Nop;
-
- addCallFlavor(CurrPC, ShortCall);
- }
-
- SparcV9.emitWord(0xDEADBEEF); // marker so that we know it's really a stub
- return (intptr_t)MCE.finishFunctionStub(*F)+4; /* 1 instr past the restore */
-}
-