1 //===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the X86 implementation of TargetFrameLowering class.
12 //===----------------------------------------------------------------------===//
14 #include "X86FrameLowering.h"
15 #include "X86InstrBuilder.h"
16 #include "X86InstrInfo.h"
17 #include "X86MachineFunctionInfo.h"
18 #include "X86Subtarget.h"
19 #include "X86TargetMachine.h"
20 #include "llvm/ADT/SmallSet.h"
21 #include "llvm/CodeGen/MachineFrameInfo.h"
22 #include "llvm/CodeGen/MachineFunction.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineModuleInfo.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/IR/DataLayout.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/MC/MCAsmInfo.h"
29 #include "llvm/MC/MCSymbol.h"
30 #include "llvm/Support/CommandLine.h"
31 #include "llvm/Target/TargetOptions.h"
35 // FIXME: completely move here.
36 extern cl::opt<bool> ForceStackAlign;
38 bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
39 return !MF.getFrameInfo()->hasVarSizedObjects();
42 /// hasFP - Return true if the specified function should have a dedicated frame
43 /// pointer register. This is true if the function has variable sized allocas
44 /// or if frame pointer elimination is disabled.
45 bool X86FrameLowering::hasFP(const MachineFunction &MF) const {
46 const MachineFrameInfo *MFI = MF.getFrameInfo();
47 const MachineModuleInfo &MMI = MF.getMMI();
48 const TargetRegisterInfo *RegInfo = TM.getRegisterInfo();
50 return (MF.getTarget().Options.DisableFramePointerElim(MF) ||
51 RegInfo->needsStackRealignment(MF) ||
52 MFI->hasVarSizedObjects() ||
53 MFI->isFrameAddressTaken() || MFI->hasInlineAsmWithSPAdjust() ||
54 MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() ||
55 MMI.callsUnwindInit() || MMI.callsEHReturn());
58 static unsigned getSUBriOpcode(unsigned IsLP64, int64_t Imm) {
62 return X86::SUB64ri32;
70 static unsigned getADDriOpcode(unsigned IsLP64, int64_t Imm) {
74 return X86::ADD64ri32;
82 static unsigned getLEArOpcode(unsigned IsLP64) {
83 return IsLP64 ? X86::LEA64r : X86::LEA32r;
86 /// findDeadCallerSavedReg - Return a caller-saved register that isn't live
87 /// when it reaches the "return" instruction. We can then pop a stack object
88 /// to this register without worry about clobbering it.
89 static unsigned findDeadCallerSavedReg(MachineBasicBlock &MBB,
90 MachineBasicBlock::iterator &MBBI,
91 const TargetRegisterInfo &TRI,
93 const MachineFunction *MF = MBB.getParent();
94 const Function *F = MF->getFunction();
95 if (!F || MF->getMMI().callsEHReturn())
98 static const uint16_t CallerSavedRegs32Bit[] = {
99 X86::EAX, X86::EDX, X86::ECX, 0
102 static const uint16_t CallerSavedRegs64Bit[] = {
103 X86::RAX, X86::RDX, X86::RCX, X86::RSI, X86::RDI,
104 X86::R8, X86::R9, X86::R10, X86::R11, 0
107 unsigned Opc = MBBI->getOpcode();
114 case X86::TCRETURNdi:
115 case X86::TCRETURNri:
116 case X86::TCRETURNmi:
117 case X86::TCRETURNdi64:
118 case X86::TCRETURNri64:
119 case X86::TCRETURNmi64:
121 case X86::EH_RETURN64: {
122 SmallSet<uint16_t, 8> Uses;
123 for (unsigned i = 0, e = MBBI->getNumOperands(); i != e; ++i) {
124 MachineOperand &MO = MBBI->getOperand(i);
125 if (!MO.isReg() || MO.isDef())
127 unsigned Reg = MO.getReg();
130 for (MCRegAliasIterator AI(Reg, &TRI, true); AI.isValid(); ++AI)
134 const uint16_t *CS = Is64Bit ? CallerSavedRegs64Bit : CallerSavedRegs32Bit;
136 if (!Uses.count(*CS))
145 /// emitSPUpdate - Emit a series of instructions to increment / decrement the
146 /// stack pointer by a constant value.
148 void emitSPUpdate(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
149 unsigned StackPtr, int64_t NumBytes,
150 bool Is64Bit, bool IsLP64, bool UseLEA,
151 const TargetInstrInfo &TII, const TargetRegisterInfo &TRI) {
152 bool isSub = NumBytes < 0;
153 uint64_t Offset = isSub ? -NumBytes : NumBytes;
156 Opc = getLEArOpcode(IsLP64);
159 ? getSUBriOpcode(IsLP64, Offset)
160 : getADDriOpcode(IsLP64, Offset);
162 uint64_t Chunk = (1LL << 31) - 1;
163 DebugLoc DL = MBB.findDebugLoc(MBBI);
166 uint64_t ThisVal = (Offset > Chunk) ? Chunk : Offset;
167 if (ThisVal == (Is64Bit ? 8 : 4)) {
168 // Use push / pop instead.
170 ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX)
171 : findDeadCallerSavedReg(MBB, MBBI, TRI, Is64Bit);
174 ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r)
175 : (Is64Bit ? X86::POP64r : X86::POP32r);
176 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(Opc))
177 .addReg(Reg, getDefRegState(!isSub) | getUndefRegState(isSub));
179 MI->setFlag(MachineInstr::FrameSetup);
185 MachineInstr *MI = NULL;
188 MI = addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr),
189 StackPtr, false, isSub ? -ThisVal : ThisVal);
191 MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
194 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
198 MI->setFlag(MachineInstr::FrameSetup);
204 /// mergeSPUpdatesUp - Merge two stack-manipulating instructions upper iterator.
206 void mergeSPUpdatesUp(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
207 unsigned StackPtr, uint64_t *NumBytes = NULL) {
208 if (MBBI == MBB.begin()) return;
210 MachineBasicBlock::iterator PI = std::prev(MBBI);
211 unsigned Opc = PI->getOpcode();
212 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 ||
213 Opc == X86::ADD32ri || Opc == X86::ADD32ri8 ||
214 Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
215 PI->getOperand(0).getReg() == StackPtr) {
217 *NumBytes += PI->getOperand(2).getImm();
219 } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB64ri8 ||
220 Opc == X86::SUB32ri || Opc == X86::SUB32ri8) &&
221 PI->getOperand(0).getReg() == StackPtr) {
223 *NumBytes -= PI->getOperand(2).getImm();
228 /// mergeSPUpdatesDown - Merge two stack-manipulating instructions lower iterator.
230 void mergeSPUpdatesDown(MachineBasicBlock &MBB,
231 MachineBasicBlock::iterator &MBBI,
232 unsigned StackPtr, uint64_t *NumBytes = NULL) {
233 // FIXME: THIS ISN'T RUN!!!
236 if (MBBI == MBB.end()) return;
238 MachineBasicBlock::iterator NI = std::next(MBBI);
239 if (NI == MBB.end()) return;
241 unsigned Opc = NI->getOpcode();
242 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 ||
243 Opc == X86::ADD32ri || Opc == X86::ADD32ri8) &&
244 NI->getOperand(0).getReg() == StackPtr) {
246 *NumBytes -= NI->getOperand(2).getImm();
249 } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB64ri8 ||
250 Opc == X86::SUB32ri || Opc == X86::SUB32ri8) &&
251 NI->getOperand(0).getReg() == StackPtr) {
253 *NumBytes += NI->getOperand(2).getImm();
259 /// mergeSPUpdates - Checks the instruction before/after the passed
260 /// instruction. If it is an ADD/SUB/LEA instruction it is deleted argument and the
261 /// stack adjustment is returned as a positive value for ADD/LEA and a negative for
263 static int mergeSPUpdates(MachineBasicBlock &MBB,
264 MachineBasicBlock::iterator &MBBI,
266 bool doMergeWithPrevious) {
267 if ((doMergeWithPrevious && MBBI == MBB.begin()) ||
268 (!doMergeWithPrevious && MBBI == MBB.end()))
271 MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(MBBI) : MBBI;
272 MachineBasicBlock::iterator NI = doMergeWithPrevious ? 0 : std::next(MBBI);
273 unsigned Opc = PI->getOpcode();
276 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 ||
277 Opc == X86::ADD32ri || Opc == X86::ADD32ri8 ||
278 Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
279 PI->getOperand(0).getReg() == StackPtr){
280 Offset += PI->getOperand(2).getImm();
282 if (!doMergeWithPrevious) MBBI = NI;
283 } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB64ri8 ||
284 Opc == X86::SUB32ri || Opc == X86::SUB32ri8) &&
285 PI->getOperand(0).getReg() == StackPtr) {
286 Offset -= PI->getOperand(2).getImm();
288 if (!doMergeWithPrevious) MBBI = NI;
294 static bool isEAXLiveIn(MachineFunction &MF) {
295 for (MachineRegisterInfo::livein_iterator II = MF.getRegInfo().livein_begin(),
296 EE = MF.getRegInfo().livein_end(); II != EE; ++II) {
297 unsigned Reg = II->first;
299 if (Reg == X86::EAX || Reg == X86::AX ||
300 Reg == X86::AH || Reg == X86::AL)
307 void X86FrameLowering::emitCalleeSavedFrameMoves(
308 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL,
309 unsigned FramePtr) const {
310 MachineFunction &MF = *MBB.getParent();
311 MachineFrameInfo *MFI = MF.getFrameInfo();
312 MachineModuleInfo &MMI = MF.getMMI();
313 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
314 const X86InstrInfo &TII = *TM.getInstrInfo();
316 // Add callee saved registers to move list.
317 const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
318 if (CSI.empty()) return;
320 const X86RegisterInfo *RegInfo = TM.getRegisterInfo();
321 bool HasFP = hasFP(MF);
323 // Calculate amount of bytes used for return address storing.
324 int stackGrowth = -RegInfo->getSlotSize();
326 // FIXME: This is dirty hack. The code itself is pretty mess right now.
327 // It should be rewritten from scratch and generalized sometimes.
329 // Determine maximum offset (minimum due to stack growth).
330 int64_t MaxOffset = 0;
331 for (std::vector<CalleeSavedInfo>::const_iterator
332 I = CSI.begin(), E = CSI.end(); I != E; ++I)
333 MaxOffset = std::min(MaxOffset,
334 MFI->getObjectOffset(I->getFrameIdx()));
336 // Calculate offsets.
337 int64_t saveAreaOffset = (HasFP ? 3 : 2) * stackGrowth;
338 for (std::vector<CalleeSavedInfo>::const_iterator
339 I = CSI.begin(), E = CSI.end(); I != E; ++I) {
340 int64_t Offset = MFI->getObjectOffset(I->getFrameIdx());
341 unsigned Reg = I->getReg();
342 Offset = MaxOffset - Offset + saveAreaOffset;
344 // Don't output a new machine move if we're re-saving the frame
345 // pointer. This happens when the PrologEpilogInserter has inserted an extra
346 // "PUSH" of the frame pointer -- the "emitPrologue" method automatically
347 // generates one when frame pointers are used. If we generate a "machine
348 // move" for this extra "PUSH", the linker will lose track of the fact that
349 // the frame pointer should have the value of the first "PUSH" when it's
352 // FIXME: This looks inelegant. It's possibly correct, but it's covering up
353 // another bug. I.e., one where we generate a prolog like this:
361 // The immediate re-push of EBP is unnecessary. At the least, it's an
362 // optimization bug. EBP can be used as a scratch register in certain
363 // cases, but probably not when we have a frame pointer.
364 if (HasFP && FramePtr == Reg)
367 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
369 MMI.addFrameInst(MCCFIInstruction::createOffset(0, DwarfReg, Offset));
370 BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION)).addCFIIndex(CFIIndex);
374 /// usesTheStack - This function checks if any of the users of EFLAGS
375 /// copies the EFLAGS. We know that the code that lowers COPY of EFLAGS has
376 /// to use the stack, and if we don't adjust the stack we clobber the first
378 /// See X86InstrInfo::copyPhysReg.
379 static bool usesTheStack(const MachineFunction &MF) {
380 const MachineRegisterInfo &MRI = MF.getRegInfo();
382 for (MachineRegisterInfo::reg_iterator ri = MRI.reg_begin(X86::EFLAGS),
383 re = MRI.reg_end(); ri != re; ++ri)
390 /// emitPrologue - Push callee-saved registers onto the stack, which
391 /// automatically adjust the stack pointer. Adjust the stack pointer to allocate
392 /// space for local variables. Also emit labels used by the exception handler to
393 /// generate the exception handling frames.
394 void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
395 MachineBasicBlock &MBB = MF.front(); // Prologue goes in entry BB.
396 MachineBasicBlock::iterator MBBI = MBB.begin();
397 MachineFrameInfo *MFI = MF.getFrameInfo();
398 const Function *Fn = MF.getFunction();
399 const X86RegisterInfo *RegInfo = TM.getRegisterInfo();
400 const X86InstrInfo &TII = *TM.getInstrInfo();
401 MachineModuleInfo &MMI = MF.getMMI();
402 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
403 bool needsFrameMoves = MMI.hasDebugInfo() ||
404 Fn->needsUnwindTableEntry();
405 uint64_t MaxAlign = MFI->getMaxAlignment(); // Desired stack alignment.
406 uint64_t StackSize = MFI->getStackSize(); // Number of bytes to allocate.
407 bool HasFP = hasFP(MF);
408 bool Is64Bit = STI.is64Bit();
409 bool IsLP64 = STI.isTarget64BitLP64();
410 bool IsWin64 = STI.isTargetWin64();
411 bool UseLEA = STI.useLeaForSP();
412 unsigned StackAlign = getStackAlignment();
413 unsigned SlotSize = RegInfo->getSlotSize();
414 unsigned FramePtr = RegInfo->getFrameRegister(MF);
415 unsigned StackPtr = RegInfo->getStackRegister();
416 unsigned BasePtr = RegInfo->getBaseRegister();
419 // If we're forcing a stack realignment we can't rely on just the frame
420 // info, we need to know the ABI stack alignment as well in case we
421 // have a call out. Otherwise just make sure we have some alignment - we'll
422 // go with the minimum SlotSize.
423 if (ForceStackAlign) {
425 MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign;
426 else if (MaxAlign < SlotSize)
430 // Add RETADDR move area to callee saved frame size.
431 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
432 if (TailCallReturnAddrDelta < 0)
433 X86FI->setCalleeSavedFrameSize(
434 X86FI->getCalleeSavedFrameSize() - TailCallReturnAddrDelta);
436 // If this is x86-64 and the Red Zone is not disabled, if we are a leaf
437 // function, and use up to 128 bytes of stack space, don't have a frame
438 // pointer, calls, or dynamic alloca then we do not need to adjust the
439 // stack pointer (we fit in the Red Zone). We also check that we don't
440 // push and pop from the stack.
441 if (Is64Bit && !Fn->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
442 Attribute::NoRedZone) &&
443 !RegInfo->needsStackRealignment(MF) &&
444 !MFI->hasVarSizedObjects() && // No dynamic alloca.
445 !MFI->adjustsStack() && // No calls.
446 !IsWin64 && // Win64 has no Red Zone
447 !usesTheStack(MF) && // Don't push and pop.
448 !MF.getTarget().Options.EnableSegmentedStacks) { // Regular stack
449 uint64_t MinSize = X86FI->getCalleeSavedFrameSize();
450 if (HasFP) MinSize += SlotSize;
451 StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0);
452 MFI->setStackSize(StackSize);
455 // Insert stack pointer adjustment for later moving of return addr. Only
456 // applies to tail call optimized functions where the callee argument stack
457 // size is bigger than the callers.
458 if (TailCallReturnAddrDelta < 0) {
460 BuildMI(MBB, MBBI, DL,
461 TII.get(getSUBriOpcode(IsLP64, -TailCallReturnAddrDelta)),
464 .addImm(-TailCallReturnAddrDelta)
465 .setMIFlag(MachineInstr::FrameSetup);
466 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
469 // Mapping for machine moves:
471 // DST: VirtualFP AND
472 // SRC: VirtualFP => DW_CFA_def_cfa_offset
473 // ELSE => DW_CFA_def_cfa
475 // SRC: VirtualFP AND
476 // DST: Register => DW_CFA_def_cfa_register
479 // OFFSET < 0 => DW_CFA_offset_extended_sf
480 // REG < 64 => DW_CFA_offset + Reg
481 // ELSE => DW_CFA_offset_extended
483 uint64_t NumBytes = 0;
484 int stackGrowth = -SlotSize;
487 // Calculate required stack adjustment.
488 uint64_t FrameSize = StackSize - SlotSize;
489 if (RegInfo->needsStackRealignment(MF)) {
490 // Callee-saved registers are pushed on stack before the stack
492 FrameSize -= X86FI->getCalleeSavedFrameSize();
493 NumBytes = (FrameSize + MaxAlign - 1) / MaxAlign * MaxAlign;
495 NumBytes = FrameSize - X86FI->getCalleeSavedFrameSize();
498 // Get the offset of the stack slot for the EBP register, which is
499 // guaranteed to be the last slot by processFunctionBeforeFrameFinalized.
500 // Update the frame offset adjustment.
501 MFI->setOffsetAdjustment(-NumBytes);
503 // Save EBP/RBP into the appropriate stack slot.
504 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))
505 .addReg(FramePtr, RegState::Kill)
506 .setMIFlag(MachineInstr::FrameSetup);
508 if (needsFrameMoves) {
509 // Mark the place where EBP/RBP was saved.
510 // Define the current CFA rule to use the provided offset.
512 unsigned CFIIndex = MMI.addFrameInst(
513 MCCFIInstruction::createDefCfaOffset(0, 2 * stackGrowth));
514 BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION))
515 .addCFIIndex(CFIIndex);
517 // Change the rule for the FramePtr to be an "offset" rule.
518 unsigned DwarfFramePtr = RegInfo->getDwarfRegNum(FramePtr, true);
519 CFIIndex = MMI.addFrameInst(
520 MCCFIInstruction::createOffset(0, DwarfFramePtr, 2 * stackGrowth));
521 BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION))
522 .addCFIIndex(CFIIndex);
525 // Update EBP with the new base value.
526 BuildMI(MBB, MBBI, DL,
527 TII.get(Is64Bit ? X86::MOV64rr : X86::MOV32rr), FramePtr)
529 .setMIFlag(MachineInstr::FrameSetup);
531 if (needsFrameMoves) {
532 // Mark effective beginning of when frame pointer becomes valid.
533 // Define the current CFA to use the EBP/RBP register.
534 unsigned DwarfFramePtr = RegInfo->getDwarfRegNum(FramePtr, true);
535 unsigned CFIIndex = MMI.addFrameInst(
536 MCCFIInstruction::createDefCfaRegister(0, DwarfFramePtr));
537 BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION))
538 .addCFIIndex(CFIIndex);
541 // Mark the FramePtr as live-in in every block except the entry.
542 for (MachineFunction::iterator I = std::next(MF.begin()), E = MF.end();
544 I->addLiveIn(FramePtr);
546 NumBytes = StackSize - X86FI->getCalleeSavedFrameSize();
549 // Skip the callee-saved push instructions.
550 bool PushedRegs = false;
551 int StackOffset = 2 * stackGrowth;
553 while (MBBI != MBB.end() &&
554 (MBBI->getOpcode() == X86::PUSH32r ||
555 MBBI->getOpcode() == X86::PUSH64r)) {
557 MBBI->setFlag(MachineInstr::FrameSetup);
560 if (!HasFP && needsFrameMoves) {
561 // Mark callee-saved push instruction.
562 // Define the current CFA rule to use the provided offset.
564 unsigned CFIIndex = MMI.addFrameInst(
565 MCCFIInstruction::createDefCfaOffset(nullptr, StackOffset));
566 BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION))
567 .addCFIIndex(CFIIndex);
568 StackOffset += stackGrowth;
572 // Realign stack after we pushed callee-saved registers (so that we'll be
573 // able to calculate their offsets from the frame pointer).
575 // NOTE: We push the registers before realigning the stack, so
576 // vector callee-saved (xmm) registers may be saved w/o proper
577 // alignment in this way. However, currently these regs are saved in
578 // stack slots (see X86FrameLowering::spillCalleeSavedRegisters()), so
579 // this shouldn't be a problem.
580 if (RegInfo->needsStackRealignment(MF)) {
581 assert(HasFP && "There should be a frame pointer if stack is realigned.");
583 BuildMI(MBB, MBBI, DL,
584 TII.get(Is64Bit ? X86::AND64ri32 : X86::AND32ri), StackPtr)
587 .setMIFlag(MachineInstr::FrameSetup);
589 // The EFLAGS implicit def is dead.
590 MI->getOperand(3).setIsDead();
593 // If there is an SUB32ri of ESP immediately before this instruction, merge
594 // the two. This can be the case when tail call elimination is enabled and
595 // the callee has more arguments then the caller.
596 NumBytes -= mergeSPUpdates(MBB, MBBI, StackPtr, true);
598 // If there is an ADD32ri or SUB32ri of ESP immediately after this
599 // instruction, merge the two instructions.
600 mergeSPUpdatesDown(MBB, MBBI, StackPtr, &NumBytes);
602 // Adjust stack pointer: ESP -= numbytes.
604 // Windows and cygwin/mingw require a prologue helper routine when allocating
605 // more than 4K bytes on the stack. Windows uses __chkstk and cygwin/mingw
606 // uses __alloca. __alloca and the 32-bit version of __chkstk will probe the
607 // stack and adjust the stack pointer in one go. The 64-bit version of
608 // __chkstk is only responsible for probing the stack. The 64-bit prologue is
609 // responsible for adjusting the stack pointer. Touching the stack at 4K
610 // increments is necessary to ensure that the guard pages used by the OS
611 // virtual memory manager are allocated in correct sequence.
612 if (NumBytes >= 4096 && STI.isOSWindows() && !STI.isTargetMacho()) {
613 const char *StackProbeSymbol;
616 if (STI.isTargetCygMing()) {
617 StackProbeSymbol = "___chkstk_ms";
619 StackProbeSymbol = "__chkstk";
621 } else if (STI.isTargetCygMing())
622 StackProbeSymbol = "_alloca";
624 StackProbeSymbol = "_chkstk";
626 // Check whether EAX is livein for this function.
627 bool isEAXAlive = isEAXLiveIn(MF);
630 // Sanity check that EAX is not livein for this function.
631 // It should not be, so throw an assert.
632 assert(!Is64Bit && "EAX is livein in x64 case!");
635 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH32r))
636 .addReg(X86::EAX, RegState::Kill)
637 .setMIFlag(MachineInstr::FrameSetup);
641 // Handle the 64-bit Windows ABI case where we need to call __chkstk.
642 // Function prologue is responsible for adjusting the stack pointer.
643 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::RAX)
645 .setMIFlag(MachineInstr::FrameSetup);
647 // Allocate NumBytes-4 bytes on stack in case of isEAXAlive.
648 // We'll also use 4 already allocated bytes for EAX.
649 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
650 .addImm(isEAXAlive ? NumBytes - 4 : NumBytes)
651 .setMIFlag(MachineInstr::FrameSetup);
654 BuildMI(MBB, MBBI, DL,
655 TII.get(Is64Bit ? X86::W64ALLOCA : X86::CALLpcrel32))
656 .addExternalSymbol(StackProbeSymbol)
657 .addReg(StackPtr, RegState::Define | RegState::Implicit)
658 .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit)
659 .setMIFlag(MachineInstr::FrameSetup);
662 // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp
663 // themself. It also does not clobber %rax so we can reuse it when
665 BuildMI(MBB, MBBI, DL, TII.get(X86::SUB64rr), StackPtr)
668 .setMIFlag(MachineInstr::FrameSetup);
672 MachineInstr *MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV32rm),
674 StackPtr, false, NumBytes - 4);
675 MI->setFlag(MachineInstr::FrameSetup);
676 MBB.insert(MBBI, MI);
679 emitSPUpdate(MBB, MBBI, StackPtr, -(int64_t)NumBytes, Is64Bit, IsLP64,
680 UseLEA, TII, *RegInfo);
682 // If we need a base pointer, set it up here. It's whatever the value
683 // of the stack pointer is at this point. Any variable size objects
684 // will be allocated after this, so we can still use the base pointer
685 // to reference locals.
686 if (RegInfo->hasBasePointer(MF)) {
687 // Update the frame pointer with the current stack pointer.
688 unsigned Opc = Is64Bit ? X86::MOV64rr : X86::MOV32rr;
689 BuildMI(MBB, MBBI, DL, TII.get(Opc), BasePtr)
691 .setMIFlag(MachineInstr::FrameSetup);
694 if (( (!HasFP && NumBytes) || PushedRegs) && needsFrameMoves) {
695 // Mark end of stack pointer adjustment.
696 if (!HasFP && NumBytes) {
697 // Define the current CFA rule to use the provided offset.
699 unsigned CFIIndex = MMI.addFrameInst(
700 MCCFIInstruction::createDefCfaOffset(0, -StackSize + stackGrowth));
702 BuildMI(MBB, MBBI, DL, TII.get(X86::CFI_INSTRUCTION))
703 .addCFIIndex(CFIIndex);
706 // Emit DWARF info specifying the offsets of the callee-saved registers.
708 emitCalleeSavedFrameMoves(MBB, MBBI, DL, HasFP ? FramePtr : StackPtr);
712 void X86FrameLowering::emitEpilogue(MachineFunction &MF,
713 MachineBasicBlock &MBB) const {
714 const MachineFrameInfo *MFI = MF.getFrameInfo();
715 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
716 const X86RegisterInfo *RegInfo = TM.getRegisterInfo();
717 const X86InstrInfo &TII = *TM.getInstrInfo();
718 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
719 assert(MBBI != MBB.end() && "Returning block has no instructions");
720 unsigned RetOpcode = MBBI->getOpcode();
721 DebugLoc DL = MBBI->getDebugLoc();
722 bool Is64Bit = STI.is64Bit();
723 bool IsLP64 = STI.isTarget64BitLP64();
724 bool UseLEA = STI.useLeaForSP();
725 unsigned StackAlign = getStackAlignment();
726 unsigned SlotSize = RegInfo->getSlotSize();
727 unsigned FramePtr = RegInfo->getFrameRegister(MF);
728 unsigned StackPtr = RegInfo->getStackRegister();
732 llvm_unreachable("Can only insert epilog into returning blocks");
737 case X86::TCRETURNdi:
738 case X86::TCRETURNri:
739 case X86::TCRETURNmi:
740 case X86::TCRETURNdi64:
741 case X86::TCRETURNri64:
742 case X86::TCRETURNmi64:
744 case X86::EH_RETURN64:
745 break; // These are ok
748 // Get the number of bytes to allocate from the FrameInfo.
749 uint64_t StackSize = MFI->getStackSize();
750 uint64_t MaxAlign = MFI->getMaxAlignment();
751 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
752 uint64_t NumBytes = 0;
754 // If we're forcing a stack realignment we can't rely on just the frame
755 // info, we need to know the ABI stack alignment as well in case we
756 // have a call out. Otherwise just make sure we have some alignment - we'll
757 // go with the minimum.
758 if (ForceStackAlign) {
760 MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign;
762 MaxAlign = MaxAlign ? MaxAlign : 4;
766 // Calculate required stack adjustment.
767 uint64_t FrameSize = StackSize - SlotSize;
768 if (RegInfo->needsStackRealignment(MF)) {
769 // Callee-saved registers were pushed on stack before the stack
772 NumBytes = (FrameSize + MaxAlign - 1) / MaxAlign * MaxAlign;
774 NumBytes = FrameSize - CSSize;
778 BuildMI(MBB, MBBI, DL,
779 TII.get(Is64Bit ? X86::POP64r : X86::POP32r), FramePtr);
781 NumBytes = StackSize - CSSize;
784 // Skip the callee-saved pop instructions.
785 while (MBBI != MBB.begin()) {
786 MachineBasicBlock::iterator PI = std::prev(MBBI);
787 unsigned Opc = PI->getOpcode();
789 if (Opc != X86::POP32r && Opc != X86::POP64r && Opc != X86::DBG_VALUE &&
795 MachineBasicBlock::iterator FirstCSPop = MBBI;
797 DL = MBBI->getDebugLoc();
799 // If there is an ADD32ri or SUB32ri of ESP immediately before this
800 // instruction, merge the two instructions.
801 if (NumBytes || MFI->hasVarSizedObjects())
802 mergeSPUpdatesUp(MBB, MBBI, StackPtr, &NumBytes);
804 // If dynamic alloca is used, then reset esp to point to the last callee-saved
805 // slot before popping them off! Same applies for the case, when stack was
807 if (RegInfo->needsStackRealignment(MF) || MFI->hasVarSizedObjects()) {
808 if (RegInfo->needsStackRealignment(MF))
811 unsigned Opc = getLEArOpcode(IsLP64);
812 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr),
813 FramePtr, false, -CSSize);
815 unsigned Opc = (Is64Bit ? X86::MOV64rr : X86::MOV32rr);
816 BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
819 } else if (NumBytes) {
820 // Adjust stack pointer back: ESP += numbytes.
821 emitSPUpdate(MBB, MBBI, StackPtr, NumBytes, Is64Bit, IsLP64, UseLEA,
825 // We're returning from function via eh_return.
826 if (RetOpcode == X86::EH_RETURN || RetOpcode == X86::EH_RETURN64) {
827 MBBI = MBB.getLastNonDebugInstr();
828 MachineOperand &DestAddr = MBBI->getOperand(0);
829 assert(DestAddr.isReg() && "Offset should be in register!");
830 BuildMI(MBB, MBBI, DL,
831 TII.get(Is64Bit ? X86::MOV64rr : X86::MOV32rr),
832 StackPtr).addReg(DestAddr.getReg());
833 } else if (RetOpcode == X86::TCRETURNri || RetOpcode == X86::TCRETURNdi ||
834 RetOpcode == X86::TCRETURNmi ||
835 RetOpcode == X86::TCRETURNri64 || RetOpcode == X86::TCRETURNdi64 ||
836 RetOpcode == X86::TCRETURNmi64) {
837 bool isMem = RetOpcode == X86::TCRETURNmi || RetOpcode == X86::TCRETURNmi64;
838 // Tail call return: adjust the stack pointer and jump to callee.
839 MBBI = MBB.getLastNonDebugInstr();
840 MachineOperand &JumpTarget = MBBI->getOperand(0);
841 MachineOperand &StackAdjust = MBBI->getOperand(isMem ? 5 : 1);
842 assert(StackAdjust.isImm() && "Expecting immediate value.");
844 // Adjust stack pointer.
845 int StackAdj = StackAdjust.getImm();
846 int MaxTCDelta = X86FI->getTCReturnAddrDelta();
848 assert(MaxTCDelta <= 0 && "MaxTCDelta should never be positive");
850 // Incoporate the retaddr area.
851 Offset = StackAdj-MaxTCDelta;
852 assert(Offset >= 0 && "Offset should never be negative");
855 // Check for possible merge with preceding ADD instruction.
856 Offset += mergeSPUpdates(MBB, MBBI, StackPtr, true);
857 emitSPUpdate(MBB, MBBI, StackPtr, Offset, Is64Bit, IsLP64,
858 UseLEA, TII, *RegInfo);
861 // Jump to label or value in register.
862 if (RetOpcode == X86::TCRETURNdi || RetOpcode == X86::TCRETURNdi64) {
863 MachineInstrBuilder MIB =
864 BuildMI(MBB, MBBI, DL, TII.get((RetOpcode == X86::TCRETURNdi)
865 ? X86::TAILJMPd : X86::TAILJMPd64));
866 if (JumpTarget.isGlobal())
867 MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
868 JumpTarget.getTargetFlags());
870 assert(JumpTarget.isSymbol());
871 MIB.addExternalSymbol(JumpTarget.getSymbolName(),
872 JumpTarget.getTargetFlags());
874 } else if (RetOpcode == X86::TCRETURNmi || RetOpcode == X86::TCRETURNmi64) {
875 MachineInstrBuilder MIB =
876 BuildMI(MBB, MBBI, DL, TII.get((RetOpcode == X86::TCRETURNmi)
877 ? X86::TAILJMPm : X86::TAILJMPm64));
878 for (unsigned i = 0; i != 5; ++i)
879 MIB.addOperand(MBBI->getOperand(i));
880 } else if (RetOpcode == X86::TCRETURNri64) {
881 BuildMI(MBB, MBBI, DL, TII.get(X86::TAILJMPr64)).
882 addReg(JumpTarget.getReg(), RegState::Kill);
884 BuildMI(MBB, MBBI, DL, TII.get(X86::TAILJMPr)).
885 addReg(JumpTarget.getReg(), RegState::Kill);
888 MachineInstr *NewMI = std::prev(MBBI);
889 NewMI->copyImplicitOps(MF, MBBI);
891 // Delete the pseudo instruction TCRETURN.
893 } else if ((RetOpcode == X86::RETQ || RetOpcode == X86::RETL ||
894 RetOpcode == X86::RETIQ || RetOpcode == X86::RETIL) &&
895 (X86FI->getTCReturnAddrDelta() < 0)) {
896 // Add the return addr area delta back since we are not tail calling.
897 int delta = -1*X86FI->getTCReturnAddrDelta();
898 MBBI = MBB.getLastNonDebugInstr();
900 // Check for possible merge with preceding ADD instruction.
901 delta += mergeSPUpdates(MBB, MBBI, StackPtr, true);
902 emitSPUpdate(MBB, MBBI, StackPtr, delta, Is64Bit, IsLP64, UseLEA, TII,
907 int X86FrameLowering::getFrameIndexOffset(const MachineFunction &MF, int FI) const {
908 const X86RegisterInfo *RegInfo =
909 static_cast<const X86RegisterInfo*>(MF.getTarget().getRegisterInfo());
910 const MachineFrameInfo *MFI = MF.getFrameInfo();
911 int Offset = MFI->getObjectOffset(FI) - getOffsetOfLocalArea();
912 uint64_t StackSize = MFI->getStackSize();
914 if (RegInfo->hasBasePointer(MF)) {
915 assert (hasFP(MF) && "VLAs and dynamic stack realign, but no FP?!");
917 // Skip the saved EBP.
918 return Offset + RegInfo->getSlotSize();
920 assert((-(Offset + StackSize)) % MFI->getObjectAlignment(FI) == 0);
921 return Offset + StackSize;
923 } else if (RegInfo->needsStackRealignment(MF)) {
925 // Skip the saved EBP.
926 return Offset + RegInfo->getSlotSize();
928 assert((-(Offset + StackSize)) % MFI->getObjectAlignment(FI) == 0);
929 return Offset + StackSize;
931 // FIXME: Support tail calls
934 return Offset + StackSize;
936 // Skip the saved EBP.
937 Offset += RegInfo->getSlotSize();
939 // Skip the RETADDR move area
940 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
941 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
942 if (TailCallReturnAddrDelta < 0)
943 Offset -= TailCallReturnAddrDelta;
949 int X86FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
950 unsigned &FrameReg) const {
951 const X86RegisterInfo *RegInfo =
952 static_cast<const X86RegisterInfo*>(MF.getTarget().getRegisterInfo());
953 // We can't calculate offset from frame pointer if the stack is realigned,
954 // so enforce usage of stack/base pointer. The base pointer is used when we
955 // have dynamic allocas in addition to dynamic realignment.
956 if (RegInfo->hasBasePointer(MF))
957 FrameReg = RegInfo->getBaseRegister();
958 else if (RegInfo->needsStackRealignment(MF))
959 FrameReg = RegInfo->getStackRegister();
961 FrameReg = RegInfo->getFrameRegister(MF);
962 return getFrameIndexOffset(MF, FI);
965 bool X86FrameLowering::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
966 MachineBasicBlock::iterator MI,
967 const std::vector<CalleeSavedInfo> &CSI,
968 const TargetRegisterInfo *TRI) const {
972 DebugLoc DL = MBB.findDebugLoc(MI);
974 MachineFunction &MF = *MBB.getParent();
976 unsigned SlotSize = STI.is64Bit() ? 8 : 4;
977 unsigned FPReg = TRI->getFrameRegister(MF);
978 unsigned CalleeFrameSize = 0;
980 const TargetInstrInfo &TII = *MF.getTarget().getInstrInfo();
981 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
983 // Push GPRs. It increases frame size.
984 unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
985 for (unsigned i = CSI.size(); i != 0; --i) {
986 unsigned Reg = CSI[i-1].getReg();
987 if (!X86::GR64RegClass.contains(Reg) &&
988 !X86::GR32RegClass.contains(Reg))
990 // Add the callee-saved register as live-in. It's killed at the spill.
993 // X86RegisterInfo::emitPrologue will handle spilling of frame register.
995 CalleeFrameSize += SlotSize;
996 BuildMI(MBB, MI, DL, TII.get(Opc)).addReg(Reg, RegState::Kill)
997 .setMIFlag(MachineInstr::FrameSetup);
1000 X86FI->setCalleeSavedFrameSize(CalleeFrameSize);
1002 // Make XMM regs spilled. X86 does not have ability of push/pop XMM.
1003 // It can be done by spilling XMMs to stack frame.
1004 // Note that only Win64 ABI might spill XMMs.
1005 for (unsigned i = CSI.size(); i != 0; --i) {
1006 unsigned Reg = CSI[i-1].getReg();
1007 if (X86::GR64RegClass.contains(Reg) ||
1008 X86::GR32RegClass.contains(Reg))
1010 // Add the callee-saved register as live-in. It's killed at the spill.
1012 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1013 TII.storeRegToStackSlot(MBB, MI, Reg, true, CSI[i-1].getFrameIdx(),
1020 bool X86FrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
1021 MachineBasicBlock::iterator MI,
1022 const std::vector<CalleeSavedInfo> &CSI,
1023 const TargetRegisterInfo *TRI) const {
1027 DebugLoc DL = MBB.findDebugLoc(MI);
1029 MachineFunction &MF = *MBB.getParent();
1030 const TargetInstrInfo &TII = *MF.getTarget().getInstrInfo();
1032 // Reload XMMs from stack frame.
1033 for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
1034 unsigned Reg = CSI[i].getReg();
1035 if (X86::GR64RegClass.contains(Reg) ||
1036 X86::GR32RegClass.contains(Reg))
1038 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1039 TII.loadRegFromStackSlot(MBB, MI, Reg, CSI[i].getFrameIdx(),
1044 unsigned FPReg = TRI->getFrameRegister(MF);
1045 unsigned Opc = STI.is64Bit() ? X86::POP64r : X86::POP32r;
1046 for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
1047 unsigned Reg = CSI[i].getReg();
1048 if (!X86::GR64RegClass.contains(Reg) &&
1049 !X86::GR32RegClass.contains(Reg))
1052 // X86RegisterInfo::emitEpilogue will handle restoring of frame register.
1054 BuildMI(MBB, MI, DL, TII.get(Opc), Reg);
1060 X86FrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
1061 RegScavenger *RS) const {
1062 MachineFrameInfo *MFI = MF.getFrameInfo();
1063 const X86RegisterInfo *RegInfo = TM.getRegisterInfo();
1064 unsigned SlotSize = RegInfo->getSlotSize();
1066 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1067 int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
1069 if (TailCallReturnAddrDelta < 0) {
1070 // create RETURNADDR area
1079 MFI->CreateFixedObject(-TailCallReturnAddrDelta,
1080 TailCallReturnAddrDelta - SlotSize, true);
1084 assert((TailCallReturnAddrDelta <= 0) &&
1085 "The Delta should always be zero or negative");
1086 const TargetFrameLowering &TFI = *MF.getTarget().getFrameLowering();
1088 // Create a frame entry for the EBP register that must be saved.
1089 int FrameIdx = MFI->CreateFixedObject(SlotSize,
1091 TFI.getOffsetOfLocalArea() +
1092 TailCallReturnAddrDelta,
1094 assert(FrameIdx == MFI->getObjectIndexBegin() &&
1095 "Slot for EBP register must be last in order to be found!");
1099 // Spill the BasePtr if it's used.
1100 if (RegInfo->hasBasePointer(MF))
1101 MF.getRegInfo().setPhysRegUsed(RegInfo->getBaseRegister());
1105 HasNestArgument(const MachineFunction *MF) {
1106 const Function *F = MF->getFunction();
1107 for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end();
1109 if (I->hasNestAttr())
1115 /// GetScratchRegister - Get a temp register for performing work in the
1116 /// segmented stack and the Erlang/HiPE stack prologue. Depending on platform
1117 /// and the properties of the function either one or two registers will be
1118 /// needed. Set primary to true for the first register, false for the second.
1120 GetScratchRegister(bool Is64Bit, const MachineFunction &MF, bool Primary) {
1121 CallingConv::ID CallingConvention = MF.getFunction()->getCallingConv();
1124 if (CallingConvention == CallingConv::HiPE) {
1126 return Primary ? X86::R14 : X86::R13;
1128 return Primary ? X86::EBX : X86::EDI;
1132 return Primary ? X86::R11 : X86::R12;
1134 bool IsNested = HasNestArgument(&MF);
1136 if (CallingConvention == CallingConv::X86_FastCall ||
1137 CallingConvention == CallingConv::Fast) {
1139 report_fatal_error("Segmented stacks does not support fastcall with "
1140 "nested function.");
1141 return Primary ? X86::EAX : X86::ECX;
1144 return Primary ? X86::EDX : X86::EAX;
1145 return Primary ? X86::ECX : X86::EAX;
1148 // The stack limit in the TCB is set to this many bytes above the actual stack
1150 static const uint64_t kSplitStackAvailable = 256;
1153 X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const {
1154 MachineBasicBlock &prologueMBB = MF.front();
1155 MachineFrameInfo *MFI = MF.getFrameInfo();
1156 const X86InstrInfo &TII = *TM.getInstrInfo();
1158 bool Is64Bit = STI.is64Bit();
1159 unsigned TlsReg, TlsOffset;
1162 unsigned ScratchReg = GetScratchRegister(Is64Bit, MF, true);
1163 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
1164 "Scratch register is live-in");
1166 if (MF.getFunction()->isVarArg())
1167 report_fatal_error("Segmented stacks do not support vararg functions.");
1168 if (!STI.isTargetLinux() && !STI.isTargetDarwin() &&
1169 !STI.isTargetWin32() && !STI.isTargetFreeBSD())
1170 report_fatal_error("Segmented stacks not supported on this platform.");
1172 MachineBasicBlock *allocMBB = MF.CreateMachineBasicBlock();
1173 MachineBasicBlock *checkMBB = MF.CreateMachineBasicBlock();
1174 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1175 bool IsNested = false;
1177 // We need to know if the function has a nest argument only in 64 bit mode.
1179 IsNested = HasNestArgument(&MF);
1181 // The MOV R10, RAX needs to be in a different block, since the RET we emit in
1182 // allocMBB needs to be last (terminating) instruction.
1184 for (MachineBasicBlock::livein_iterator i = prologueMBB.livein_begin(),
1185 e = prologueMBB.livein_end(); i != e; i++) {
1186 allocMBB->addLiveIn(*i);
1187 checkMBB->addLiveIn(*i);
1191 allocMBB->addLiveIn(X86::R10);
1193 MF.push_front(allocMBB);
1194 MF.push_front(checkMBB);
1196 // Eventually StackSize will be calculated by a link-time pass; which will
1197 // also decide whether checking code needs to be injected into this particular
1199 StackSize = MFI->getStackSize();
1201 // When the frame size is less than 256 we just compare the stack
1202 // boundary directly to the value of the stack pointer, per gcc.
1203 bool CompareStackPointer = StackSize < kSplitStackAvailable;
1205 // Read the limit off the current stacklet off the stack_guard location.
1207 if (STI.isTargetLinux()) {
1210 } else if (STI.isTargetDarwin()) {
1212 TlsOffset = 0x60 + 90*8; // See pthread_machdep.h. Steal TLS slot 90.
1213 } else if (STI.isTargetFreeBSD()) {
1217 report_fatal_error("Segmented stacks not supported on this platform.");
1220 if (CompareStackPointer)
1221 ScratchReg = X86::RSP;
1223 BuildMI(checkMBB, DL, TII.get(X86::LEA64r), ScratchReg).addReg(X86::RSP)
1224 .addImm(1).addReg(0).addImm(-StackSize).addReg(0);
1226 BuildMI(checkMBB, DL, TII.get(X86::CMP64rm)).addReg(ScratchReg)
1227 .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg);
1229 if (STI.isTargetLinux()) {
1232 } else if (STI.isTargetDarwin()) {
1234 TlsOffset = 0x48 + 90*4;
1235 } else if (STI.isTargetWin32()) {
1237 TlsOffset = 0x14; // pvArbitrary, reserved for application use
1238 } else if (STI.isTargetFreeBSD()) {
1239 report_fatal_error("Segmented stacks not supported on FreeBSD i386.");
1241 report_fatal_error("Segmented stacks not supported on this platform.");
1244 if (CompareStackPointer)
1245 ScratchReg = X86::ESP;
1247 BuildMI(checkMBB, DL, TII.get(X86::LEA32r), ScratchReg).addReg(X86::ESP)
1248 .addImm(1).addReg(0).addImm(-StackSize).addReg(0);
1250 if (STI.isTargetLinux() || STI.isTargetWin32()) {
1251 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm)).addReg(ScratchReg)
1252 .addReg(0).addImm(0).addReg(0).addImm(TlsOffset).addReg(TlsReg);
1253 } else if (STI.isTargetDarwin()) {
1255 // TlsOffset doesn't fit into a mod r/m byte so we need an extra register
1256 unsigned ScratchReg2;
1258 if (CompareStackPointer) {
1259 // The primary scratch register is available for holding the TLS offset
1260 ScratchReg2 = GetScratchRegister(Is64Bit, MF, true);
1261 SaveScratch2 = false;
1263 // Need to use a second register to hold the TLS offset
1264 ScratchReg2 = GetScratchRegister(Is64Bit, MF, false);
1266 // Unfortunately, with fastcc the second scratch register may hold an arg
1267 SaveScratch2 = MF.getRegInfo().isLiveIn(ScratchReg2);
1270 // If Scratch2 is live-in then it needs to be saved
1271 assert((!MF.getRegInfo().isLiveIn(ScratchReg2) || SaveScratch2) &&
1272 "Scratch register is live-in and not saved");
1275 BuildMI(checkMBB, DL, TII.get(X86::PUSH32r))
1276 .addReg(ScratchReg2, RegState::Kill);
1278 BuildMI(checkMBB, DL, TII.get(X86::MOV32ri), ScratchReg2)
1280 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm))
1282 .addReg(ScratchReg2).addImm(1).addReg(0)
1287 BuildMI(checkMBB, DL, TII.get(X86::POP32r), ScratchReg2);
1291 // This jump is taken if SP >= (Stacklet Limit + Stack Space required).
1292 // It jumps to normal execution of the function body.
1293 BuildMI(checkMBB, DL, TII.get(X86::JA_4)).addMBB(&prologueMBB);
1295 // On 32 bit we first push the arguments size and then the frame size. On 64
1296 // bit, we pass the stack frame size in r10 and the argument size in r11.
1298 // Functions with nested arguments use R10, so it needs to be saved across
1299 // the call to _morestack
1302 BuildMI(allocMBB, DL, TII.get(X86::MOV64rr), X86::RAX).addReg(X86::R10);
1304 BuildMI(allocMBB, DL, TII.get(X86::MOV64ri), X86::R10)
1306 BuildMI(allocMBB, DL, TII.get(X86::MOV64ri), X86::R11)
1307 .addImm(X86FI->getArgumentStackSize());
1308 MF.getRegInfo().setPhysRegUsed(X86::R10);
1309 MF.getRegInfo().setPhysRegUsed(X86::R11);
1311 BuildMI(allocMBB, DL, TII.get(X86::PUSHi32))
1312 .addImm(X86FI->getArgumentStackSize());
1313 BuildMI(allocMBB, DL, TII.get(X86::PUSHi32))
1317 // __morestack is in libgcc
1319 BuildMI(allocMBB, DL, TII.get(X86::CALL64pcrel32))
1320 .addExternalSymbol("__morestack");
1322 BuildMI(allocMBB, DL, TII.get(X86::CALLpcrel32))
1323 .addExternalSymbol("__morestack");
1326 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET_RESTORE_R10));
1328 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET));
1330 allocMBB->addSuccessor(&prologueMBB);
1332 checkMBB->addSuccessor(allocMBB);
1333 checkMBB->addSuccessor(&prologueMBB);
1340 /// Erlang programs may need a special prologue to handle the stack size they
1341 /// might need at runtime. That is because Erlang/OTP does not implement a C
1342 /// stack but uses a custom implementation of hybrid stack/heap architecture.
1343 /// (for more information see Eric Stenman's Ph.D. thesis:
1344 /// http://publications.uu.se/uu/fulltext/nbn_se_uu_diva-2688.pdf)
1347 /// temp0 = sp - MaxStack
1348 /// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
1352 /// call inc_stack # doubles the stack space
1353 /// temp0 = sp - MaxStack
1354 /// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
1355 void X86FrameLowering::adjustForHiPEPrologue(MachineFunction &MF) const {
1356 const X86InstrInfo &TII = *TM.getInstrInfo();
1357 MachineFrameInfo *MFI = MF.getFrameInfo();
1358 const unsigned SlotSize = TM.getRegisterInfo()->getSlotSize();
1359 const bool Is64Bit = STI.is64Bit();
1361 // HiPE-specific values
1362 const unsigned HipeLeafWords = 24;
1363 const unsigned CCRegisteredArgs = Is64Bit ? 6 : 5;
1364 const unsigned Guaranteed = HipeLeafWords * SlotSize;
1365 unsigned CallerStkArity = MF.getFunction()->arg_size() > CCRegisteredArgs ?
1366 MF.getFunction()->arg_size() - CCRegisteredArgs : 0;
1367 unsigned MaxStack = MFI->getStackSize() + CallerStkArity*SlotSize + SlotSize;
1369 assert(STI.isTargetLinux() &&
1370 "HiPE prologue is only supported on Linux operating systems.");
1372 // Compute the largest caller's frame that is needed to fit the callees'
1373 // frames. This 'MaxStack' is computed from:
1375 // a) the fixed frame size, which is the space needed for all spilled temps,
1376 // b) outgoing on-stack parameter areas, and
1377 // c) the minimum stack space this function needs to make available for the
1378 // functions it calls (a tunable ABI property).
1379 if (MFI->hasCalls()) {
1380 unsigned MoreStackForCalls = 0;
1382 for (MachineFunction::iterator MBBI = MF.begin(), MBBE = MF.end();
1383 MBBI != MBBE; ++MBBI)
1384 for (MachineBasicBlock::iterator MI = MBBI->begin(), ME = MBBI->end();
1389 // Get callee operand.
1390 const MachineOperand &MO = MI->getOperand(0);
1392 // Only take account of global function calls (no closures etc.).
1396 const Function *F = dyn_cast<Function>(MO.getGlobal());
1400 // Do not update 'MaxStack' for primitive and built-in functions
1401 // (encoded with names either starting with "erlang."/"bif_" or not
1402 // having a ".", such as a simple <Module>.<Function>.<Arity>, or an
1403 // "_", such as the BIF "suspend_0") as they are executed on another
1405 if (F->getName().find("erlang.") != StringRef::npos ||
1406 F->getName().find("bif_") != StringRef::npos ||
1407 F->getName().find_first_of("._") == StringRef::npos)
1410 unsigned CalleeStkArity =
1411 F->arg_size() > CCRegisteredArgs ? F->arg_size()-CCRegisteredArgs : 0;
1412 if (HipeLeafWords - 1 > CalleeStkArity)
1413 MoreStackForCalls = std::max(MoreStackForCalls,
1414 (HipeLeafWords - 1 - CalleeStkArity) * SlotSize);
1416 MaxStack += MoreStackForCalls;
1419 // If the stack frame needed is larger than the guaranteed then runtime checks
1420 // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue.
1421 if (MaxStack > Guaranteed) {
1422 MachineBasicBlock &prologueMBB = MF.front();
1423 MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock();
1424 MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock();
1426 for (MachineBasicBlock::livein_iterator I = prologueMBB.livein_begin(),
1427 E = prologueMBB.livein_end(); I != E; I++) {
1428 stackCheckMBB->addLiveIn(*I);
1429 incStackMBB->addLiveIn(*I);
1432 MF.push_front(incStackMBB);
1433 MF.push_front(stackCheckMBB);
1435 unsigned ScratchReg, SPReg, PReg, SPLimitOffset;
1436 unsigned LEAop, CMPop, CALLop;
1440 LEAop = X86::LEA64r;
1441 CMPop = X86::CMP64rm;
1442 CALLop = X86::CALL64pcrel32;
1443 SPLimitOffset = 0x90;
1447 LEAop = X86::LEA32r;
1448 CMPop = X86::CMP32rm;
1449 CALLop = X86::CALLpcrel32;
1450 SPLimitOffset = 0x4c;
1453 ScratchReg = GetScratchRegister(Is64Bit, MF, true);
1454 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
1455 "HiPE prologue scratch register is live-in");
1457 // Create new MBB for StackCheck:
1458 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(LEAop), ScratchReg),
1459 SPReg, false, -MaxStack);
1460 // SPLimitOffset is in a fixed heap location (pointed by BP).
1461 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(CMPop))
1462 .addReg(ScratchReg), PReg, false, SPLimitOffset);
1463 BuildMI(stackCheckMBB, DL, TII.get(X86::JAE_4)).addMBB(&prologueMBB);
1465 // Create new MBB for IncStack:
1466 BuildMI(incStackMBB, DL, TII.get(CALLop)).
1467 addExternalSymbol("inc_stack_0");
1468 addRegOffset(BuildMI(incStackMBB, DL, TII.get(LEAop), ScratchReg),
1469 SPReg, false, -MaxStack);
1470 addRegOffset(BuildMI(incStackMBB, DL, TII.get(CMPop))
1471 .addReg(ScratchReg), PReg, false, SPLimitOffset);
1472 BuildMI(incStackMBB, DL, TII.get(X86::JLE_4)).addMBB(incStackMBB);
1474 stackCheckMBB->addSuccessor(&prologueMBB, 99);
1475 stackCheckMBB->addSuccessor(incStackMBB, 1);
1476 incStackMBB->addSuccessor(&prologueMBB, 99);
1477 incStackMBB->addSuccessor(incStackMBB, 1);
1484 void X86FrameLowering::
1485 eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
1486 MachineBasicBlock::iterator I) const {
1487 const X86InstrInfo &TII = *TM.getInstrInfo();
1488 const X86RegisterInfo &RegInfo = *TM.getRegisterInfo();
1489 unsigned StackPtr = RegInfo.getStackRegister();
1490 bool reseveCallFrame = hasReservedCallFrame(MF);
1491 int Opcode = I->getOpcode();
1492 bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode();
1493 bool IsLP64 = STI.isTarget64BitLP64();
1494 DebugLoc DL = I->getDebugLoc();
1495 uint64_t Amount = !reseveCallFrame ? I->getOperand(0).getImm() : 0;
1496 uint64_t CalleeAmt = isDestroy ? I->getOperand(1).getImm() : 0;
1499 if (!reseveCallFrame) {
1500 // If the stack pointer can be changed after prologue, turn the
1501 // adjcallstackup instruction into a 'sub ESP, <amt>' and the
1502 // adjcallstackdown instruction into 'add ESP, <amt>'
1503 // TODO: consider using push / pop instead of sub + store / add
1507 // We need to keep the stack aligned properly. To do this, we round the
1508 // amount of space needed for the outgoing arguments up to the next
1509 // alignment boundary.
1510 unsigned StackAlign = TM.getFrameLowering()->getStackAlignment();
1511 Amount = (Amount + StackAlign - 1) / StackAlign * StackAlign;
1513 MachineInstr *New = 0;
1514 if (Opcode == TII.getCallFrameSetupOpcode()) {
1515 New = BuildMI(MF, DL, TII.get(getSUBriOpcode(IsLP64, Amount)),
1520 assert(Opcode == TII.getCallFrameDestroyOpcode());
1522 // Factor out the amount the callee already popped.
1523 Amount -= CalleeAmt;
1526 unsigned Opc = getADDriOpcode(IsLP64, Amount);
1527 New = BuildMI(MF, DL, TII.get(Opc), StackPtr)
1528 .addReg(StackPtr).addImm(Amount);
1533 // The EFLAGS implicit def is dead.
1534 New->getOperand(3).setIsDead();
1536 // Replace the pseudo instruction with a new instruction.
1543 if (Opcode == TII.getCallFrameDestroyOpcode() && CalleeAmt) {
1544 // If we are performing frame pointer elimination and if the callee pops
1545 // something off the stack pointer, add it back. We do this until we have
1546 // more advanced stack pointer tracking ability.
1547 unsigned Opc = getSUBriOpcode(IsLP64, CalleeAmt);
1548 MachineInstr *New = BuildMI(MF, DL, TII.get(Opc), StackPtr)
1549 .addReg(StackPtr).addImm(CalleeAmt);
1551 // The EFLAGS implicit def is dead.
1552 New->getOperand(3).setIsDead();
1554 // We are not tracking the stack pointer adjustment by the callee, so make
1555 // sure we restore the stack pointer immediately after the call, there may
1556 // be spill code inserted between the CALL and ADJCALLSTACKUP instructions.
1557 MachineBasicBlock::iterator B = MBB.begin();
1558 while (I != B && !std::prev(I)->isCall())