1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the AArch64 implementation of TargetFrameLowering class.
12 //===----------------------------------------------------------------------===//
14 #include "AArch64FrameLowering.h"
15 #include "AArch64InstrInfo.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64Subtarget.h"
18 #include "AArch64TargetMachine.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/CodeGen/MachineFrameInfo.h"
21 #include "llvm/CodeGen/MachineFunction.h"
22 #include "llvm/CodeGen/MachineInstrBuilder.h"
23 #include "llvm/CodeGen/MachineModuleInfo.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/CodeGen/RegisterScavenging.h"
26 #include "llvm/IR/DataLayout.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/Support/CommandLine.h"
29 #include "llvm/Support/Debug.h"
30 #include "llvm/Support/raw_ostream.h"
34 #define DEBUG_TYPE "frame-info"
36 static cl::opt<bool> EnableRedZone("aarch64-redzone",
37 cl::desc("enable use of redzone on AArch64"),
38 cl::init(false), cl::Hidden);
40 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
42 static unsigned estimateStackSize(MachineFunction &MF) {
43 const MachineFrameInfo *FFI = MF.getFrameInfo();
45 for (int i = FFI->getObjectIndexBegin(); i != 0; ++i) {
46 int FixedOff = -FFI->getObjectOffset(i);
47 if (FixedOff > Offset)
50 for (unsigned i = 0, e = FFI->getObjectIndexEnd(); i != e; ++i) {
51 if (FFI->isDeadObjectIndex(i))
53 Offset += FFI->getObjectSize(i);
54 unsigned Align = FFI->getObjectAlignment(i);
55 // Adjust to alignment boundary
56 Offset = (Offset + Align - 1) / Align * Align;
58 // This does not include the 16 bytes used for fp and lr.
59 return (unsigned)Offset;
62 bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
65 // Don't use the red zone if the function explicitly asks us not to.
66 // This is typically used for kernel code.
67 if (MF.getFunction()->getAttributes().hasAttribute(
68 AttributeSet::FunctionIndex, Attribute::NoRedZone))
71 const MachineFrameInfo *MFI = MF.getFrameInfo();
72 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
73 unsigned NumBytes = AFI->getLocalStackSize();
75 // Note: currently hasFP() is always true for hasCalls(), but that's an
76 // implementation detail of the current code, not a strict requirement,
77 // so stay safe here and check both.
78 if (MFI->hasCalls() || hasFP(MF) || NumBytes > 128)
83 /// hasFP - Return true if the specified function should have a dedicated frame
85 bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
86 const MachineFrameInfo *MFI = MF.getFrameInfo();
89 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
90 assert(!RegInfo->needsStackRealignment(MF) &&
91 "No stack realignment on AArch64!");
94 return (MFI->hasCalls() || MFI->hasVarSizedObjects() ||
95 MFI->isFrameAddressTaken() || MFI->hasStackMap() ||
96 MFI->hasPatchPoint());
99 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
100 /// not required, we reserve argument space for call sites in the function
101 /// immediately on entry to the current function. This eliminates the need for
102 /// add/sub sp brackets around call sites. Returns true if the call frame is
103 /// included as part of the stack frame.
105 AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
106 return !MF.getFrameInfo()->hasVarSizedObjects();
109 void AArch64FrameLowering::eliminateCallFramePseudoInstr(
110 MachineFunction &MF, MachineBasicBlock &MBB,
111 MachineBasicBlock::iterator I) const {
112 const AArch64InstrInfo *TII =
113 static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
114 DebugLoc DL = I->getDebugLoc();
115 int Opc = I->getOpcode();
116 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
117 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
119 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
120 if (!TFI->hasReservedCallFrame(MF)) {
121 unsigned Align = getStackAlignment();
123 int64_t Amount = I->getOperand(0).getImm();
124 Amount = RoundUpToAlignment(Amount, Align);
128 // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
129 // doesn't have to pop anything), then the first operand will be zero too so
130 // this adjustment is a no-op.
131 if (CalleePopAmount == 0) {
132 // FIXME: in-function stack adjustment for calls is limited to 24-bits
133 // because there's no guaranteed temporary register available.
135 // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
136 // 1) For offset <= 12-bit, we use LSL #0
137 // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
138 // LSL #0, and the other uses LSL #12.
140 // Mostly call frames will be allocated at the start of a function so
141 // this is OK, but it is a limitation that needs dealing with.
142 assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
143 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, Amount, TII);
145 } else if (CalleePopAmount != 0) {
146 // If the calling convention demands that the callee pops arguments from the
147 // stack, we want to add it back if we have a reserved call frame.
148 assert(CalleePopAmount < 0xffffff && "call frame too large");
149 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, -CalleePopAmount,
155 void AArch64FrameLowering::emitCalleeSavedFrameMoves(
156 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
157 unsigned FramePtr) const {
158 MachineFunction &MF = *MBB.getParent();
159 MachineFrameInfo *MFI = MF.getFrameInfo();
160 MachineModuleInfo &MMI = MF.getMMI();
161 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
162 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
163 DebugLoc DL = MBB.findDebugLoc(MBBI);
165 // Add callee saved registers to move list.
166 const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
170 const DataLayout *TD = MF.getSubtarget().getDataLayout();
171 bool HasFP = hasFP(MF);
173 // Calculate amount of bytes used for return address storing.
174 int stackGrowth = -TD->getPointerSize(0);
176 // Calculate offsets.
177 int64_t saveAreaOffset = (HasFP ? 2 : 1) * stackGrowth;
178 unsigned TotalSkipped = 0;
179 for (const auto &Info : CSI) {
180 unsigned Reg = Info.getReg();
181 int64_t Offset = MFI->getObjectOffset(Info.getFrameIdx()) -
182 getOffsetOfLocalArea() + saveAreaOffset;
184 // Don't output a new CFI directive if we're re-saving the frame pointer or
185 // link register. This happens when the PrologEpilogInserter has inserted an
186 // extra "STP" of the frame pointer and link register -- the "emitPrologue"
187 // method automatically generates the directives when frame pointers are
188 // used. If we generate CFI directives for the extra "STP"s, the linker will
189 // lose track of the correct values for the frame pointer and link register.
190 if (HasFP && (FramePtr == Reg || Reg == AArch64::LR)) {
191 TotalSkipped += stackGrowth;
195 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
196 unsigned CFIIndex = MMI.addFrameInst(MCCFIInstruction::createOffset(
197 nullptr, DwarfReg, Offset - TotalSkipped));
198 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
199 .addCFIIndex(CFIIndex)
200 .setMIFlags(MachineInstr::FrameSetup);
204 void AArch64FrameLowering::emitPrologue(MachineFunction &MF) const {
205 MachineBasicBlock &MBB = MF.front(); // Prologue goes in entry BB.
206 MachineBasicBlock::iterator MBBI = MBB.begin();
207 const MachineFrameInfo *MFI = MF.getFrameInfo();
208 const Function *Fn = MF.getFunction();
209 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
210 MF.getSubtarget().getRegisterInfo());
211 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
212 MachineModuleInfo &MMI = MF.getMMI();
213 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
214 bool needsFrameMoves = MMI.hasDebugInfo() || Fn->needsUnwindTableEntry();
215 bool HasFP = hasFP(MF);
216 DebugLoc DL = MBB.findDebugLoc(MBBI);
218 // All calls are tail calls in GHC calling conv, and functions have no
219 // prologue/epilogue.
220 if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
223 int NumBytes = (int)MFI->getStackSize();
224 if (!AFI->hasStackFrame()) {
225 assert(!HasFP && "unexpected function without stack frame but with FP");
227 // All of the stack allocation is for locals.
228 AFI->setLocalStackSize(NumBytes);
230 // Label used to tie together the PROLOG_LABEL and the MachineMoves.
231 MCSymbol *FrameLabel = MMI.getContext().CreateTempSymbol();
233 // REDZONE: If the stack size is less than 128 bytes, we don't need
234 // to actually allocate.
235 if (NumBytes && !canUseRedZone(MF)) {
236 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
237 MachineInstr::FrameSetup);
239 // Encode the stack size of the leaf function.
240 unsigned CFIIndex = MMI.addFrameInst(
241 MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
242 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
243 .addCFIIndex(CFIIndex)
244 .setMIFlags(MachineInstr::FrameSetup);
245 } else if (NumBytes) {
246 ++NumRedZoneFunctions;
252 // Only set up FP if we actually need to.
255 // First instruction must a) allocate the stack and b) have an immediate
256 // that is a multiple of -2.
257 assert((MBBI->getOpcode() == AArch64::STPXpre ||
258 MBBI->getOpcode() == AArch64::STPDpre) &&
259 MBBI->getOperand(3).getReg() == AArch64::SP &&
260 MBBI->getOperand(4).getImm() < 0 &&
261 (MBBI->getOperand(4).getImm() & 1) == 0);
263 // Frame pointer is fp = sp - 16. Since the STPXpre subtracts the space
264 // required for the callee saved register area we get the frame pointer
265 // by addding that offset - 16 = -getImm()*8 - 2*8 = -(getImm() + 2) * 8.
266 FPOffset = -(MBBI->getOperand(4).getImm() + 2) * 8;
267 assert(FPOffset >= 0 && "Bad Framepointer Offset");
270 // Move past the saves of the callee-saved registers.
271 while (MBBI->getOpcode() == AArch64::STPXi ||
272 MBBI->getOpcode() == AArch64::STPDi ||
273 MBBI->getOpcode() == AArch64::STPXpre ||
274 MBBI->getOpcode() == AArch64::STPDpre) {
278 assert(NumBytes >= 0 && "Negative stack allocation size!?");
280 // Issue sub fp, sp, FPOffset or
281 // mov fp,sp when FPOffset is zero.
282 // Note: All stores of callee-saved registers are marked as "FrameSetup".
283 // This code marks the instruction(s) that set the FP also.
284 emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, FPOffset, TII,
285 MachineInstr::FrameSetup);
288 // All of the remaining stack allocations are for locals.
289 AFI->setLocalStackSize(NumBytes);
291 // Allocate space for the rest of the frame.
293 // If we're a leaf function, try using the red zone.
294 if (!canUseRedZone(MF))
295 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
296 MachineInstr::FrameSetup);
299 // If we need a base pointer, set it up here. It's whatever the value of the
300 // stack pointer is at this point. Any variable size objects will be allocated
301 // after this, so we can still use the base pointer to reference locals.
303 // FIXME: Clarify FrameSetup flags here.
304 // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
307 if (RegInfo->hasBasePointer(MF))
308 TII->copyPhysReg(MBB, MBBI, DL, AArch64::X19, AArch64::SP, false);
310 if (needsFrameMoves) {
311 const DataLayout *TD = MF.getSubtarget().getDataLayout();
312 const int StackGrowth = -TD->getPointerSize(0);
313 unsigned FramePtr = RegInfo->getFrameRegister(MF);
315 // An example of the prologue:
322 // .cfi_personality 155, ___gxx_personality_v0
324 // .cfi_lsda 16, Lexception33
326 // stp xa,bx, [sp, -#offset]!
328 // stp x28, x27, [sp, #offset-32]
329 // stp fp, lr, [sp, #offset-16]
330 // add fp, sp, #offset - 16
334 // +-------------------------------------------+
335 // 10000 | ........ | ........ | ........ | ........ |
336 // 10004 | ........ | ........ | ........ | ........ |
337 // +-------------------------------------------+
338 // 10008 | ........ | ........ | ........ | ........ |
339 // 1000c | ........ | ........ | ........ | ........ |
340 // +===========================================+
341 // 10010 | X28 Register |
342 // 10014 | X28 Register |
343 // +-------------------------------------------+
344 // 10018 | X27 Register |
345 // 1001c | X27 Register |
346 // +===========================================+
347 // 10020 | Frame Pointer |
348 // 10024 | Frame Pointer |
349 // +-------------------------------------------+
350 // 10028 | Link Register |
351 // 1002c | Link Register |
352 // +===========================================+
353 // 10030 | ........ | ........ | ........ | ........ |
354 // 10034 | ........ | ........ | ........ | ........ |
355 // +-------------------------------------------+
356 // 10038 | ........ | ........ | ........ | ........ |
357 // 1003c | ........ | ........ | ........ | ........ |
358 // +-------------------------------------------+
360 // [sp] = 10030 :: >>initial value<<
361 // sp = 10020 :: stp fp, lr, [sp, #-16]!
362 // fp = sp == 10020 :: mov fp, sp
363 // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
364 // sp == 10010 :: >>final value<<
366 // The frame pointer (w29) points to address 10020. If we use an offset of
367 // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
368 // for w27, and -32 for w28:
371 // .cfi_def_cfa w29, 16
373 // .cfi_offset w30, -8
375 // .cfi_offset w29, -16
377 // .cfi_offset w27, -24
379 // .cfi_offset w28, -32
382 // Define the current CFA rule to use the provided FP.
383 unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
384 unsigned CFIIndex = MMI.addFrameInst(
385 MCCFIInstruction::createDefCfa(nullptr, Reg, 2 * StackGrowth));
386 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
387 .addCFIIndex(CFIIndex)
388 .setMIFlags(MachineInstr::FrameSetup);
390 // Record the location of the stored LR
391 unsigned LR = RegInfo->getDwarfRegNum(AArch64::LR, true);
392 CFIIndex = MMI.addFrameInst(
393 MCCFIInstruction::createOffset(nullptr, LR, StackGrowth));
394 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
395 .addCFIIndex(CFIIndex)
396 .setMIFlags(MachineInstr::FrameSetup);
398 // Record the location of the stored FP
399 CFIIndex = MMI.addFrameInst(
400 MCCFIInstruction::createOffset(nullptr, Reg, 2 * StackGrowth));
401 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
402 .addCFIIndex(CFIIndex)
403 .setMIFlags(MachineInstr::FrameSetup);
405 // Encode the stack size of the leaf function.
406 unsigned CFIIndex = MMI.addFrameInst(
407 MCCFIInstruction::createDefCfaOffset(nullptr, -MFI->getStackSize()));
408 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
409 .addCFIIndex(CFIIndex)
410 .setMIFlags(MachineInstr::FrameSetup);
413 // Now emit the moves for whatever callee saved regs we have.
414 emitCalleeSavedFrameMoves(MBB, MBBI, FramePtr);
418 static bool isCalleeSavedRegister(unsigned Reg, const MCPhysReg *CSRegs) {
419 for (unsigned i = 0; CSRegs[i]; ++i)
420 if (Reg == CSRegs[i])
425 static bool isCSRestore(MachineInstr *MI, const MCPhysReg *CSRegs) {
427 if (MI->getOpcode() == AArch64::LDPXpost ||
428 MI->getOpcode() == AArch64::LDPDpost)
431 if (MI->getOpcode() == AArch64::LDPXpost ||
432 MI->getOpcode() == AArch64::LDPDpost ||
433 MI->getOpcode() == AArch64::LDPXi || MI->getOpcode() == AArch64::LDPDi) {
434 if (!isCalleeSavedRegister(MI->getOperand(RtIdx).getReg(), CSRegs) ||
435 !isCalleeSavedRegister(MI->getOperand(RtIdx + 1).getReg(), CSRegs) ||
436 MI->getOperand(RtIdx + 2).getReg() != AArch64::SP)
444 void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
445 MachineBasicBlock &MBB) const {
446 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
447 assert(MBBI->isReturn() && "Can only insert epilog into returning blocks");
448 MachineFrameInfo *MFI = MF.getFrameInfo();
449 const AArch64InstrInfo *TII =
450 static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
451 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
452 MF.getSubtarget().getRegisterInfo());
453 DebugLoc DL = MBBI->getDebugLoc();
454 unsigned RetOpcode = MBBI->getOpcode();
456 int NumBytes = MFI->getStackSize();
457 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
459 // All calls are tail calls in GHC calling conv, and functions have no
460 // prologue/epilogue.
461 if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
464 // Initial and residual are named for consitency with the prologue. Note that
465 // in the epilogue, the residual adjustment is executed first.
466 uint64_t ArgumentPopSize = 0;
467 if (RetOpcode == AArch64::TCRETURNdi || RetOpcode == AArch64::TCRETURNri) {
468 MachineOperand &StackAdjust = MBBI->getOperand(1);
470 // For a tail-call in a callee-pops-arguments environment, some or all of
471 // the stack may actually be in use for the call's arguments, this is
472 // calculated during LowerCall and consumed here...
473 ArgumentPopSize = StackAdjust.getImm();
475 // ... otherwise the amount to pop is *all* of the argument space,
476 // conveniently stored in the MachineFunctionInfo by
477 // LowerFormalArguments. This will, of course, be zero for the C calling
479 ArgumentPopSize = AFI->getArgumentStackToRestore();
482 // The stack frame should be like below,
484 // ---------------------- ---
486 // | BytesInStackArgArea| CalleeArgStackSize
487 // | (NumReusableBytes) | (of tail call)
490 // ---------------------| --- |
492 // | CalleeSavedReg | | |
493 // | (NumRestores * 16) | | |
495 // ---------------------| | NumBytes
496 // | | StackSize (StackAdjustUp)
497 // | LocalStackSize | | |
498 // | (covering callee | | |
501 // ---------------------- --- ---
503 // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
504 // = StackSize + ArgumentPopSize
506 // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
507 // it as the 2nd argument of AArch64ISD::TC_RETURN.
508 NumBytes += ArgumentPopSize;
510 unsigned NumRestores = 0;
511 // Move past the restores of the callee-saved registers.
512 MachineBasicBlock::iterator LastPopI = MBBI;
513 const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
514 if (LastPopI != MBB.begin()) {
518 } while (LastPopI != MBB.begin() && isCSRestore(LastPopI, CSRegs));
519 if (!isCSRestore(LastPopI, CSRegs)) {
524 NumBytes -= NumRestores * 16;
525 assert(NumBytes >= 0 && "Negative stack allocation size!?");
528 // If this was a redzone leaf function, we don't need to restore the
530 if (!canUseRedZone(MF))
531 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, NumBytes,
536 // Restore the original stack pointer.
537 // FIXME: Rather than doing the math here, we should instead just use
538 // non-post-indexed loads for the restores if we aren't actually going to
539 // be able to save any instructions.
540 if (NumBytes || MFI->hasVarSizedObjects())
541 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
542 -(NumRestores - 1) * 16, TII, MachineInstr::NoFlags);
545 /// getFrameIndexOffset - Returns the displacement from the frame register to
546 /// the stack frame of the specified index.
547 int AArch64FrameLowering::getFrameIndexOffset(const MachineFunction &MF,
550 return getFrameIndexReference(MF, FI, FrameReg);
553 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
554 /// debug info. It's the same as what we use for resolving the code-gen
555 /// references for now. FIXME: This can go wrong when references are
556 /// SP-relative and simple call frames aren't used.
557 int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
559 unsigned &FrameReg) const {
560 return resolveFrameIndexReference(MF, FI, FrameReg);
563 int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF,
564 int FI, unsigned &FrameReg,
565 bool PreferFP) const {
566 const MachineFrameInfo *MFI = MF.getFrameInfo();
567 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
568 MF.getSubtarget().getRegisterInfo());
569 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
570 int FPOffset = MFI->getObjectOffset(FI) + 16;
571 int Offset = MFI->getObjectOffset(FI) + MFI->getStackSize();
572 bool isFixed = MFI->isFixedObjectIndex(FI);
574 // Use frame pointer to reference fixed objects. Use it for locals if
575 // there are VLAs (and thus the SP isn't reliable as a base).
576 // Make sure useFPForScavengingIndex() does the right thing for the emergency
579 if (AFI->hasStackFrame()) {
580 // Note: Keeping the following as multiple 'if' statements rather than
581 // merging to a single expression for readability.
583 // Argument access should always use the FP.
586 } else if (hasFP(MF) && !RegInfo->hasBasePointer(MF)) {
587 // Use SP or FP, whichever gives us the best chance of the offset
588 // being in range for direct access. If the FPOffset is positive,
589 // that'll always be best, as the SP will be even further away.
590 // If the FPOffset is negative, we have to keep in mind that the
591 // available offset range for negative offsets is smaller than for
592 // positive ones. If we have variable sized objects, we're stuck with
593 // using the FP regardless, though, as the SP offset is unknown
594 // and we don't have a base pointer available. If an offset is
595 // available via the FP and the SP, use whichever is closest.
596 if (PreferFP || MFI->hasVarSizedObjects() || FPOffset >= 0 ||
597 (FPOffset >= -256 && Offset > -FPOffset))
603 FrameReg = RegInfo->getFrameRegister(MF);
607 // Use the base pointer if we have one.
608 if (RegInfo->hasBasePointer(MF))
609 FrameReg = RegInfo->getBaseRegister();
611 FrameReg = AArch64::SP;
612 // If we're using the red zone for this function, the SP won't actually
613 // be adjusted, so the offsets will be negative. They're also all
614 // within range of the signed 9-bit immediate instructions.
615 if (canUseRedZone(MF))
616 Offset -= AFI->getLocalStackSize();
622 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
623 if (Reg != AArch64::LR)
624 return getKillRegState(true);
626 // LR maybe referred to later by an @llvm.returnaddress intrinsic.
627 bool LRLiveIn = MF.getRegInfo().isLiveIn(AArch64::LR);
628 bool LRKill = !(LRLiveIn && MF.getFrameInfo()->isReturnAddressTaken());
629 return getKillRegState(LRKill);
632 bool AArch64FrameLowering::spillCalleeSavedRegisters(
633 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
634 const std::vector<CalleeSavedInfo> &CSI,
635 const TargetRegisterInfo *TRI) const {
636 MachineFunction &MF = *MBB.getParent();
637 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
638 unsigned Count = CSI.size();
640 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
643 DL = MI->getDebugLoc();
645 for (unsigned i = 0; i < Count; i += 2) {
646 unsigned idx = Count - i - 2;
647 unsigned Reg1 = CSI[idx].getReg();
648 unsigned Reg2 = CSI[idx + 1].getReg();
649 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
650 // list to come in sorted by frame index so that we can issue the store
651 // pair instructions directly. Assert if we see anything otherwise.
653 // The order of the registers in the list is controlled by
654 // getCalleeSavedRegs(), so they will always be in-order, as well.
655 assert(CSI[idx].getFrameIdx() + 1 == CSI[idx + 1].getFrameIdx() &&
656 "Out of order callee saved regs!");
658 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
659 assert((i & 1) == 0 && "Odd index for callee-saved reg spill!");
660 // Issue sequence of non-sp increment and pi sp spills for cs regs. The
661 // first spill is a pre-increment that allocates the stack.
663 // stp x22, x21, [sp, #-48]! // addImm(-6)
664 // stp x20, x19, [sp, #16] // addImm(+2)
665 // stp fp, lr, [sp, #32] // addImm(+4)
666 // Rationale: This sequence saves uop updates compared to a sequence of
667 // pre-increment spills like stp xi,xj,[sp,#-16]!
668 // Note: Similar rational and sequence for restores in epilog.
669 if (AArch64::GPR64RegClass.contains(Reg1)) {
670 assert(AArch64::GPR64RegClass.contains(Reg2) &&
671 "Expected GPR64 callee-saved register pair!");
672 // For first spill use pre-increment store.
674 StrOpc = AArch64::STPXpre;
676 StrOpc = AArch64::STPXi;
677 } else if (AArch64::FPR64RegClass.contains(Reg1)) {
678 assert(AArch64::FPR64RegClass.contains(Reg2) &&
679 "Expected FPR64 callee-saved register pair!");
680 // For first spill use pre-increment store.
682 StrOpc = AArch64::STPDpre;
684 StrOpc = AArch64::STPDi;
686 llvm_unreachable("Unexpected callee saved register!");
687 DEBUG(dbgs() << "CSR spill: (" << TRI->getName(Reg1) << ", "
688 << TRI->getName(Reg2) << ") -> fi#(" << CSI[idx].getFrameIdx()
689 << ", " << CSI[idx + 1].getFrameIdx() << ")\n");
690 // Compute offset: i = 0 => offset = -Count;
691 // i = 2 => offset = -(Count - 2) + Count = 2 = i; etc.
692 const int Offset = (i == 0) ? -Count : i;
693 assert((Offset >= -64 && Offset <= 63) &&
694 "Offset out of bounds for STP immediate");
695 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
696 if (StrOpc == AArch64::STPDpre || StrOpc == AArch64::STPXpre)
697 MIB.addReg(AArch64::SP, RegState::Define);
699 MIB.addReg(Reg2, getPrologueDeath(MF, Reg2))
700 .addReg(Reg1, getPrologueDeath(MF, Reg1))
702 .addImm(Offset) // [sp, #offset * 8], where factor * 8 is implicit
703 .setMIFlag(MachineInstr::FrameSetup);
708 bool AArch64FrameLowering::restoreCalleeSavedRegisters(
709 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
710 const std::vector<CalleeSavedInfo> &CSI,
711 const TargetRegisterInfo *TRI) const {
712 MachineFunction &MF = *MBB.getParent();
713 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
714 unsigned Count = CSI.size();
716 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
719 DL = MI->getDebugLoc();
721 for (unsigned i = 0; i < Count; i += 2) {
722 unsigned Reg1 = CSI[i].getReg();
723 unsigned Reg2 = CSI[i + 1].getReg();
724 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
725 // list to come in sorted by frame index so that we can issue the store
726 // pair instructions directly. Assert if we see anything otherwise.
727 assert(CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx() &&
728 "Out of order callee saved regs!");
729 // Issue sequence of non-sp increment and sp-pi restores for cs regs. Only
730 // the last load is sp-pi post-increment and de-allocates the stack:
732 // ldp fp, lr, [sp, #32] // addImm(+4)
733 // ldp x20, x19, [sp, #16] // addImm(+2)
734 // ldp x22, x21, [sp], #48 // addImm(+6)
735 // Note: see comment in spillCalleeSavedRegisters()
738 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
739 assert((i & 1) == 0 && "Odd index for callee-saved reg spill!");
740 if (AArch64::GPR64RegClass.contains(Reg1)) {
741 assert(AArch64::GPR64RegClass.contains(Reg2) &&
742 "Expected GPR64 callee-saved register pair!");
744 LdrOpc = AArch64::LDPXpost;
746 LdrOpc = AArch64::LDPXi;
747 } else if (AArch64::FPR64RegClass.contains(Reg1)) {
748 assert(AArch64::FPR64RegClass.contains(Reg2) &&
749 "Expected FPR64 callee-saved register pair!");
751 LdrOpc = AArch64::LDPDpost;
753 LdrOpc = AArch64::LDPDi;
755 llvm_unreachable("Unexpected callee saved register!");
756 DEBUG(dbgs() << "CSR restore: (" << TRI->getName(Reg1) << ", "
757 << TRI->getName(Reg2) << ") -> fi#(" << CSI[i].getFrameIdx()
758 << ", " << CSI[i + 1].getFrameIdx() << ")\n");
760 // Compute offset: i = 0 => offset = Count - 2; i = 2 => offset = Count - 4;
762 const int Offset = (i == Count - 2) ? Count : Count - i - 2;
763 assert((Offset >= -64 && Offset <= 63) &&
764 "Offset out of bounds for LDP immediate");
765 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
766 if (LdrOpc == AArch64::LDPXpost || LdrOpc == AArch64::LDPDpost)
767 MIB.addReg(AArch64::SP, RegState::Define);
769 MIB.addReg(Reg2, getDefRegState(true))
770 .addReg(Reg1, getDefRegState(true))
772 .addImm(Offset); // [sp], #offset * 8 or [sp, #offset * 8]
773 // where the factor * 8 is implicit
778 void AArch64FrameLowering::processFunctionBeforeCalleeSavedScan(
779 MachineFunction &MF, RegScavenger *RS) const {
780 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
781 MF.getSubtarget().getRegisterInfo());
782 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
783 MachineRegisterInfo *MRI = &MF.getRegInfo();
784 SmallVector<unsigned, 4> UnspilledCSGPRs;
785 SmallVector<unsigned, 4> UnspilledCSFPRs;
787 // The frame record needs to be created by saving the appropriate registers
789 MRI->setPhysRegUsed(AArch64::FP);
790 MRI->setPhysRegUsed(AArch64::LR);
793 // Spill the BasePtr if it's used. Do this first thing so that the
794 // getCalleeSavedRegs() below will get the right answer.
795 if (RegInfo->hasBasePointer(MF))
796 MRI->setPhysRegUsed(RegInfo->getBaseRegister());
798 // If any callee-saved registers are used, the frame cannot be eliminated.
799 unsigned NumGPRSpilled = 0;
800 unsigned NumFPRSpilled = 0;
801 bool ExtraCSSpill = false;
802 bool CanEliminateFrame = true;
803 DEBUG(dbgs() << "*** processFunctionBeforeCalleeSavedScan\nUsed CSRs:");
804 const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
806 // Check pairs of consecutive callee-saved registers.
807 for (unsigned i = 0; CSRegs[i]; i += 2) {
808 assert(CSRegs[i + 1] && "Odd number of callee-saved registers!");
810 const unsigned OddReg = CSRegs[i];
811 const unsigned EvenReg = CSRegs[i + 1];
812 assert((AArch64::GPR64RegClass.contains(OddReg) &&
813 AArch64::GPR64RegClass.contains(EvenReg)) ^
814 (AArch64::FPR64RegClass.contains(OddReg) &&
815 AArch64::FPR64RegClass.contains(EvenReg)) &&
816 "Register class mismatch!");
818 const bool OddRegUsed = MRI->isPhysRegUsed(OddReg);
819 const bool EvenRegUsed = MRI->isPhysRegUsed(EvenReg);
821 // Early exit if none of the registers in the register pair is actually
823 if (!OddRegUsed && !EvenRegUsed) {
824 if (AArch64::GPR64RegClass.contains(OddReg)) {
825 UnspilledCSGPRs.push_back(OddReg);
826 UnspilledCSGPRs.push_back(EvenReg);
828 UnspilledCSFPRs.push_back(OddReg);
829 UnspilledCSFPRs.push_back(EvenReg);
834 unsigned Reg = AArch64::NoRegister;
835 // If only one of the registers of the register pair is used, make sure to
836 // mark the other one as used as well.
837 if (OddRegUsed ^ EvenRegUsed) {
838 // Find out which register is the additional spill.
839 Reg = OddRegUsed ? EvenReg : OddReg;
840 MRI->setPhysRegUsed(Reg);
843 DEBUG(dbgs() << ' ' << PrintReg(OddReg, RegInfo));
844 DEBUG(dbgs() << ' ' << PrintReg(EvenReg, RegInfo));
846 assert(((OddReg == AArch64::LR && EvenReg == AArch64::FP) ||
847 (RegInfo->getEncodingValue(OddReg) + 1 ==
848 RegInfo->getEncodingValue(EvenReg))) &&
849 "Register pair of non-adjacent registers!");
850 if (AArch64::GPR64RegClass.contains(OddReg)) {
852 // If it's not a reserved register, we can use it in lieu of an
853 // emergency spill slot for the register scavenger.
854 // FIXME: It would be better to instead keep looking and choose another
855 // unspilled register that isn't reserved, if there is one.
856 if (Reg != AArch64::NoRegister && !RegInfo->isReservedReg(MF, Reg))
861 CanEliminateFrame = false;
864 // FIXME: Set BigStack if any stack slot references may be out of range.
865 // For now, just conservatively guestimate based on unscaled indexing
866 // range. We'll end up allocating an unnecessary spill slot a lot, but
867 // realistically that's not a big deal at this stage of the game.
868 // The CSR spill slots have not been allocated yet, so estimateStackSize
869 // won't include them.
870 MachineFrameInfo *MFI = MF.getFrameInfo();
871 unsigned CFSize = estimateStackSize(MF) + 8 * (NumGPRSpilled + NumFPRSpilled);
872 DEBUG(dbgs() << "Estimated stack frame size: " << CFSize << " bytes.\n");
873 bool BigStack = (CFSize >= 256);
874 if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
875 AFI->setHasStackFrame(true);
877 // Estimate if we might need to scavenge a register at some point in order
878 // to materialize a stack offset. If so, either spill one additional
879 // callee-saved register or reserve a special spill slot to facilitate
880 // register scavenging. If we already spilled an extra callee-saved register
881 // above to keep the number of spills even, we don't need to do anything else
883 if (BigStack && !ExtraCSSpill) {
885 // If we're adding a register to spill here, we have to add two of them
886 // to keep the number of regs to spill even.
887 assert(((UnspilledCSGPRs.size() & 1) == 0) && "Odd number of registers!");
889 while (!UnspilledCSGPRs.empty() && Count < 2) {
890 unsigned Reg = UnspilledCSGPRs.back();
891 UnspilledCSGPRs.pop_back();
892 DEBUG(dbgs() << "Spilling " << PrintReg(Reg, RegInfo)
893 << " to get a scratch register.\n");
894 MRI->setPhysRegUsed(Reg);
899 // If we didn't find an extra callee-saved register to spill, create
900 // an emergency spill slot.
902 const TargetRegisterClass *RC = &AArch64::GPR64RegClass;
903 int FI = MFI->CreateStackObject(RC->getSize(), RC->getAlignment(), false);
904 RS->addScavengingFrameIndex(FI);
905 DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
906 << " as the emergency spill slot.\n");