1 //===- ARM64FrameLowering.cpp - ARM64 Frame Lowering -----------*- C++ -*-====//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the ARM64 implementation of TargetFrameLowering class.
12 //===----------------------------------------------------------------------===//
14 #define DEBUG_TYPE "frame-info"
15 #include "ARM64FrameLowering.h"
16 #include "ARM64InstrInfo.h"
17 #include "ARM64MachineFunctionInfo.h"
18 #include "ARM64Subtarget.h"
19 #include "ARM64TargetMachine.h"
20 #include "llvm/ADT/Statistic.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Function.h"
23 #include "llvm/CodeGen/MachineFrameInfo.h"
24 #include "llvm/CodeGen/MachineFunction.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineModuleInfo.h"
27 #include "llvm/CodeGen/MachineRegisterInfo.h"
28 #include "llvm/CodeGen/RegisterScavenging.h"
29 #include "llvm/Support/Debug.h"
30 #include "llvm/Support/CommandLine.h"
31 #include "llvm/Support/raw_ostream.h"
35 static cl::opt<bool> EnableRedZone("arm64-redzone",
36 cl::desc("enable use of redzone on ARM64"),
37 cl::init(false), cl::Hidden);
39 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
41 static unsigned estimateStackSize(MachineFunction &MF) {
42 const MachineFrameInfo *FFI = MF.getFrameInfo();
44 for (int i = FFI->getObjectIndexBegin(); i != 0; ++i) {
45 int FixedOff = -FFI->getObjectOffset(i);
46 if (FixedOff > Offset)
49 for (unsigned i = 0, e = FFI->getObjectIndexEnd(); i != e; ++i) {
50 if (FFI->isDeadObjectIndex(i))
52 Offset += FFI->getObjectSize(i);
53 unsigned Align = FFI->getObjectAlignment(i);
54 // Adjust to alignment boundary
55 Offset = (Offset + Align - 1) / Align * Align;
57 // This does not include the 16 bytes used for fp and lr.
58 return (unsigned)Offset;
61 bool ARM64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
64 // Don't use the red zone if the function explicitly asks us not to.
65 // This is typically used for kernel code.
66 if (MF.getFunction()->getAttributes().hasAttribute(
67 AttributeSet::FunctionIndex, Attribute::NoRedZone))
70 const MachineFrameInfo *MFI = MF.getFrameInfo();
71 const ARM64FunctionInfo *AFI = MF.getInfo<ARM64FunctionInfo>();
72 unsigned NumBytes = AFI->getLocalStackSize();
74 // Note: currently hasFP() is always true for hasCalls(), but that's an
75 // implementation detail of the current code, not a strict requirement,
76 // so stay safe here and check both.
77 if (MFI->hasCalls() || hasFP(MF) || NumBytes > 128)
82 /// hasFP - Return true if the specified function should have a dedicated frame
84 bool ARM64FrameLowering::hasFP(const MachineFunction &MF) const {
85 const MachineFrameInfo *MFI = MF.getFrameInfo();
88 const TargetRegisterInfo *RegInfo = MF.getTarget().getRegisterInfo();
89 assert(!RegInfo->needsStackRealignment(MF) &&
90 "No stack realignment on ARM64!");
93 return (MFI->hasCalls() || MFI->hasVarSizedObjects() ||
94 MFI->isFrameAddressTaken());
97 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
98 /// not required, we reserve argument space for call sites in the function
99 /// immediately on entry to the current function. This eliminates the need for
100 /// add/sub sp brackets around call sites. Returns true if the call frame is
101 /// included as part of the stack frame.
102 bool ARM64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
103 return !MF.getFrameInfo()->hasVarSizedObjects();
106 void ARM64FrameLowering::eliminateCallFramePseudoInstr(
107 MachineFunction &MF, MachineBasicBlock &MBB,
108 MachineBasicBlock::iterator I) const {
109 const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
110 const ARM64InstrInfo *TII =
111 static_cast<const ARM64InstrInfo *>(MF.getTarget().getInstrInfo());
112 if (!TFI->hasReservedCallFrame(MF)) {
113 // If we have alloca, convert as follows:
114 // ADJCALLSTACKDOWN -> sub, sp, sp, amount
115 // ADJCALLSTACKUP -> add, sp, sp, amount
116 MachineInstr *Old = I;
117 DebugLoc DL = Old->getDebugLoc();
118 unsigned Amount = Old->getOperand(0).getImm();
120 // We need to keep the stack aligned properly. To do this, we round the
121 // amount of space needed for the outgoing arguments up to the next
122 // alignment boundary.
123 unsigned Align = TFI->getStackAlignment();
124 Amount = (Amount + Align - 1) / Align * Align;
126 // Replace the pseudo instruction with a new instruction...
127 unsigned Opc = Old->getOpcode();
128 if (Opc == ARM64::ADJCALLSTACKDOWN) {
129 emitFrameOffset(MBB, I, DL, ARM64::SP, ARM64::SP, -Amount, TII);
131 assert(Opc == ARM64::ADJCALLSTACKUP && "expected ADJCALLSTACKUP");
132 emitFrameOffset(MBB, I, DL, ARM64::SP, ARM64::SP, Amount, TII);
140 ARM64FrameLowering::emitCalleeSavedFrameMoves(MachineBasicBlock &MBB,
141 MachineBasicBlock::iterator MBBI,
142 unsigned FramePtr) const {
143 MachineFunction &MF = *MBB.getParent();
144 MachineFrameInfo *MFI = MF.getFrameInfo();
145 MachineModuleInfo &MMI = MF.getMMI();
146 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
147 const ARM64InstrInfo *TII = TM.getInstrInfo();
148 DebugLoc DL = MBB.findDebugLoc(MBBI);
150 // Add callee saved registers to move list.
151 const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
155 const DataLayout *TD = MF.getTarget().getDataLayout();
156 bool HasFP = hasFP(MF);
158 // Calculate amount of bytes used for return address storing.
159 int stackGrowth = -TD->getPointerSize(0);
161 // Calculate offsets.
162 int64_t saveAreaOffset = (HasFP ? 2 : 1) * stackGrowth;
163 unsigned TotalSkipped = 0;
164 for (std::vector<CalleeSavedInfo>::const_iterator I = CSI.begin(),
167 unsigned Reg = I->getReg();
168 int64_t Offset = MFI->getObjectOffset(I->getFrameIdx()) -
169 getOffsetOfLocalArea() + saveAreaOffset;
171 // Don't output a new CFI directive if we're re-saving the frame pointer or
172 // link register. This happens when the PrologEpilogInserter has inserted an
173 // extra "STP" of the frame pointer and link register -- the "emitPrologue"
174 // method automatically generates the directives when frame pointers are
175 // used. If we generate CFI directives for the extra "STP"s, the linker will
176 // lose track of the correct values for the frame pointer and link register.
177 if (HasFP && (FramePtr == Reg || Reg == ARM64::LR)) {
178 TotalSkipped += stackGrowth;
182 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
183 unsigned CFIIndex = MMI.addFrameInst(MCCFIInstruction::createOffset(
184 nullptr, DwarfReg, Offset - TotalSkipped));
185 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
186 .addCFIIndex(CFIIndex);
190 void ARM64FrameLowering::emitPrologue(MachineFunction &MF) const {
191 MachineBasicBlock &MBB = MF.front(); // Prologue goes in entry BB.
192 MachineBasicBlock::iterator MBBI = MBB.begin();
193 const MachineFrameInfo *MFI = MF.getFrameInfo();
194 const Function *Fn = MF.getFunction();
195 const ARM64RegisterInfo *RegInfo = TM.getRegisterInfo();
196 const ARM64InstrInfo *TII = TM.getInstrInfo();
197 MachineModuleInfo &MMI = MF.getMMI();
198 ARM64FunctionInfo *AFI = MF.getInfo<ARM64FunctionInfo>();
199 bool needsFrameMoves = MMI.hasDebugInfo() || Fn->needsUnwindTableEntry();
200 bool HasFP = hasFP(MF);
201 DebugLoc DL = MBB.findDebugLoc(MBBI);
203 int NumBytes = (int)MFI->getStackSize();
204 if (!AFI->hasStackFrame()) {
205 assert(!HasFP && "unexpected function without stack frame but with FP");
207 // All of the stack allocation is for locals.
208 AFI->setLocalStackSize(NumBytes);
210 // Label used to tie together the PROLOG_LABEL and the MachineMoves.
211 MCSymbol *FrameLabel = MMI.getContext().CreateTempSymbol();
213 // REDZONE: If the stack size is less than 128 bytes, we don't need
214 // to actually allocate.
215 if (NumBytes && !canUseRedZone(MF)) {
216 emitFrameOffset(MBB, MBBI, DL, ARM64::SP, ARM64::SP, -NumBytes, TII,
217 MachineInstr::FrameSetup);
219 // Encode the stack size of the leaf function.
220 unsigned CFIIndex = MMI.addFrameInst(
221 MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
222 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
223 .addCFIIndex(CFIIndex);
224 } else if (NumBytes) {
225 ++NumRedZoneFunctions;
231 // Only set up FP if we actually need to.
234 // First instruction must a) allocate the stack and b) have an immediate
235 // that is a multiple of -2.
236 assert((MBBI->getOpcode() == ARM64::STPXpre ||
237 MBBI->getOpcode() == ARM64::STPDpre) &&
238 MBBI->getOperand(2).getReg() == ARM64::SP &&
239 MBBI->getOperand(3).getImm() < 0 &&
240 (MBBI->getOperand(3).getImm() & 1) == 0);
242 // Frame pointer is fp = sp - 16. Since the STPXpre subtracts the space
243 // required for the callee saved register area we get the frame pointer
244 // by addding that offset - 16 = -getImm()*8 - 2*8 = -(getImm() + 2) * 8.
245 FPOffset = -(MBBI->getOperand(3).getImm() + 2) * 8;
246 assert(FPOffset >= 0 && "Bad Framepointer Offset");
249 // Move past the saves of the callee-saved registers.
250 while (MBBI->getOpcode() == ARM64::STPXi ||
251 MBBI->getOpcode() == ARM64::STPDi ||
252 MBBI->getOpcode() == ARM64::STPXpre ||
253 MBBI->getOpcode() == ARM64::STPDpre) {
257 assert(NumBytes >= 0 && "Negative stack allocation size!?");
259 // Issue sub fp, sp, FPOffset or
260 // mov fp,sp when FPOffset is zero.
261 // Note: All stores of callee-saved registers are marked as "FrameSetup".
262 // This code marks the instruction(s) that set the FP also.
263 emitFrameOffset(MBB, MBBI, DL, ARM64::FP, ARM64::SP, FPOffset, TII,
264 MachineInstr::FrameSetup);
267 // All of the remaining stack allocations are for locals.
268 AFI->setLocalStackSize(NumBytes);
270 // Allocate space for the rest of the frame.
272 // If we're a leaf function, try using the red zone.
273 if (!canUseRedZone(MF))
274 emitFrameOffset(MBB, MBBI, DL, ARM64::SP, ARM64::SP, -NumBytes, TII,
275 MachineInstr::FrameSetup);
278 // If we need a base pointer, set it up here. It's whatever the value of the
279 // stack pointer is at this point. Any variable size objects will be allocated
280 // after this, so we can still use the base pointer to reference locals.
282 // FIXME: Clarify FrameSetup flags here.
283 // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
286 if (RegInfo->hasBasePointer(MF))
287 TII->copyPhysReg(MBB, MBBI, DL, ARM64::X19, ARM64::SP, false);
289 if (needsFrameMoves) {
290 const DataLayout *TD = MF.getTarget().getDataLayout();
291 const int StackGrowth = -TD->getPointerSize(0);
292 unsigned FramePtr = RegInfo->getFrameRegister(MF);
294 // An example of the prologue:
301 // .cfi_personality 155, ___gxx_personality_v0
303 // .cfi_lsda 16, Lexception33
305 // stp xa,bx, [sp, -#offset]!
307 // stp x28, x27, [sp, #offset-32]
308 // stp fp, lr, [sp, #offset-16]
309 // add fp, sp, #offset - 16
313 // +-------------------------------------------+
314 // 10000 | ........ | ........ | ........ | ........ |
315 // 10004 | ........ | ........ | ........ | ........ |
316 // +-------------------------------------------+
317 // 10008 | ........ | ........ | ........ | ........ |
318 // 1000c | ........ | ........ | ........ | ........ |
319 // +===========================================+
320 // 10010 | X28 Register |
321 // 10014 | X28 Register |
322 // +-------------------------------------------+
323 // 10018 | X27 Register |
324 // 1001c | X27 Register |
325 // +===========================================+
326 // 10020 | Frame Pointer |
327 // 10024 | Frame Pointer |
328 // +-------------------------------------------+
329 // 10028 | Link Register |
330 // 1002c | Link Register |
331 // +===========================================+
332 // 10030 | ........ | ........ | ........ | ........ |
333 // 10034 | ........ | ........ | ........ | ........ |
334 // +-------------------------------------------+
335 // 10038 | ........ | ........ | ........ | ........ |
336 // 1003c | ........ | ........ | ........ | ........ |
337 // +-------------------------------------------+
339 // [sp] = 10030 :: >>initial value<<
340 // sp = 10020 :: stp fp, lr, [sp, #-16]!
341 // fp = sp == 10020 :: mov fp, sp
342 // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
343 // sp == 10010 :: >>final value<<
345 // The frame pointer (w29) points to address 10020. If we use an offset of
346 // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
347 // for w27, and -32 for w28:
350 // .cfi_def_cfa w29, 16
352 // .cfi_offset w30, -8
354 // .cfi_offset w29, -16
356 // .cfi_offset w27, -24
358 // .cfi_offset w28, -32
361 // Define the current CFA rule to use the provided FP.
362 unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
363 unsigned CFIIndex = MMI.addFrameInst(
364 MCCFIInstruction::createDefCfa(nullptr, Reg, 2 * StackGrowth));
365 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
366 .addCFIIndex(CFIIndex);
368 // Record the location of the stored LR
369 unsigned LR = RegInfo->getDwarfRegNum(ARM64::LR, true);
370 CFIIndex = MMI.addFrameInst(
371 MCCFIInstruction::createOffset(nullptr, LR, StackGrowth));
372 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
373 .addCFIIndex(CFIIndex);
375 // Record the location of the stored FP
376 CFIIndex = MMI.addFrameInst(
377 MCCFIInstruction::createOffset(nullptr, Reg, 2 * StackGrowth));
378 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
379 .addCFIIndex(CFIIndex);
381 // Encode the stack size of the leaf function.
382 unsigned CFIIndex = MMI.addFrameInst(
383 MCCFIInstruction::createDefCfaOffset(nullptr, -MFI->getStackSize()));
384 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
385 .addCFIIndex(CFIIndex);
388 // Now emit the moves for whatever callee saved regs we have.
389 emitCalleeSavedFrameMoves(MBB, MBBI, FramePtr);
393 static bool isCalleeSavedRegister(unsigned Reg, const uint16_t *CSRegs) {
394 for (unsigned i = 0; CSRegs[i]; ++i)
395 if (Reg == CSRegs[i])
400 static bool isCSRestore(MachineInstr *MI, const uint16_t *CSRegs) {
401 if (MI->getOpcode() == ARM64::LDPXpost ||
402 MI->getOpcode() == ARM64::LDPDpost || MI->getOpcode() == ARM64::LDPXi ||
403 MI->getOpcode() == ARM64::LDPDi) {
404 if (!isCalleeSavedRegister(MI->getOperand(0).getReg(), CSRegs) ||
405 !isCalleeSavedRegister(MI->getOperand(1).getReg(), CSRegs) ||
406 MI->getOperand(2).getReg() != ARM64::SP)
414 void ARM64FrameLowering::emitEpilogue(MachineFunction &MF,
415 MachineBasicBlock &MBB) const {
416 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
417 assert(MBBI->isReturn() && "Can only insert epilog into returning blocks");
418 MachineFrameInfo *MFI = MF.getFrameInfo();
419 const ARM64InstrInfo *TII =
420 static_cast<const ARM64InstrInfo *>(MF.getTarget().getInstrInfo());
421 const ARM64RegisterInfo *RegInfo =
422 static_cast<const ARM64RegisterInfo *>(MF.getTarget().getRegisterInfo());
423 DebugLoc DL = MBBI->getDebugLoc();
425 int NumBytes = MFI->getStackSize();
426 unsigned NumRestores = 0;
427 // Move past the restores of the callee-saved registers.
428 MachineBasicBlock::iterator LastPopI = MBBI;
429 const uint16_t *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
430 if (LastPopI != MBB.begin()) {
434 } while (LastPopI != MBB.begin() && isCSRestore(LastPopI, CSRegs));
435 if (!isCSRestore(LastPopI, CSRegs)) {
440 NumBytes -= NumRestores * 16;
441 assert(NumBytes >= 0 && "Negative stack allocation size!?");
444 // If this was a redzone leaf function, we don't need to restore the
446 if (!canUseRedZone(MF))
447 emitFrameOffset(MBB, LastPopI, DL, ARM64::SP, ARM64::SP, NumBytes, TII);
451 // Restore the original stack pointer.
452 // FIXME: Rather than doing the math here, we should instead just use
453 // non-post-indexed loads for the restores if we aren't actually going to
454 // be able to save any instructions.
455 if (NumBytes || MFI->hasVarSizedObjects())
456 emitFrameOffset(MBB, LastPopI, DL, ARM64::SP, ARM64::FP,
457 -(NumRestores - 1) * 16, TII, MachineInstr::NoFlags);
460 /// getFrameIndexOffset - Returns the displacement from the frame register to
461 /// the stack frame of the specified index.
462 int ARM64FrameLowering::getFrameIndexOffset(const MachineFunction &MF,
465 return getFrameIndexReference(MF, FI, FrameReg);
468 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
469 /// debug info. It's the same as what we use for resolving the code-gen
470 /// references for now. FIXME: This can go wrong when references are
471 /// SP-relative and simple call frames aren't used.
472 int ARM64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
474 unsigned &FrameReg) const {
475 return resolveFrameIndexReference(MF, FI, FrameReg);
478 int ARM64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF,
479 int FI, unsigned &FrameReg,
480 bool PreferFP) const {
481 const MachineFrameInfo *MFI = MF.getFrameInfo();
482 const ARM64RegisterInfo *RegInfo =
483 static_cast<const ARM64RegisterInfo *>(MF.getTarget().getRegisterInfo());
484 const ARM64FunctionInfo *AFI = MF.getInfo<ARM64FunctionInfo>();
485 int FPOffset = MFI->getObjectOffset(FI) + 16;
486 int Offset = MFI->getObjectOffset(FI) + MFI->getStackSize();
487 bool isFixed = MFI->isFixedObjectIndex(FI);
489 // Use frame pointer to reference fixed objects. Use it for locals if
490 // there are VLAs (and thus the SP isn't reliable as a base).
491 // Make sure useFPForScavengingIndex() does the right thing for the emergency
494 if (AFI->hasStackFrame()) {
495 // Note: Keeping the following as multiple 'if' statements rather than
496 // merging to a single expression for readability.
498 // Argument access should always use the FP.
501 } else if (hasFP(MF) && !RegInfo->hasBasePointer(MF)) {
502 // Use SP or FP, whichever gives us the best chance of the offset
503 // being in range for direct access. If the FPOffset is positive,
504 // that'll always be best, as the SP will be even further away.
505 // If the FPOffset is negative, we have to keep in mind that the
506 // available offset range for negative offsets is smaller than for
507 // positive ones. If we have variable sized objects, we're stuck with
508 // using the FP regardless, though, as the SP offset is unknown
509 // and we don't have a base pointer available. If an offset is
510 // available via the FP and the SP, use whichever is closest.
511 if (PreferFP || MFI->hasVarSizedObjects() || FPOffset >= 0 ||
512 (FPOffset >= -256 && Offset > -FPOffset))
518 FrameReg = RegInfo->getFrameRegister(MF);
522 // Use the base pointer if we have one.
523 if (RegInfo->hasBasePointer(MF))
524 FrameReg = RegInfo->getBaseRegister();
526 FrameReg = ARM64::SP;
527 // If we're using the red zone for this function, the SP won't actually
528 // be adjusted, so the offsets will be negative. They're also all
529 // within range of the signed 9-bit immediate instructions.
530 if (canUseRedZone(MF))
531 Offset -= AFI->getLocalStackSize();
537 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
538 if (Reg != ARM64::LR)
539 return getKillRegState(true);
541 // LR maybe referred to later by an @llvm.returnaddress intrinsic.
542 bool LRLiveIn = MF.getRegInfo().isLiveIn(ARM64::LR);
543 bool LRKill = !(LRLiveIn && MF.getFrameInfo()->isReturnAddressTaken());
544 return getKillRegState(LRKill);
547 bool ARM64FrameLowering::spillCalleeSavedRegisters(
548 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
549 const std::vector<CalleeSavedInfo> &CSI,
550 const TargetRegisterInfo *TRI) const {
551 MachineFunction &MF = *MBB.getParent();
552 const TargetInstrInfo &TII = *MF.getTarget().getInstrInfo();
553 unsigned Count = CSI.size();
555 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
558 DL = MI->getDebugLoc();
560 for (unsigned i = 0; i < Count; i += 2) {
561 unsigned idx = Count - i - 2;
562 unsigned Reg1 = CSI[idx].getReg();
563 unsigned Reg2 = CSI[idx + 1].getReg();
564 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
565 // list to come in sorted by frame index so that we can issue the store
566 // pair instructions directly. Assert if we see anything otherwise.
568 // The order of the registers in the list is controlled by
569 // getCalleeSavedRegs(), so they will always be in-order, as well.
570 assert(CSI[idx].getFrameIdx() + 1 == CSI[idx + 1].getFrameIdx() &&
571 "Out of order callee saved regs!");
573 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
574 assert((i & 1) == 0 && "Odd index for callee-saved reg spill!");
575 // Issue sequence of non-sp increment and pi sp spills for cs regs. The
576 // first spill is a pre-increment that allocates the stack.
578 // stp x22, x21, [sp, #-48]! // addImm(-6)
579 // stp x20, x19, [sp, #16] // addImm(+2)
580 // stp fp, lr, [sp, #32] // addImm(+4)
581 // Rationale: This sequence saves uop updates compared to a sequence of
582 // pre-increment spills like stp xi,xj,[sp,#-16]!
583 // Note: Similar rational and sequence for restores in epilog.
584 if (ARM64::GPR64RegClass.contains(Reg1)) {
585 assert(ARM64::GPR64RegClass.contains(Reg2) &&
586 "Expected GPR64 callee-saved register pair!");
587 // For first spill use pre-increment store.
589 StrOpc = ARM64::STPXpre;
591 StrOpc = ARM64::STPXi;
592 } else if (ARM64::FPR64RegClass.contains(Reg1)) {
593 assert(ARM64::FPR64RegClass.contains(Reg2) &&
594 "Expected FPR64 callee-saved register pair!");
595 // For first spill use pre-increment store.
597 StrOpc = ARM64::STPDpre;
599 StrOpc = ARM64::STPDi;
601 llvm_unreachable("Unexpected callee saved register!");
602 DEBUG(dbgs() << "CSR spill: (" << TRI->getName(Reg1) << ", "
603 << TRI->getName(Reg2) << ") -> fi#(" << CSI[idx].getFrameIdx()
604 << ", " << CSI[idx + 1].getFrameIdx() << ")\n");
605 // Compute offset: i = 0 => offset = -Count;
606 // i = 2 => offset = -(Count - 2) + Count = 2 = i; etc.
607 const int Offset = (i == 0) ? -Count : i;
608 assert((Offset >= -64 && Offset <= 63) &&
609 "Offset out of bounds for STP immediate");
610 BuildMI(MBB, MI, DL, TII.get(StrOpc))
611 .addReg(Reg2, getPrologueDeath(MF, Reg2))
612 .addReg(Reg1, getPrologueDeath(MF, Reg1))
614 .addImm(Offset) // [sp, #offset * 8], where factor * 8 is implicit
615 .setMIFlag(MachineInstr::FrameSetup);
620 bool ARM64FrameLowering::restoreCalleeSavedRegisters(
621 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
622 const std::vector<CalleeSavedInfo> &CSI,
623 const TargetRegisterInfo *TRI) const {
624 MachineFunction &MF = *MBB.getParent();
625 const TargetInstrInfo &TII = *MF.getTarget().getInstrInfo();
626 unsigned Count = CSI.size();
628 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
631 DL = MI->getDebugLoc();
633 for (unsigned i = 0; i < Count; i += 2) {
634 unsigned Reg1 = CSI[i].getReg();
635 unsigned Reg2 = CSI[i + 1].getReg();
636 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
637 // list to come in sorted by frame index so that we can issue the store
638 // pair instructions directly. Assert if we see anything otherwise.
639 assert(CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx() &&
640 "Out of order callee saved regs!");
641 // Issue sequence of non-sp increment and sp-pi restores for cs regs. Only
642 // the last load is sp-pi post-increment and de-allocates the stack:
644 // ldp fp, lr, [sp, #32] // addImm(+4)
645 // ldp x20, x19, [sp, #16] // addImm(+2)
646 // ldp x22, x21, [sp], #48 // addImm(+6)
647 // Note: see comment in spillCalleeSavedRegisters()
650 assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!");
651 assert((i & 1) == 0 && "Odd index for callee-saved reg spill!");
652 if (ARM64::GPR64RegClass.contains(Reg1)) {
653 assert(ARM64::GPR64RegClass.contains(Reg2) &&
654 "Expected GPR64 callee-saved register pair!");
656 LdrOpc = ARM64::LDPXpost;
658 LdrOpc = ARM64::LDPXi;
659 } else if (ARM64::FPR64RegClass.contains(Reg1)) {
660 assert(ARM64::FPR64RegClass.contains(Reg2) &&
661 "Expected FPR64 callee-saved register pair!");
663 LdrOpc = ARM64::LDPDpost;
665 LdrOpc = ARM64::LDPDi;
667 llvm_unreachable("Unexpected callee saved register!");
668 DEBUG(dbgs() << "CSR restore: (" << TRI->getName(Reg1) << ", "
669 << TRI->getName(Reg2) << ") -> fi#(" << CSI[i].getFrameIdx()
670 << ", " << CSI[i + 1].getFrameIdx() << ")\n");
672 // Compute offset: i = 0 => offset = Count - 2; i = 2 => offset = Count - 4;
674 const int Offset = (i == Count - 2) ? Count : Count - i - 2;
675 assert((Offset >= -64 && Offset <= 63) &&
676 "Offset out of bounds for LDP immediate");
677 BuildMI(MBB, MI, DL, TII.get(LdrOpc))
678 .addReg(Reg2, getDefRegState(true))
679 .addReg(Reg1, getDefRegState(true))
681 .addImm(Offset); // [sp], #offset * 8 or [sp, #offset * 8]
682 // where the factor * 8 is implicit
687 void ARM64FrameLowering::processFunctionBeforeCalleeSavedScan(
688 MachineFunction &MF, RegScavenger *RS) const {
689 const ARM64RegisterInfo *RegInfo =
690 static_cast<const ARM64RegisterInfo *>(MF.getTarget().getRegisterInfo());
691 ARM64FunctionInfo *AFI = MF.getInfo<ARM64FunctionInfo>();
692 MachineRegisterInfo *MRI = &MF.getRegInfo();
693 SmallVector<unsigned, 4> UnspilledCSGPRs;
694 SmallVector<unsigned, 4> UnspilledCSFPRs;
696 // The frame record needs to be created by saving the appropriate registers
698 MRI->setPhysRegUsed(ARM64::FP);
699 MRI->setPhysRegUsed(ARM64::LR);
702 // Spill the BasePtr if it's used. Do this first thing so that the
703 // getCalleeSavedRegs() below will get the right answer.
704 if (RegInfo->hasBasePointer(MF))
705 MRI->setPhysRegUsed(RegInfo->getBaseRegister());
707 // If any callee-saved registers are used, the frame cannot be eliminated.
708 unsigned NumGPRSpilled = 0;
709 unsigned NumFPRSpilled = 0;
710 bool ExtraCSSpill = false;
711 bool CanEliminateFrame = true;
712 DEBUG(dbgs() << "*** processFunctionBeforeCalleeSavedScan\nUsed CSRs:");
713 const uint16_t *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
715 // Check pairs of consecutive callee-saved registers.
716 for (unsigned i = 0; CSRegs[i]; i += 2) {
717 assert(CSRegs[i + 1] && "Odd number of callee-saved registers!");
719 const unsigned OddReg = CSRegs[i];
720 const unsigned EvenReg = CSRegs[i + 1];
721 assert((ARM64::GPR64RegClass.contains(OddReg) &&
722 ARM64::GPR64RegClass.contains(EvenReg)) ^
723 (ARM64::FPR64RegClass.contains(OddReg) &&
724 ARM64::FPR64RegClass.contains(EvenReg)) &&
725 "Register class mismatch!");
727 const bool OddRegUsed = MRI->isPhysRegUsed(OddReg);
728 const bool EvenRegUsed = MRI->isPhysRegUsed(EvenReg);
730 // Early exit if none of the registers in the register pair is actually
732 if (!OddRegUsed && !EvenRegUsed) {
733 if (ARM64::GPR64RegClass.contains(OddReg)) {
734 UnspilledCSGPRs.push_back(OddReg);
735 UnspilledCSGPRs.push_back(EvenReg);
737 UnspilledCSFPRs.push_back(OddReg);
738 UnspilledCSFPRs.push_back(EvenReg);
743 unsigned Reg = ARM64::NoRegister;
744 // If only one of the registers of the register pair is used, make sure to
745 // mark the other one as used as well.
746 if (OddRegUsed ^ EvenRegUsed) {
747 // Find out which register is the additional spill.
748 Reg = OddRegUsed ? EvenReg : OddReg;
749 MRI->setPhysRegUsed(Reg);
752 DEBUG(dbgs() << ' ' << PrintReg(OddReg, RegInfo));
753 DEBUG(dbgs() << ' ' << PrintReg(EvenReg, RegInfo));
755 assert(((OddReg == ARM64::LR && EvenReg == ARM64::FP) ||
756 (RegInfo->getEncodingValue(OddReg) + 1 ==
757 RegInfo->getEncodingValue(EvenReg))) &&
758 "Register pair of non-adjacent registers!");
759 if (ARM64::GPR64RegClass.contains(OddReg)) {
761 // If it's not a reserved register, we can use it in lieu of an
762 // emergency spill slot for the register scavenger.
763 // FIXME: It would be better to instead keep looking and choose another
764 // unspilled register that isn't reserved, if there is one.
765 if (Reg != ARM64::NoRegister && !RegInfo->isReservedReg(MF, Reg))
770 CanEliminateFrame = false;
773 // FIXME: Set BigStack if any stack slot references may be out of range.
774 // For now, just conservatively guestimate based on unscaled indexing
775 // range. We'll end up allocating an unnecessary spill slot a lot, but
776 // realistically that's not a big deal at this stage of the game.
777 // The CSR spill slots have not been allocated yet, so estimateStackSize
778 // won't include them.
779 MachineFrameInfo *MFI = MF.getFrameInfo();
780 unsigned CFSize = estimateStackSize(MF) + 8 * (NumGPRSpilled + NumFPRSpilled);
781 DEBUG(dbgs() << "Estimated stack frame size: " << CFSize << " bytes.\n");
782 bool BigStack = (CFSize >= 256);
783 if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
784 AFI->setHasStackFrame(true);
786 // Estimate if we might need to scavenge a register at some point in order
787 // to materialize a stack offset. If so, either spill one additional
788 // callee-saved register or reserve a special spill slot to facilitate
789 // register scavenging. If we already spilled an extra callee-saved register
790 // above to keep the number of spills even, we don't need to do anything else
792 if (BigStack && !ExtraCSSpill) {
794 // If we're adding a register to spill here, we have to add two of them
795 // to keep the number of regs to spill even.
796 assert(((UnspilledCSGPRs.size() & 1) == 0) && "Odd number of registers!");
798 while (!UnspilledCSGPRs.empty() && Count < 2) {
799 unsigned Reg = UnspilledCSGPRs.back();
800 UnspilledCSGPRs.pop_back();
801 DEBUG(dbgs() << "Spilling " << PrintReg(Reg, RegInfo)
802 << " to get a scratch register.\n");
803 MRI->setPhysRegUsed(Reg);
808 // If we didn't find an extra callee-saved register to spill, create
809 // an emergency spill slot.
811 const TargetRegisterClass *RC = &ARM64::GPR64RegClass;
812 int FI = MFI->CreateStackObject(RC->getSize(), RC->getAlignment(), false);
813 RS->addScavengingFrameIndex(FI);
814 DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
815 << " as the emergency spill slot.\n");