1 //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file contains the AArch64 implementation of the TargetInstrInfo class.
12 //===----------------------------------------------------------------------===//
14 #include "AArch64InstrInfo.h"
15 #include "AArch64Subtarget.h"
16 #include "MCTargetDesc/AArch64AddressingModes.h"
17 #include "AArch64MachineCombinerPattern.h"
18 #include "llvm/CodeGen/MachineFrameInfo.h"
19 #include "llvm/CodeGen/MachineInstrBuilder.h"
20 #include "llvm/CodeGen/MachineMemOperand.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/CodeGen/PseudoSourceValue.h"
23 #include "llvm/MC/MCInst.h"
24 #include "llvm/Support/ErrorHandling.h"
25 #include "llvm/Support/TargetRegistry.h"
29 #define GET_INSTRINFO_CTOR_DTOR
30 #include "AArch64GenInstrInfo.inc"
32 AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
33 : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP),
34 RI(this, &STI), Subtarget(STI) {}
36 /// GetInstSize - Return the number of bytes of code the specified
37 /// instruction may be. This returns the maximum number of bytes.
38 unsigned AArch64InstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const {
39 const MachineBasicBlock &MBB = *MI->getParent();
40 const MachineFunction *MF = MBB.getParent();
41 const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
43 if (MI->getOpcode() == AArch64::INLINEASM)
44 return getInlineAsmLength(MI->getOperand(0).getSymbolName(), *MAI);
46 const MCInstrDesc &Desc = MI->getDesc();
47 switch (Desc.getOpcode()) {
49 // Anything not explicitly designated otherwise is a nomal 4-byte insn.
51 case TargetOpcode::DBG_VALUE:
52 case TargetOpcode::EH_LABEL:
53 case TargetOpcode::IMPLICIT_DEF:
54 case TargetOpcode::KILL:
58 llvm_unreachable("GetInstSizeInBytes()- Unable to determin insn size");
61 static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
62 SmallVectorImpl<MachineOperand> &Cond) {
63 // Block ends with fall-through condbranch.
64 switch (LastInst->getOpcode()) {
66 llvm_unreachable("Unknown branch instruction?");
68 Target = LastInst->getOperand(1).getMBB();
69 Cond.push_back(LastInst->getOperand(0));
75 Target = LastInst->getOperand(1).getMBB();
76 Cond.push_back(MachineOperand::CreateImm(-1));
77 Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
78 Cond.push_back(LastInst->getOperand(0));
84 Target = LastInst->getOperand(2).getMBB();
85 Cond.push_back(MachineOperand::CreateImm(-1));
86 Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
87 Cond.push_back(LastInst->getOperand(0));
88 Cond.push_back(LastInst->getOperand(1));
93 bool AArch64InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
94 MachineBasicBlock *&TBB,
95 MachineBasicBlock *&FBB,
96 SmallVectorImpl<MachineOperand> &Cond,
97 bool AllowModify) const {
98 // If the block has no terminators, it just falls into the block after it.
99 MachineBasicBlock::iterator I = MBB.end();
100 if (I == MBB.begin())
103 while (I->isDebugValue()) {
104 if (I == MBB.begin())
108 if (!isUnpredicatedTerminator(I))
111 // Get the last instruction in the block.
112 MachineInstr *LastInst = I;
114 // If there is only one terminator instruction, process it.
115 unsigned LastOpc = LastInst->getOpcode();
116 if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) {
117 if (isUncondBranchOpcode(LastOpc)) {
118 TBB = LastInst->getOperand(0).getMBB();
121 if (isCondBranchOpcode(LastOpc)) {
122 // Block ends with fall-through condbranch.
123 parseCondBranch(LastInst, TBB, Cond);
126 return true; // Can't handle indirect branch.
129 // Get the instruction before it if it is a terminator.
130 MachineInstr *SecondLastInst = I;
131 unsigned SecondLastOpc = SecondLastInst->getOpcode();
133 // If AllowModify is true and the block ends with two or more unconditional
134 // branches, delete all but the first unconditional branch.
135 if (AllowModify && isUncondBranchOpcode(LastOpc)) {
136 while (isUncondBranchOpcode(SecondLastOpc)) {
137 LastInst->eraseFromParent();
138 LastInst = SecondLastInst;
139 LastOpc = LastInst->getOpcode();
140 if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) {
141 // Return now the only terminator is an unconditional branch.
142 TBB = LastInst->getOperand(0).getMBB();
146 SecondLastOpc = SecondLastInst->getOpcode();
151 // If there are three terminators, we don't know what sort of block this is.
152 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(--I))
155 // If the block ends with a B and a Bcc, handle it.
156 if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
157 parseCondBranch(SecondLastInst, TBB, Cond);
158 FBB = LastInst->getOperand(0).getMBB();
162 // If the block ends with two unconditional branches, handle it. The second
163 // one is not executed, so remove it.
164 if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
165 TBB = SecondLastInst->getOperand(0).getMBB();
168 I->eraseFromParent();
172 // ...likewise if it ends with an indirect branch followed by an unconditional
174 if (isIndirectBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
177 I->eraseFromParent();
181 // Otherwise, can't handle this.
185 bool AArch64InstrInfo::ReverseBranchCondition(
186 SmallVectorImpl<MachineOperand> &Cond) const {
187 if (Cond[0].getImm() != -1) {
189 AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
190 Cond[0].setImm(AArch64CC::getInvertedCondCode(CC));
192 // Folded compare-and-branch
193 switch (Cond[1].getImm()) {
195 llvm_unreachable("Unknown conditional branch!");
197 Cond[1].setImm(AArch64::CBNZW);
200 Cond[1].setImm(AArch64::CBZW);
203 Cond[1].setImm(AArch64::CBNZX);
206 Cond[1].setImm(AArch64::CBZX);
209 Cond[1].setImm(AArch64::TBNZW);
212 Cond[1].setImm(AArch64::TBZW);
215 Cond[1].setImm(AArch64::TBNZX);
218 Cond[1].setImm(AArch64::TBZX);
226 unsigned AArch64InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
227 MachineBasicBlock::iterator I = MBB.end();
228 if (I == MBB.begin())
231 while (I->isDebugValue()) {
232 if (I == MBB.begin())
236 if (!isUncondBranchOpcode(I->getOpcode()) &&
237 !isCondBranchOpcode(I->getOpcode()))
240 // Remove the branch.
241 I->eraseFromParent();
245 if (I == MBB.begin())
248 if (!isCondBranchOpcode(I->getOpcode()))
251 // Remove the branch.
252 I->eraseFromParent();
256 void AArch64InstrInfo::instantiateCondBranch(
257 MachineBasicBlock &MBB, DebugLoc DL, MachineBasicBlock *TBB,
258 const SmallVectorImpl<MachineOperand> &Cond) const {
259 if (Cond[0].getImm() != -1) {
261 BuildMI(&MBB, DL, get(AArch64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB);
263 // Folded compare-and-branch
264 const MachineInstrBuilder MIB =
265 BuildMI(&MBB, DL, get(Cond[1].getImm())).addReg(Cond[2].getReg());
267 MIB.addImm(Cond[3].getImm());
272 unsigned AArch64InstrInfo::InsertBranch(
273 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
274 const SmallVectorImpl<MachineOperand> &Cond, DebugLoc DL) const {
275 // Shouldn't be a fall through.
276 assert(TBB && "InsertBranch must not be told to insert a fallthrough");
279 if (Cond.empty()) // Unconditional branch?
280 BuildMI(&MBB, DL, get(AArch64::B)).addMBB(TBB);
282 instantiateCondBranch(MBB, DL, TBB, Cond);
286 // Two-way conditional branch.
287 instantiateCondBranch(MBB, DL, TBB, Cond);
288 BuildMI(&MBB, DL, get(AArch64::B)).addMBB(FBB);
292 // Find the original register that VReg is copied from.
293 static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
294 while (TargetRegisterInfo::isVirtualRegister(VReg)) {
295 const MachineInstr *DefMI = MRI.getVRegDef(VReg);
296 if (!DefMI->isFullCopy())
298 VReg = DefMI->getOperand(1).getReg();
303 // Determine if VReg is defined by an instruction that can be folded into a
304 // csel instruction. If so, return the folded opcode, and the replacement
306 static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
307 unsigned *NewVReg = nullptr) {
308 VReg = removeCopies(MRI, VReg);
309 if (!TargetRegisterInfo::isVirtualRegister(VReg))
312 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(MRI.getRegClass(VReg));
313 const MachineInstr *DefMI = MRI.getVRegDef(VReg);
315 unsigned SrcOpNum = 0;
316 switch (DefMI->getOpcode()) {
317 case AArch64::ADDSXri:
318 case AArch64::ADDSWri:
319 // if NZCV is used, do not fold.
320 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
322 // fall-through to ADDXri and ADDWri.
323 case AArch64::ADDXri:
324 case AArch64::ADDWri:
325 // add x, 1 -> csinc.
326 if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 ||
327 DefMI->getOperand(3).getImm() != 0)
330 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
333 case AArch64::ORNXrr:
334 case AArch64::ORNWrr: {
335 // not x -> csinv, represented as orn dst, xzr, src.
336 unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
337 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
340 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
344 case AArch64::SUBSXrr:
345 case AArch64::SUBSWrr:
346 // if NZCV is used, do not fold.
347 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
349 // fall-through to SUBXrr and SUBWrr.
350 case AArch64::SUBXrr:
351 case AArch64::SUBWrr: {
352 // neg x -> csneg, represented as sub dst, xzr, src.
353 unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
354 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
357 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
363 assert(Opc && SrcOpNum && "Missing parameters");
366 *NewVReg = DefMI->getOperand(SrcOpNum).getReg();
370 bool AArch64InstrInfo::canInsertSelect(
371 const MachineBasicBlock &MBB, const SmallVectorImpl<MachineOperand> &Cond,
372 unsigned TrueReg, unsigned FalseReg, int &CondCycles, int &TrueCycles,
373 int &FalseCycles) const {
374 // Check register classes.
375 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
376 const TargetRegisterClass *RC =
377 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
381 // Expanding cbz/tbz requires an extra cycle of latency on the condition.
382 unsigned ExtraCondLat = Cond.size() != 1;
384 // GPRs are handled by csel.
385 // FIXME: Fold in x+1, -x, and ~x when applicable.
386 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
387 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
388 // Single-cycle csel, csinc, csinv, and csneg.
389 CondCycles = 1 + ExtraCondLat;
390 TrueCycles = FalseCycles = 1;
391 if (canFoldIntoCSel(MRI, TrueReg))
393 else if (canFoldIntoCSel(MRI, FalseReg))
398 // Scalar floating point is handled by fcsel.
399 // FIXME: Form fabs, fmin, and fmax when applicable.
400 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
401 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
402 CondCycles = 5 + ExtraCondLat;
403 TrueCycles = FalseCycles = 2;
411 void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
412 MachineBasicBlock::iterator I, DebugLoc DL,
414 const SmallVectorImpl<MachineOperand> &Cond,
415 unsigned TrueReg, unsigned FalseReg) const {
416 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
418 // Parse the condition code, see parseCondBranch() above.
419 AArch64CC::CondCode CC;
420 switch (Cond.size()) {
422 llvm_unreachable("Unknown condition opcode in Cond");
424 CC = AArch64CC::CondCode(Cond[0].getImm());
426 case 3: { // cbz/cbnz
427 // We must insert a compare against 0.
429 switch (Cond[1].getImm()) {
431 llvm_unreachable("Unknown branch opcode in Cond");
449 unsigned SrcReg = Cond[2].getReg();
451 // cmp reg, #0 is actually subs xzr, reg, #0.
452 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
453 BuildMI(MBB, I, DL, get(AArch64::SUBSXri), AArch64::XZR)
458 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
459 BuildMI(MBB, I, DL, get(AArch64::SUBSWri), AArch64::WZR)
466 case 4: { // tbz/tbnz
467 // We must insert a tst instruction.
468 switch (Cond[1].getImm()) {
470 llvm_unreachable("Unknown branch opcode in Cond");
480 // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
481 if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
482 BuildMI(MBB, I, DL, get(AArch64::ANDSWri), AArch64::WZR)
483 .addReg(Cond[2].getReg())
485 AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 32));
487 BuildMI(MBB, I, DL, get(AArch64::ANDSXri), AArch64::XZR)
488 .addReg(Cond[2].getReg())
490 AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 64));
496 const TargetRegisterClass *RC = nullptr;
497 bool TryFold = false;
498 if (MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
499 RC = &AArch64::GPR64RegClass;
500 Opc = AArch64::CSELXr;
502 } else if (MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
503 RC = &AArch64::GPR32RegClass;
504 Opc = AArch64::CSELWr;
506 } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
507 RC = &AArch64::FPR64RegClass;
508 Opc = AArch64::FCSELDrrr;
509 } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
510 RC = &AArch64::FPR32RegClass;
511 Opc = AArch64::FCSELSrrr;
513 assert(RC && "Unsupported regclass");
515 // Try folding simple instructions into the csel.
517 unsigned NewVReg = 0;
518 unsigned FoldedOpc = canFoldIntoCSel(MRI, TrueReg, &NewVReg);
520 // The folded opcodes csinc, csinc and csneg apply the operation to
521 // FalseReg, so we need to invert the condition.
522 CC = AArch64CC::getInvertedCondCode(CC);
525 FoldedOpc = canFoldIntoCSel(MRI, FalseReg, &NewVReg);
527 // Fold the operation. Leave any dead instructions for DCE to clean up.
531 // The extends the live range of NewVReg.
532 MRI.clearKillFlags(NewVReg);
536 // Pull all virtual register into the appropriate class.
537 MRI.constrainRegClass(TrueReg, RC);
538 MRI.constrainRegClass(FalseReg, RC);
541 BuildMI(MBB, I, DL, get(Opc), DstReg).addReg(TrueReg).addReg(FalseReg).addImm(
545 // FIXME: this implementation should be micro-architecture dependent, so a
546 // micro-architecture target hook should be introduced here in future.
547 bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr *MI) const {
548 if (!Subtarget.isCortexA57() && !Subtarget.isCortexA53())
549 return MI->isAsCheapAsAMove();
551 switch (MI->getOpcode()) {
555 // add/sub on register without shift
556 case AArch64::ADDWri:
557 case AArch64::ADDXri:
558 case AArch64::SUBWri:
559 case AArch64::SUBXri:
560 return (MI->getOperand(3).getImm() == 0);
562 // logical ops on immediate
563 case AArch64::ANDWri:
564 case AArch64::ANDXri:
565 case AArch64::EORWri:
566 case AArch64::EORXri:
567 case AArch64::ORRWri:
568 case AArch64::ORRXri:
571 // logical ops on register without shift
572 case AArch64::ANDWrr:
573 case AArch64::ANDXrr:
574 case AArch64::BICWrr:
575 case AArch64::BICXrr:
576 case AArch64::EONWrr:
577 case AArch64::EONXrr:
578 case AArch64::EORWrr:
579 case AArch64::EORXrr:
580 case AArch64::ORNWrr:
581 case AArch64::ORNXrr:
582 case AArch64::ORRWrr:
583 case AArch64::ORRXrr:
587 llvm_unreachable("Unknown opcode to check as cheap as a move!");
590 bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
591 unsigned &SrcReg, unsigned &DstReg,
592 unsigned &SubIdx) const {
593 switch (MI.getOpcode()) {
596 case AArch64::SBFMXri: // aka sxtw
597 case AArch64::UBFMXri: // aka uxtw
598 // Check for the 32 -> 64 bit extension case, these instructions can do
600 if (MI.getOperand(2).getImm() != 0 || MI.getOperand(3).getImm() != 31)
602 // This is a signed or unsigned 32 -> 64 bit extension.
603 SrcReg = MI.getOperand(1).getReg();
604 DstReg = MI.getOperand(0).getReg();
605 SubIdx = AArch64::sub_32;
610 /// analyzeCompare - For a comparison instruction, return the source registers
611 /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
612 /// Return true if the comparison instruction can be analyzed.
613 bool AArch64InstrInfo::analyzeCompare(const MachineInstr *MI, unsigned &SrcReg,
614 unsigned &SrcReg2, int &CmpMask,
615 int &CmpValue) const {
616 switch (MI->getOpcode()) {
619 case AArch64::SUBSWrr:
620 case AArch64::SUBSWrs:
621 case AArch64::SUBSWrx:
622 case AArch64::SUBSXrr:
623 case AArch64::SUBSXrs:
624 case AArch64::SUBSXrx:
625 case AArch64::ADDSWrr:
626 case AArch64::ADDSWrs:
627 case AArch64::ADDSWrx:
628 case AArch64::ADDSXrr:
629 case AArch64::ADDSXrs:
630 case AArch64::ADDSXrx:
631 // Replace SUBSWrr with SUBWrr if NZCV is not used.
632 SrcReg = MI->getOperand(1).getReg();
633 SrcReg2 = MI->getOperand(2).getReg();
637 case AArch64::SUBSWri:
638 case AArch64::ADDSWri:
639 case AArch64::SUBSXri:
640 case AArch64::ADDSXri:
641 SrcReg = MI->getOperand(1).getReg();
644 // FIXME: In order to convert CmpValue to 0 or 1
645 CmpValue = (MI->getOperand(2).getImm() != 0);
647 case AArch64::ANDSWri:
648 case AArch64::ANDSXri:
649 // ANDS does not use the same encoding scheme as the others xxxS
651 SrcReg = MI->getOperand(1).getReg();
654 // FIXME:The return val type of decodeLogicalImmediate is uint64_t,
655 // while the type of CmpValue is int. When converting uint64_t to int,
656 // the high 32 bits of uint64_t will be lost.
657 // In fact it causes a bug in spec2006-483.xalancbmk
658 // CmpValue is only used to compare with zero in OptimizeCompareInstr
659 CmpValue = (AArch64_AM::decodeLogicalImmediate(
660 MI->getOperand(2).getImm(),
661 MI->getOpcode() == AArch64::ANDSWri ? 32 : 64) != 0);
668 static bool UpdateOperandRegClass(MachineInstr *Instr) {
669 MachineBasicBlock *MBB = Instr->getParent();
670 assert(MBB && "Can't get MachineBasicBlock here");
671 MachineFunction *MF = MBB->getParent();
672 assert(MF && "Can't get MachineFunction here");
673 const TargetMachine *TM = &MF->getTarget();
674 const TargetInstrInfo *TII = TM->getSubtargetImpl()->getInstrInfo();
675 const TargetRegisterInfo *TRI = TM->getSubtargetImpl()->getRegisterInfo();
676 MachineRegisterInfo *MRI = &MF->getRegInfo();
678 for (unsigned OpIdx = 0, EndIdx = Instr->getNumOperands(); OpIdx < EndIdx;
680 MachineOperand &MO = Instr->getOperand(OpIdx);
681 const TargetRegisterClass *OpRegCstraints =
682 Instr->getRegClassConstraint(OpIdx, TII, TRI);
684 // If there's no constraint, there's nothing to do.
687 // If the operand is a frame index, there's nothing to do here.
688 // A frame index operand will resolve correctly during PEI.
693 "Operand has register constraints without being a register!");
695 unsigned Reg = MO.getReg();
696 if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
697 if (!OpRegCstraints->contains(Reg))
699 } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) &&
700 !MRI->constrainRegClass(Reg, OpRegCstraints))
707 /// convertFlagSettingOpcode - return opcode that does not
708 /// set flags when possible. The caller is responsible to do
709 /// the actual substitution and legality checking.
710 static unsigned convertFlagSettingOpcode(MachineInstr *MI) {
712 switch (MI->getOpcode()) {
715 case AArch64::ADDSWrr: NewOpc = AArch64::ADDWrr; break;
716 case AArch64::ADDSWri: NewOpc = AArch64::ADDWri; break;
717 case AArch64::ADDSWrs: NewOpc = AArch64::ADDWrs; break;
718 case AArch64::ADDSWrx: NewOpc = AArch64::ADDWrx; break;
719 case AArch64::ADDSXrr: NewOpc = AArch64::ADDXrr; break;
720 case AArch64::ADDSXri: NewOpc = AArch64::ADDXri; break;
721 case AArch64::ADDSXrs: NewOpc = AArch64::ADDXrs; break;
722 case AArch64::ADDSXrx: NewOpc = AArch64::ADDXrx; break;
723 case AArch64::SUBSWrr: NewOpc = AArch64::SUBWrr; break;
724 case AArch64::SUBSWri: NewOpc = AArch64::SUBWri; break;
725 case AArch64::SUBSWrs: NewOpc = AArch64::SUBWrs; break;
726 case AArch64::SUBSWrx: NewOpc = AArch64::SUBWrx; break;
727 case AArch64::SUBSXrr: NewOpc = AArch64::SUBXrr; break;
728 case AArch64::SUBSXri: NewOpc = AArch64::SUBXri; break;
729 case AArch64::SUBSXrs: NewOpc = AArch64::SUBXrs; break;
730 case AArch64::SUBSXrx: NewOpc = AArch64::SUBXrx; break;
735 /// optimizeCompareInstr - Convert the instruction supplying the argument to the
736 /// comparison into one that sets the zero bit in the flags register.
737 bool AArch64InstrInfo::optimizeCompareInstr(
738 MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2, int CmpMask,
739 int CmpValue, const MachineRegisterInfo *MRI) const {
741 // Replace SUBSWrr with SUBWrr if NZCV is not used.
742 int Cmp_NZCV = CmpInstr->findRegisterDefOperandIdx(AArch64::NZCV, true);
743 if (Cmp_NZCV != -1) {
744 unsigned Opc = CmpInstr->getOpcode();
745 unsigned NewOpc = convertFlagSettingOpcode(CmpInstr);
748 const MCInstrDesc &MCID = get(NewOpc);
749 CmpInstr->setDesc(MCID);
750 CmpInstr->RemoveOperand(Cmp_NZCV);
751 bool succeeded = UpdateOperandRegClass(CmpInstr);
753 assert(succeeded && "Some operands reg class are incompatible!");
757 // Continue only if we have a "ri" where immediate is zero.
758 // FIXME:CmpValue has already been converted to 0 or 1 in analyzeCompare
760 assert((CmpValue == 0 || CmpValue == 1) && "CmpValue must be 0 or 1!");
761 if (CmpValue != 0 || SrcReg2 != 0)
764 // CmpInstr is a Compare instruction if destination register is not used.
765 if (!MRI->use_nodbg_empty(CmpInstr->getOperand(0).getReg()))
768 // Get the unique definition of SrcReg.
769 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
773 // We iterate backward, starting from the instruction before CmpInstr and
774 // stop when reaching the definition of the source register or done with the
775 // basic block, to check whether NZCV is used or modified in between.
776 MachineBasicBlock::iterator I = CmpInstr, E = MI,
777 B = CmpInstr->getParent()->begin();
779 // Early exit if CmpInstr is at the beginning of the BB.
783 // Check whether the definition of SrcReg is in the same basic block as
784 // Compare. If not, we can't optimize away the Compare.
785 if (MI->getParent() != CmpInstr->getParent())
788 // Check that NZCV isn't set between the comparison instruction and the one we
790 const TargetRegisterInfo *TRI = &getRegisterInfo();
791 for (--I; I != E; --I) {
792 const MachineInstr &Instr = *I;
794 if (Instr.modifiesRegister(AArch64::NZCV, TRI) ||
795 Instr.readsRegister(AArch64::NZCV, TRI))
796 // This instruction modifies or uses NZCV after the one we want to
797 // change. We can't do this transformation.
800 // The 'and' is below the comparison instruction.
804 unsigned NewOpc = MI->getOpcode();
805 switch (MI->getOpcode()) {
808 case AArch64::ADDSWrr:
809 case AArch64::ADDSWri:
810 case AArch64::ADDSXrr:
811 case AArch64::ADDSXri:
812 case AArch64::SUBSWrr:
813 case AArch64::SUBSWri:
814 case AArch64::SUBSXrr:
815 case AArch64::SUBSXri:
817 case AArch64::ADDWrr: NewOpc = AArch64::ADDSWrr; break;
818 case AArch64::ADDWri: NewOpc = AArch64::ADDSWri; break;
819 case AArch64::ADDXrr: NewOpc = AArch64::ADDSXrr; break;
820 case AArch64::ADDXri: NewOpc = AArch64::ADDSXri; break;
821 case AArch64::ADCWr: NewOpc = AArch64::ADCSWr; break;
822 case AArch64::ADCXr: NewOpc = AArch64::ADCSXr; break;
823 case AArch64::SUBWrr: NewOpc = AArch64::SUBSWrr; break;
824 case AArch64::SUBWri: NewOpc = AArch64::SUBSWri; break;
825 case AArch64::SUBXrr: NewOpc = AArch64::SUBSXrr; break;
826 case AArch64::SUBXri: NewOpc = AArch64::SUBSXri; break;
827 case AArch64::SBCWr: NewOpc = AArch64::SBCSWr; break;
828 case AArch64::SBCXr: NewOpc = AArch64::SBCSXr; break;
829 case AArch64::ANDWri: NewOpc = AArch64::ANDSWri; break;
830 case AArch64::ANDXri: NewOpc = AArch64::ANDSXri; break;
833 // Scan forward for the use of NZCV.
834 // When checking against MI: if it's a conditional code requires
835 // checking of V bit, then this is not safe to do.
836 // It is safe to remove CmpInstr if NZCV is redefined or killed.
837 // If we are done with the basic block, we need to check whether NZCV is
840 for (MachineBasicBlock::iterator I = CmpInstr,
841 E = CmpInstr->getParent()->end();
842 !IsSafe && ++I != E;) {
843 const MachineInstr &Instr = *I;
844 for (unsigned IO = 0, EO = Instr.getNumOperands(); !IsSafe && IO != EO;
846 const MachineOperand &MO = Instr.getOperand(IO);
847 if (MO.isRegMask() && MO.clobbersPhysReg(AArch64::NZCV)) {
851 if (!MO.isReg() || MO.getReg() != AArch64::NZCV)
858 // Decode the condition code.
859 unsigned Opc = Instr.getOpcode();
860 AArch64CC::CondCode CC;
865 CC = (AArch64CC::CondCode)Instr.getOperand(IO - 2).getImm();
867 case AArch64::CSINVWr:
868 case AArch64::CSINVXr:
869 case AArch64::CSINCWr:
870 case AArch64::CSINCXr:
871 case AArch64::CSELWr:
872 case AArch64::CSELXr:
873 case AArch64::CSNEGWr:
874 case AArch64::CSNEGXr:
875 case AArch64::FCSELSrrr:
876 case AArch64::FCSELDrrr:
877 CC = (AArch64CC::CondCode)Instr.getOperand(IO - 1).getImm();
881 // It is not safe to remove Compare instruction if Overflow(V) is used.
884 // NZCV can be used multiple times, we should continue.
897 // If NZCV is not killed nor re-defined, we should check whether it is
898 // live-out. If it is live-out, do not optimize.
900 MachineBasicBlock *ParentBlock = CmpInstr->getParent();
901 for (auto *MBB : ParentBlock->successors())
902 if (MBB->isLiveIn(AArch64::NZCV))
906 // Update the instruction to set NZCV.
907 MI->setDesc(get(NewOpc));
908 CmpInstr->eraseFromParent();
909 bool succeeded = UpdateOperandRegClass(MI);
911 assert(succeeded && "Some operands reg class are incompatible!");
912 MI->addRegisterDefined(AArch64::NZCV, TRI);
917 AArch64InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
918 if (MI->getOpcode() != TargetOpcode::LOAD_STACK_GUARD)
921 MachineBasicBlock &MBB = *MI->getParent();
922 DebugLoc DL = MI->getDebugLoc();
923 unsigned Reg = MI->getOperand(0).getReg();
924 const GlobalValue *GV =
925 cast<GlobalValue>((*MI->memoperands_begin())->getValue());
926 const TargetMachine &TM = MBB.getParent()->getTarget();
927 unsigned char OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
928 const unsigned char MO_NC = AArch64II::MO_NC;
930 if ((OpFlags & AArch64II::MO_GOT) != 0) {
931 BuildMI(MBB, MI, DL, get(AArch64::LOADgot), Reg)
932 .addGlobalAddress(GV, 0, AArch64II::MO_GOT);
933 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
934 .addReg(Reg, RegState::Kill).addImm(0)
935 .addMemOperand(*MI->memoperands_begin());
936 } else if (TM.getCodeModel() == CodeModel::Large) {
937 BuildMI(MBB, MI, DL, get(AArch64::MOVZXi), Reg)
938 .addGlobalAddress(GV, 0, AArch64II::MO_G3).addImm(48);
939 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
940 .addReg(Reg, RegState::Kill)
941 .addGlobalAddress(GV, 0, AArch64II::MO_G2 | MO_NC).addImm(32);
942 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
943 .addReg(Reg, RegState::Kill)
944 .addGlobalAddress(GV, 0, AArch64II::MO_G1 | MO_NC).addImm(16);
945 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
946 .addReg(Reg, RegState::Kill)
947 .addGlobalAddress(GV, 0, AArch64II::MO_G0 | MO_NC).addImm(0);
948 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
949 .addReg(Reg, RegState::Kill).addImm(0)
950 .addMemOperand(*MI->memoperands_begin());
952 BuildMI(MBB, MI, DL, get(AArch64::ADRP), Reg)
953 .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE);
954 unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
955 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
956 .addReg(Reg, RegState::Kill)
957 .addGlobalAddress(GV, 0, LoFlags)
958 .addMemOperand(*MI->memoperands_begin());
966 /// Return true if this is this instruction has a non-zero immediate
967 bool AArch64InstrInfo::hasShiftedReg(const MachineInstr *MI) const {
968 switch (MI->getOpcode()) {
971 case AArch64::ADDSWrs:
972 case AArch64::ADDSXrs:
973 case AArch64::ADDWrs:
974 case AArch64::ADDXrs:
975 case AArch64::ANDSWrs:
976 case AArch64::ANDSXrs:
977 case AArch64::ANDWrs:
978 case AArch64::ANDXrs:
979 case AArch64::BICSWrs:
980 case AArch64::BICSXrs:
981 case AArch64::BICWrs:
982 case AArch64::BICXrs:
983 case AArch64::CRC32Brr:
984 case AArch64::CRC32CBrr:
985 case AArch64::CRC32CHrr:
986 case AArch64::CRC32CWrr:
987 case AArch64::CRC32CXrr:
988 case AArch64::CRC32Hrr:
989 case AArch64::CRC32Wrr:
990 case AArch64::CRC32Xrr:
991 case AArch64::EONWrs:
992 case AArch64::EONXrs:
993 case AArch64::EORWrs:
994 case AArch64::EORXrs:
995 case AArch64::ORNWrs:
996 case AArch64::ORNXrs:
997 case AArch64::ORRWrs:
998 case AArch64::ORRXrs:
999 case AArch64::SUBSWrs:
1000 case AArch64::SUBSXrs:
1001 case AArch64::SUBWrs:
1002 case AArch64::SUBXrs:
1003 if (MI->getOperand(3).isImm()) {
1004 unsigned val = MI->getOperand(3).getImm();
1012 /// Return true if this is this instruction has a non-zero immediate
1013 bool AArch64InstrInfo::hasExtendedReg(const MachineInstr *MI) const {
1014 switch (MI->getOpcode()) {
1017 case AArch64::ADDSWrx:
1018 case AArch64::ADDSXrx:
1019 case AArch64::ADDSXrx64:
1020 case AArch64::ADDWrx:
1021 case AArch64::ADDXrx:
1022 case AArch64::ADDXrx64:
1023 case AArch64::SUBSWrx:
1024 case AArch64::SUBSXrx:
1025 case AArch64::SUBSXrx64:
1026 case AArch64::SUBWrx:
1027 case AArch64::SUBXrx:
1028 case AArch64::SUBXrx64:
1029 if (MI->getOperand(3).isImm()) {
1030 unsigned val = MI->getOperand(3).getImm();
1039 // Return true if this instruction simply sets its single destination register
1040 // to zero. This is equivalent to a register rename of the zero-register.
1041 bool AArch64InstrInfo::isGPRZero(const MachineInstr *MI) const {
1042 switch (MI->getOpcode()) {
1045 case AArch64::MOVZWi:
1046 case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
1047 if (MI->getOperand(1).isImm() && MI->getOperand(1).getImm() == 0) {
1048 assert(MI->getDesc().getNumOperands() == 3 &&
1049 MI->getOperand(2).getImm() == 0 && "invalid MOVZi operands");
1053 case AArch64::ANDWri: // and Rd, Rzr, #imm
1054 return MI->getOperand(1).getReg() == AArch64::WZR;
1055 case AArch64::ANDXri:
1056 return MI->getOperand(1).getReg() == AArch64::XZR;
1057 case TargetOpcode::COPY:
1058 return MI->getOperand(1).getReg() == AArch64::WZR;
1063 // Return true if this instruction simply renames a general register without
1065 bool AArch64InstrInfo::isGPRCopy(const MachineInstr *MI) const {
1066 switch (MI->getOpcode()) {
1069 case TargetOpcode::COPY: {
1070 // GPR32 copies will by lowered to ORRXrs
1071 unsigned DstReg = MI->getOperand(0).getReg();
1072 return (AArch64::GPR32RegClass.contains(DstReg) ||
1073 AArch64::GPR64RegClass.contains(DstReg));
1075 case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
1076 if (MI->getOperand(1).getReg() == AArch64::XZR) {
1077 assert(MI->getDesc().getNumOperands() == 4 &&
1078 MI->getOperand(3).getImm() == 0 && "invalid ORRrs operands");
1082 case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
1083 if (MI->getOperand(2).getImm() == 0) {
1084 assert(MI->getDesc().getNumOperands() == 4 &&
1085 MI->getOperand(3).getImm() == 0 && "invalid ADDXri operands");
1093 // Return true if this instruction simply renames a general register without
1095 bool AArch64InstrInfo::isFPRCopy(const MachineInstr *MI) const {
1096 switch (MI->getOpcode()) {
1099 case TargetOpcode::COPY: {
1100 // FPR64 copies will by lowered to ORR.16b
1101 unsigned DstReg = MI->getOperand(0).getReg();
1102 return (AArch64::FPR64RegClass.contains(DstReg) ||
1103 AArch64::FPR128RegClass.contains(DstReg));
1105 case AArch64::ORRv16i8:
1106 if (MI->getOperand(1).getReg() == MI->getOperand(2).getReg()) {
1107 assert(MI->getDesc().getNumOperands() == 3 && MI->getOperand(0).isReg() &&
1108 "invalid ORRv16i8 operands");
1116 unsigned AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr *MI,
1117 int &FrameIndex) const {
1118 switch (MI->getOpcode()) {
1121 case AArch64::LDRWui:
1122 case AArch64::LDRXui:
1123 case AArch64::LDRBui:
1124 case AArch64::LDRHui:
1125 case AArch64::LDRSui:
1126 case AArch64::LDRDui:
1127 case AArch64::LDRQui:
1128 if (MI->getOperand(0).getSubReg() == 0 && MI->getOperand(1).isFI() &&
1129 MI->getOperand(2).isImm() && MI->getOperand(2).getImm() == 0) {
1130 FrameIndex = MI->getOperand(1).getIndex();
1131 return MI->getOperand(0).getReg();
1139 unsigned AArch64InstrInfo::isStoreToStackSlot(const MachineInstr *MI,
1140 int &FrameIndex) const {
1141 switch (MI->getOpcode()) {
1144 case AArch64::STRWui:
1145 case AArch64::STRXui:
1146 case AArch64::STRBui:
1147 case AArch64::STRHui:
1148 case AArch64::STRSui:
1149 case AArch64::STRDui:
1150 case AArch64::STRQui:
1151 if (MI->getOperand(0).getSubReg() == 0 && MI->getOperand(1).isFI() &&
1152 MI->getOperand(2).isImm() && MI->getOperand(2).getImm() == 0) {
1153 FrameIndex = MI->getOperand(1).getIndex();
1154 return MI->getOperand(0).getReg();
1161 /// Return true if this is load/store scales or extends its register offset.
1162 /// This refers to scaling a dynamic index as opposed to scaled immediates.
1163 /// MI should be a memory op that allows scaled addressing.
1164 bool AArch64InstrInfo::isScaledAddr(const MachineInstr *MI) const {
1165 switch (MI->getOpcode()) {
1168 case AArch64::LDRBBroW:
1169 case AArch64::LDRBroW:
1170 case AArch64::LDRDroW:
1171 case AArch64::LDRHHroW:
1172 case AArch64::LDRHroW:
1173 case AArch64::LDRQroW:
1174 case AArch64::LDRSBWroW:
1175 case AArch64::LDRSBXroW:
1176 case AArch64::LDRSHWroW:
1177 case AArch64::LDRSHXroW:
1178 case AArch64::LDRSWroW:
1179 case AArch64::LDRSroW:
1180 case AArch64::LDRWroW:
1181 case AArch64::LDRXroW:
1182 case AArch64::STRBBroW:
1183 case AArch64::STRBroW:
1184 case AArch64::STRDroW:
1185 case AArch64::STRHHroW:
1186 case AArch64::STRHroW:
1187 case AArch64::STRQroW:
1188 case AArch64::STRSroW:
1189 case AArch64::STRWroW:
1190 case AArch64::STRXroW:
1191 case AArch64::LDRBBroX:
1192 case AArch64::LDRBroX:
1193 case AArch64::LDRDroX:
1194 case AArch64::LDRHHroX:
1195 case AArch64::LDRHroX:
1196 case AArch64::LDRQroX:
1197 case AArch64::LDRSBWroX:
1198 case AArch64::LDRSBXroX:
1199 case AArch64::LDRSHWroX:
1200 case AArch64::LDRSHXroX:
1201 case AArch64::LDRSWroX:
1202 case AArch64::LDRSroX:
1203 case AArch64::LDRWroX:
1204 case AArch64::LDRXroX:
1205 case AArch64::STRBBroX:
1206 case AArch64::STRBroX:
1207 case AArch64::STRDroX:
1208 case AArch64::STRHHroX:
1209 case AArch64::STRHroX:
1210 case AArch64::STRQroX:
1211 case AArch64::STRSroX:
1212 case AArch64::STRWroX:
1213 case AArch64::STRXroX:
1215 unsigned Val = MI->getOperand(3).getImm();
1216 AArch64_AM::ShiftExtendType ExtType = AArch64_AM::getMemExtendType(Val);
1217 return (ExtType != AArch64_AM::UXTX) || AArch64_AM::getMemDoShift(Val);
1222 /// Check all MachineMemOperands for a hint to suppress pairing.
1223 bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr *MI) const {
1224 assert(MOSuppressPair < (1 << MachineMemOperand::MOTargetNumBits) &&
1225 "Too many target MO flags");
1226 for (auto *MM : MI->memoperands()) {
1227 if (MM->getFlags() &
1228 (MOSuppressPair << MachineMemOperand::MOTargetStartBit)) {
1235 /// Set a flag on the first MachineMemOperand to suppress pairing.
1236 void AArch64InstrInfo::suppressLdStPair(MachineInstr *MI) const {
1237 if (MI->memoperands_empty())
1240 assert(MOSuppressPair < (1 << MachineMemOperand::MOTargetNumBits) &&
1241 "Too many target MO flags");
1242 (*MI->memoperands_begin())
1243 ->setFlags(MOSuppressPair << MachineMemOperand::MOTargetStartBit);
1247 AArch64InstrInfo::getLdStBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
1249 const TargetRegisterInfo *TRI) const {
1250 switch (LdSt->getOpcode()) {
1253 case AArch64::STRSui:
1254 case AArch64::STRDui:
1255 case AArch64::STRQui:
1256 case AArch64::STRXui:
1257 case AArch64::STRWui:
1258 case AArch64::LDRSui:
1259 case AArch64::LDRDui:
1260 case AArch64::LDRQui:
1261 case AArch64::LDRXui:
1262 case AArch64::LDRWui:
1263 if (!LdSt->getOperand(1).isReg() || !LdSt->getOperand(2).isImm())
1265 BaseReg = LdSt->getOperand(1).getReg();
1266 MachineFunction &MF = *LdSt->getParent()->getParent();
1267 unsigned Width = getRegClass(LdSt->getDesc(), 0, TRI, MF)->getSize();
1268 Offset = LdSt->getOperand(2).getImm() * Width;
1273 /// Detect opportunities for ldp/stp formation.
1275 /// Only called for LdSt for which getLdStBaseRegImmOfs returns true.
1276 bool AArch64InstrInfo::shouldClusterLoads(MachineInstr *FirstLdSt,
1277 MachineInstr *SecondLdSt,
1278 unsigned NumLoads) const {
1279 // Only cluster up to a single pair.
1282 if (FirstLdSt->getOpcode() != SecondLdSt->getOpcode())
1284 // getLdStBaseRegImmOfs guarantees that oper 2 isImm.
1285 unsigned Ofs1 = FirstLdSt->getOperand(2).getImm();
1286 // Allow 6 bits of positive range.
1289 // The caller should already have ordered First/SecondLdSt by offset.
1290 unsigned Ofs2 = SecondLdSt->getOperand(2).getImm();
1291 return Ofs1 + 1 == Ofs2;
1294 bool AArch64InstrInfo::shouldScheduleAdjacent(MachineInstr *First,
1295 MachineInstr *Second) const {
1296 // Cyclone can fuse CMN, CMP followed by Bcc.
1298 // FIXME: B0 can also fuse:
1299 // AND, BIC, ORN, ORR, or EOR (optional S) followed by Bcc or CBZ or CBNZ.
1300 if (Second->getOpcode() != AArch64::Bcc)
1302 switch (First->getOpcode()) {
1305 case AArch64::SUBSWri:
1306 case AArch64::ADDSWri:
1307 case AArch64::ANDSWri:
1308 case AArch64::SUBSXri:
1309 case AArch64::ADDSXri:
1310 case AArch64::ANDSXri:
1315 MachineInstr *AArch64InstrInfo::emitFrameIndexDebugValue(MachineFunction &MF,
1318 const MDNode *MDPtr,
1319 DebugLoc DL) const {
1320 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AArch64::DBG_VALUE))
1321 .addFrameIndex(FrameIx)
1324 .addMetadata(MDPtr);
1328 static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
1329 unsigned Reg, unsigned SubIdx,
1331 const TargetRegisterInfo *TRI) {
1333 return MIB.addReg(Reg, State);
1335 if (TargetRegisterInfo::isPhysicalRegister(Reg))
1336 return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
1337 return MIB.addReg(Reg, State, SubIdx);
1340 static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg,
1342 // We really want the positive remainder mod 32 here, that happens to be
1343 // easily obtainable with a mask.
1344 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
1347 void AArch64InstrInfo::copyPhysRegTuple(
1348 MachineBasicBlock &MBB, MachineBasicBlock::iterator I, DebugLoc DL,
1349 unsigned DestReg, unsigned SrcReg, bool KillSrc, unsigned Opcode,
1350 llvm::ArrayRef<unsigned> Indices) const {
1351 assert(Subtarget.hasNEON() &&
1352 "Unexpected register copy without NEON");
1353 const TargetRegisterInfo *TRI = &getRegisterInfo();
1354 uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
1355 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
1356 unsigned NumRegs = Indices.size();
1358 int SubReg = 0, End = NumRegs, Incr = 1;
1359 if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs)) {
1360 SubReg = NumRegs - 1;
1365 for (; SubReg != End; SubReg += Incr) {
1366 const MachineInstrBuilder &MIB = BuildMI(MBB, I, DL, get(Opcode));
1367 AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
1368 AddSubReg(MIB, SrcReg, Indices[SubReg], 0, TRI);
1369 AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
1373 void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
1374 MachineBasicBlock::iterator I, DebugLoc DL,
1375 unsigned DestReg, unsigned SrcReg,
1376 bool KillSrc) const {
1377 if (AArch64::GPR32spRegClass.contains(DestReg) &&
1378 (AArch64::GPR32spRegClass.contains(SrcReg) || SrcReg == AArch64::WZR)) {
1379 const TargetRegisterInfo *TRI = &getRegisterInfo();
1381 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
1382 // If either operand is WSP, expand to ADD #0.
1383 if (Subtarget.hasZeroCycleRegMove()) {
1384 // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
1385 unsigned DestRegX = TRI->getMatchingSuperReg(DestReg, AArch64::sub_32,
1386 &AArch64::GPR64spRegClass);
1387 unsigned SrcRegX = TRI->getMatchingSuperReg(SrcReg, AArch64::sub_32,
1388 &AArch64::GPR64spRegClass);
1389 // This instruction is reading and writing X registers. This may upset
1390 // the register scavenger and machine verifier, so we need to indicate
1391 // that we are reading an undefined value from SrcRegX, but a proper
1392 // value from SrcReg.
1393 BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestRegX)
1394 .addReg(SrcRegX, RegState::Undef)
1396 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
1397 .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
1399 BuildMI(MBB, I, DL, get(AArch64::ADDWri), DestReg)
1400 .addReg(SrcReg, getKillRegState(KillSrc))
1402 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1404 } else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroing()) {
1405 BuildMI(MBB, I, DL, get(AArch64::MOVZWi), DestReg).addImm(0).addImm(
1406 AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1408 if (Subtarget.hasZeroCycleRegMove()) {
1409 // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
1410 unsigned DestRegX = TRI->getMatchingSuperReg(DestReg, AArch64::sub_32,
1411 &AArch64::GPR64spRegClass);
1412 unsigned SrcRegX = TRI->getMatchingSuperReg(SrcReg, AArch64::sub_32,
1413 &AArch64::GPR64spRegClass);
1414 // This instruction is reading and writing X registers. This may upset
1415 // the register scavenger and machine verifier, so we need to indicate
1416 // that we are reading an undefined value from SrcRegX, but a proper
1417 // value from SrcReg.
1418 BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestRegX)
1419 .addReg(AArch64::XZR)
1420 .addReg(SrcRegX, RegState::Undef)
1421 .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
1423 // Otherwise, expand to ORR WZR.
1424 BuildMI(MBB, I, DL, get(AArch64::ORRWrr), DestReg)
1425 .addReg(AArch64::WZR)
1426 .addReg(SrcReg, getKillRegState(KillSrc));
1432 if (AArch64::GPR64spRegClass.contains(DestReg) &&
1433 (AArch64::GPR64spRegClass.contains(SrcReg) || SrcReg == AArch64::XZR)) {
1434 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
1435 // If either operand is SP, expand to ADD #0.
1436 BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestReg)
1437 .addReg(SrcReg, getKillRegState(KillSrc))
1439 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1440 } else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroing()) {
1441 BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestReg).addImm(0).addImm(
1442 AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1444 // Otherwise, expand to ORR XZR.
1445 BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestReg)
1446 .addReg(AArch64::XZR)
1447 .addReg(SrcReg, getKillRegState(KillSrc));
1452 // Copy a DDDD register quad by copying the individual sub-registers.
1453 if (AArch64::DDDDRegClass.contains(DestReg) &&
1454 AArch64::DDDDRegClass.contains(SrcReg)) {
1455 static const unsigned Indices[] = { AArch64::dsub0, AArch64::dsub1,
1456 AArch64::dsub2, AArch64::dsub3 };
1457 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
1462 // Copy a DDD register triple by copying the individual sub-registers.
1463 if (AArch64::DDDRegClass.contains(DestReg) &&
1464 AArch64::DDDRegClass.contains(SrcReg)) {
1465 static const unsigned Indices[] = { AArch64::dsub0, AArch64::dsub1,
1467 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
1472 // Copy a DD register pair by copying the individual sub-registers.
1473 if (AArch64::DDRegClass.contains(DestReg) &&
1474 AArch64::DDRegClass.contains(SrcReg)) {
1475 static const unsigned Indices[] = { AArch64::dsub0, AArch64::dsub1 };
1476 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
1481 // Copy a QQQQ register quad by copying the individual sub-registers.
1482 if (AArch64::QQQQRegClass.contains(DestReg) &&
1483 AArch64::QQQQRegClass.contains(SrcReg)) {
1484 static const unsigned Indices[] = { AArch64::qsub0, AArch64::qsub1,
1485 AArch64::qsub2, AArch64::qsub3 };
1486 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
1491 // Copy a QQQ register triple by copying the individual sub-registers.
1492 if (AArch64::QQQRegClass.contains(DestReg) &&
1493 AArch64::QQQRegClass.contains(SrcReg)) {
1494 static const unsigned Indices[] = { AArch64::qsub0, AArch64::qsub1,
1496 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
1501 // Copy a QQ register pair by copying the individual sub-registers.
1502 if (AArch64::QQRegClass.contains(DestReg) &&
1503 AArch64::QQRegClass.contains(SrcReg)) {
1504 static const unsigned Indices[] = { AArch64::qsub0, AArch64::qsub1 };
1505 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
1510 if (AArch64::FPR128RegClass.contains(DestReg) &&
1511 AArch64::FPR128RegClass.contains(SrcReg)) {
1512 if(Subtarget.hasNEON()) {
1513 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1515 .addReg(SrcReg, getKillRegState(KillSrc));
1517 BuildMI(MBB, I, DL, get(AArch64::STRQpre))
1518 .addReg(AArch64::SP, RegState::Define)
1519 .addReg(SrcReg, getKillRegState(KillSrc))
1520 .addReg(AArch64::SP)
1522 BuildMI(MBB, I, DL, get(AArch64::LDRQpre))
1523 .addReg(AArch64::SP, RegState::Define)
1524 .addReg(DestReg, RegState::Define)
1525 .addReg(AArch64::SP)
1531 if (AArch64::FPR64RegClass.contains(DestReg) &&
1532 AArch64::FPR64RegClass.contains(SrcReg)) {
1533 if(Subtarget.hasNEON()) {
1534 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::dsub,
1535 &AArch64::FPR128RegClass);
1536 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::dsub,
1537 &AArch64::FPR128RegClass);
1538 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1540 .addReg(SrcReg, getKillRegState(KillSrc));
1542 BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestReg)
1543 .addReg(SrcReg, getKillRegState(KillSrc));
1548 if (AArch64::FPR32RegClass.contains(DestReg) &&
1549 AArch64::FPR32RegClass.contains(SrcReg)) {
1550 if(Subtarget.hasNEON()) {
1551 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
1552 &AArch64::FPR128RegClass);
1553 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
1554 &AArch64::FPR128RegClass);
1555 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1557 .addReg(SrcReg, getKillRegState(KillSrc));
1559 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
1560 .addReg(SrcReg, getKillRegState(KillSrc));
1565 if (AArch64::FPR16RegClass.contains(DestReg) &&
1566 AArch64::FPR16RegClass.contains(SrcReg)) {
1567 if(Subtarget.hasNEON()) {
1568 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
1569 &AArch64::FPR128RegClass);
1570 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
1571 &AArch64::FPR128RegClass);
1572 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1574 .addReg(SrcReg, getKillRegState(KillSrc));
1576 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
1577 &AArch64::FPR32RegClass);
1578 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
1579 &AArch64::FPR32RegClass);
1580 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
1581 .addReg(SrcReg, getKillRegState(KillSrc));
1586 if (AArch64::FPR8RegClass.contains(DestReg) &&
1587 AArch64::FPR8RegClass.contains(SrcReg)) {
1588 if(Subtarget.hasNEON()) {
1589 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
1590 &AArch64::FPR128RegClass);
1591 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
1592 &AArch64::FPR128RegClass);
1593 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1595 .addReg(SrcReg, getKillRegState(KillSrc));
1597 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
1598 &AArch64::FPR32RegClass);
1599 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
1600 &AArch64::FPR32RegClass);
1601 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
1602 .addReg(SrcReg, getKillRegState(KillSrc));
1607 // Copies between GPR64 and FPR64.
1608 if (AArch64::FPR64RegClass.contains(DestReg) &&
1609 AArch64::GPR64RegClass.contains(SrcReg)) {
1610 BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg)
1611 .addReg(SrcReg, getKillRegState(KillSrc));
1614 if (AArch64::GPR64RegClass.contains(DestReg) &&
1615 AArch64::FPR64RegClass.contains(SrcReg)) {
1616 BuildMI(MBB, I, DL, get(AArch64::FMOVDXr), DestReg)
1617 .addReg(SrcReg, getKillRegState(KillSrc));
1620 // Copies between GPR32 and FPR32.
1621 if (AArch64::FPR32RegClass.contains(DestReg) &&
1622 AArch64::GPR32RegClass.contains(SrcReg)) {
1623 BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg)
1624 .addReg(SrcReg, getKillRegState(KillSrc));
1627 if (AArch64::GPR32RegClass.contains(DestReg) &&
1628 AArch64::FPR32RegClass.contains(SrcReg)) {
1629 BuildMI(MBB, I, DL, get(AArch64::FMOVSWr), DestReg)
1630 .addReg(SrcReg, getKillRegState(KillSrc));
1634 if (DestReg == AArch64::NZCV) {
1635 assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
1636 BuildMI(MBB, I, DL, get(AArch64::MSR))
1637 .addImm(AArch64SysReg::NZCV)
1638 .addReg(SrcReg, getKillRegState(KillSrc))
1639 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define);
1643 if (SrcReg == AArch64::NZCV) {
1644 assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
1645 BuildMI(MBB, I, DL, get(AArch64::MRS))
1647 .addImm(AArch64SysReg::NZCV)
1648 .addReg(AArch64::NZCV, RegState::Implicit | getKillRegState(KillSrc));
1652 llvm_unreachable("unimplemented reg-to-reg copy");
1655 void AArch64InstrInfo::storeRegToStackSlot(
1656 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned SrcReg,
1657 bool isKill, int FI, const TargetRegisterClass *RC,
1658 const TargetRegisterInfo *TRI) const {
1660 if (MBBI != MBB.end())
1661 DL = MBBI->getDebugLoc();
1662 MachineFunction &MF = *MBB.getParent();
1663 MachineFrameInfo &MFI = *MF.getFrameInfo();
1664 unsigned Align = MFI.getObjectAlignment(FI);
1666 MachinePointerInfo PtrInfo(PseudoSourceValue::getFixedStack(FI));
1667 MachineMemOperand *MMO = MF.getMachineMemOperand(
1668 PtrInfo, MachineMemOperand::MOStore, MFI.getObjectSize(FI), Align);
1671 switch (RC->getSize()) {
1673 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
1674 Opc = AArch64::STRBui;
1677 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
1678 Opc = AArch64::STRHui;
1681 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1682 Opc = AArch64::STRWui;
1683 if (TargetRegisterInfo::isVirtualRegister(SrcReg))
1684 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR32RegClass);
1686 assert(SrcReg != AArch64::WSP);
1687 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
1688 Opc = AArch64::STRSui;
1691 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
1692 Opc = AArch64::STRXui;
1693 if (TargetRegisterInfo::isVirtualRegister(SrcReg))
1694 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
1696 assert(SrcReg != AArch64::SP);
1697 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
1698 Opc = AArch64::STRDui;
1701 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
1702 Opc = AArch64::STRQui;
1703 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
1704 assert(Subtarget.hasNEON() &&
1705 "Unexpected register store without NEON");
1706 Opc = AArch64::ST1Twov1d, Offset = false;
1710 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
1711 assert(Subtarget.hasNEON() &&
1712 "Unexpected register store without NEON");
1713 Opc = AArch64::ST1Threev1d, Offset = false;
1717 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
1718 assert(Subtarget.hasNEON() &&
1719 "Unexpected register store without NEON");
1720 Opc = AArch64::ST1Fourv1d, Offset = false;
1721 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
1722 assert(Subtarget.hasNEON() &&
1723 "Unexpected register store without NEON");
1724 Opc = AArch64::ST1Twov2d, Offset = false;
1728 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
1729 assert(Subtarget.hasNEON() &&
1730 "Unexpected register store without NEON");
1731 Opc = AArch64::ST1Threev2d, Offset = false;
1735 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
1736 assert(Subtarget.hasNEON() &&
1737 "Unexpected register store without NEON");
1738 Opc = AArch64::ST1Fourv2d, Offset = false;
1742 assert(Opc && "Unknown register class");
1744 const MachineInstrBuilder &MI = BuildMI(MBB, MBBI, DL, get(Opc))
1745 .addReg(SrcReg, getKillRegState(isKill))
1750 MI.addMemOperand(MMO);
1753 void AArch64InstrInfo::loadRegFromStackSlot(
1754 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned DestReg,
1755 int FI, const TargetRegisterClass *RC,
1756 const TargetRegisterInfo *TRI) const {
1758 if (MBBI != MBB.end())
1759 DL = MBBI->getDebugLoc();
1760 MachineFunction &MF = *MBB.getParent();
1761 MachineFrameInfo &MFI = *MF.getFrameInfo();
1762 unsigned Align = MFI.getObjectAlignment(FI);
1763 MachinePointerInfo PtrInfo(PseudoSourceValue::getFixedStack(FI));
1764 MachineMemOperand *MMO = MF.getMachineMemOperand(
1765 PtrInfo, MachineMemOperand::MOLoad, MFI.getObjectSize(FI), Align);
1769 switch (RC->getSize()) {
1771 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
1772 Opc = AArch64::LDRBui;
1775 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
1776 Opc = AArch64::LDRHui;
1779 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1780 Opc = AArch64::LDRWui;
1781 if (TargetRegisterInfo::isVirtualRegister(DestReg))
1782 MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR32RegClass);
1784 assert(DestReg != AArch64::WSP);
1785 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
1786 Opc = AArch64::LDRSui;
1789 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
1790 Opc = AArch64::LDRXui;
1791 if (TargetRegisterInfo::isVirtualRegister(DestReg))
1792 MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR64RegClass);
1794 assert(DestReg != AArch64::SP);
1795 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
1796 Opc = AArch64::LDRDui;
1799 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
1800 Opc = AArch64::LDRQui;
1801 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
1802 assert(Subtarget.hasNEON() &&
1803 "Unexpected register load without NEON");
1804 Opc = AArch64::LD1Twov1d, Offset = false;
1808 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
1809 assert(Subtarget.hasNEON() &&
1810 "Unexpected register load without NEON");
1811 Opc = AArch64::LD1Threev1d, Offset = false;
1815 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
1816 assert(Subtarget.hasNEON() &&
1817 "Unexpected register load without NEON");
1818 Opc = AArch64::LD1Fourv1d, Offset = false;
1819 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
1820 assert(Subtarget.hasNEON() &&
1821 "Unexpected register load without NEON");
1822 Opc = AArch64::LD1Twov2d, Offset = false;
1826 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
1827 assert(Subtarget.hasNEON() &&
1828 "Unexpected register load without NEON");
1829 Opc = AArch64::LD1Threev2d, Offset = false;
1833 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
1834 assert(Subtarget.hasNEON() &&
1835 "Unexpected register load without NEON");
1836 Opc = AArch64::LD1Fourv2d, Offset = false;
1840 assert(Opc && "Unknown register class");
1842 const MachineInstrBuilder &MI = BuildMI(MBB, MBBI, DL, get(Opc))
1843 .addReg(DestReg, getDefRegState(true))
1847 MI.addMemOperand(MMO);
1850 void llvm::emitFrameOffset(MachineBasicBlock &MBB,
1851 MachineBasicBlock::iterator MBBI, DebugLoc DL,
1852 unsigned DestReg, unsigned SrcReg, int Offset,
1853 const TargetInstrInfo *TII,
1854 MachineInstr::MIFlag Flag, bool SetNZCV) {
1855 if (DestReg == SrcReg && Offset == 0)
1858 bool isSub = Offset < 0;
1862 // FIXME: If the offset won't fit in 24-bits, compute the offset into a
1863 // scratch register. If DestReg is a virtual register, use it as the
1864 // scratch register; otherwise, create a new virtual register (to be
1865 // replaced by the scavenger at the end of PEI). That case can be optimized
1866 // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
1867 // register can be loaded with offset%8 and the add/sub can use an extending
1868 // instruction with LSL#3.
1869 // Currently the function handles any offsets but generates a poor sequence
1871 // assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
1875 Opc = isSub ? AArch64::SUBSXri : AArch64::ADDSXri;
1877 Opc = isSub ? AArch64::SUBXri : AArch64::ADDXri;
1878 const unsigned MaxEncoding = 0xfff;
1879 const unsigned ShiftSize = 12;
1880 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
1881 while (((unsigned)Offset) >= (1 << ShiftSize)) {
1883 if (((unsigned)Offset) > MaxEncodableValue) {
1884 ThisVal = MaxEncodableValue;
1886 ThisVal = Offset & MaxEncodableValue;
1888 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
1889 "Encoding cannot handle value that big");
1890 BuildMI(MBB, MBBI, DL, TII->get(Opc), DestReg)
1892 .addImm(ThisVal >> ShiftSize)
1893 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftSize))
1901 BuildMI(MBB, MBBI, DL, TII->get(Opc), DestReg)
1904 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
1909 AArch64InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
1910 const SmallVectorImpl<unsigned> &Ops,
1911 int FrameIndex) const {
1912 // This is a bit of a hack. Consider this instruction:
1914 // %vreg0<def> = COPY %SP; GPR64all:%vreg0
1916 // We explicitly chose GPR64all for the virtual register so such a copy might
1917 // be eliminated by RegisterCoalescer. However, that may not be possible, and
1918 // %vreg0 may even spill. We can't spill %SP, and since it is in the GPR64all
1919 // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
1921 // To prevent that, we are going to constrain the %vreg0 register class here.
1923 // <rdar://problem/11522048>
1926 unsigned DstReg = MI->getOperand(0).getReg();
1927 unsigned SrcReg = MI->getOperand(1).getReg();
1928 if (SrcReg == AArch64::SP &&
1929 TargetRegisterInfo::isVirtualRegister(DstReg)) {
1930 MF.getRegInfo().constrainRegClass(DstReg, &AArch64::GPR64RegClass);
1933 if (DstReg == AArch64::SP &&
1934 TargetRegisterInfo::isVirtualRegister(SrcReg)) {
1935 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
1944 int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI, int &Offset,
1945 bool *OutUseUnscaledOp,
1946 unsigned *OutUnscaledOp,
1947 int *EmittableOffset) {
1949 bool IsSigned = false;
1950 // The ImmIdx should be changed case by case if it is not 2.
1951 unsigned ImmIdx = 2;
1952 unsigned UnscaledOp = 0;
1953 // Set output values in case of early exit.
1954 if (EmittableOffset)
1955 *EmittableOffset = 0;
1956 if (OutUseUnscaledOp)
1957 *OutUseUnscaledOp = false;
1960 switch (MI.getOpcode()) {
1962 llvm_unreachable("unhandled opcode in rewriteAArch64FrameIndex");
1963 // Vector spills/fills can't take an immediate offset.
1964 case AArch64::LD1Twov2d:
1965 case AArch64::LD1Threev2d:
1966 case AArch64::LD1Fourv2d:
1967 case AArch64::LD1Twov1d:
1968 case AArch64::LD1Threev1d:
1969 case AArch64::LD1Fourv1d:
1970 case AArch64::ST1Twov2d:
1971 case AArch64::ST1Threev2d:
1972 case AArch64::ST1Fourv2d:
1973 case AArch64::ST1Twov1d:
1974 case AArch64::ST1Threev1d:
1975 case AArch64::ST1Fourv1d:
1976 return AArch64FrameOffsetCannotUpdate;
1977 case AArch64::PRFMui:
1979 UnscaledOp = AArch64::PRFUMi;
1981 case AArch64::LDRXui:
1983 UnscaledOp = AArch64::LDURXi;
1985 case AArch64::LDRWui:
1987 UnscaledOp = AArch64::LDURWi;
1989 case AArch64::LDRBui:
1991 UnscaledOp = AArch64::LDURBi;
1993 case AArch64::LDRHui:
1995 UnscaledOp = AArch64::LDURHi;
1997 case AArch64::LDRSui:
1999 UnscaledOp = AArch64::LDURSi;
2001 case AArch64::LDRDui:
2003 UnscaledOp = AArch64::LDURDi;
2005 case AArch64::LDRQui:
2007 UnscaledOp = AArch64::LDURQi;
2009 case AArch64::LDRBBui:
2011 UnscaledOp = AArch64::LDURBBi;
2013 case AArch64::LDRHHui:
2015 UnscaledOp = AArch64::LDURHHi;
2017 case AArch64::LDRSBXui:
2019 UnscaledOp = AArch64::LDURSBXi;
2021 case AArch64::LDRSBWui:
2023 UnscaledOp = AArch64::LDURSBWi;
2025 case AArch64::LDRSHXui:
2027 UnscaledOp = AArch64::LDURSHXi;
2029 case AArch64::LDRSHWui:
2031 UnscaledOp = AArch64::LDURSHWi;
2033 case AArch64::LDRSWui:
2035 UnscaledOp = AArch64::LDURSWi;
2038 case AArch64::STRXui:
2040 UnscaledOp = AArch64::STURXi;
2042 case AArch64::STRWui:
2044 UnscaledOp = AArch64::STURWi;
2046 case AArch64::STRBui:
2048 UnscaledOp = AArch64::STURBi;
2050 case AArch64::STRHui:
2052 UnscaledOp = AArch64::STURHi;
2054 case AArch64::STRSui:
2056 UnscaledOp = AArch64::STURSi;
2058 case AArch64::STRDui:
2060 UnscaledOp = AArch64::STURDi;
2062 case AArch64::STRQui:
2064 UnscaledOp = AArch64::STURQi;
2066 case AArch64::STRBBui:
2068 UnscaledOp = AArch64::STURBBi;
2070 case AArch64::STRHHui:
2072 UnscaledOp = AArch64::STURHHi;
2075 case AArch64::LDPXi:
2076 case AArch64::LDPDi:
2077 case AArch64::STPXi:
2078 case AArch64::STPDi:
2082 case AArch64::LDPQi:
2083 case AArch64::STPQi:
2087 case AArch64::LDPWi:
2088 case AArch64::LDPSi:
2089 case AArch64::STPWi:
2090 case AArch64::STPSi:
2095 case AArch64::LDURXi:
2096 case AArch64::LDURWi:
2097 case AArch64::LDURBi:
2098 case AArch64::LDURHi:
2099 case AArch64::LDURSi:
2100 case AArch64::LDURDi:
2101 case AArch64::LDURQi:
2102 case AArch64::LDURHHi:
2103 case AArch64::LDURBBi:
2104 case AArch64::LDURSBXi:
2105 case AArch64::LDURSBWi:
2106 case AArch64::LDURSHXi:
2107 case AArch64::LDURSHWi:
2108 case AArch64::LDURSWi:
2109 case AArch64::STURXi:
2110 case AArch64::STURWi:
2111 case AArch64::STURBi:
2112 case AArch64::STURHi:
2113 case AArch64::STURSi:
2114 case AArch64::STURDi:
2115 case AArch64::STURQi:
2116 case AArch64::STURBBi:
2117 case AArch64::STURHHi:
2122 Offset += MI.getOperand(ImmIdx).getImm() * Scale;
2124 bool useUnscaledOp = false;
2125 // If the offset doesn't match the scale, we rewrite the instruction to
2126 // use the unscaled instruction instead. Likewise, if we have a negative
2127 // offset (and have an unscaled op to use).
2128 if ((Offset & (Scale - 1)) != 0 || (Offset < 0 && UnscaledOp != 0))
2129 useUnscaledOp = true;
2131 // Use an unscaled addressing mode if the instruction has a negative offset
2132 // (or if the instruction is already using an unscaled addressing mode).
2135 // ldp/stp instructions.
2138 } else if (UnscaledOp == 0 || useUnscaledOp) {
2148 // Attempt to fold address computation.
2149 int MaxOff = (1 << (MaskBits - IsSigned)) - 1;
2150 int MinOff = (IsSigned ? (-MaxOff - 1) : 0);
2151 if (Offset >= MinOff && Offset <= MaxOff) {
2152 if (EmittableOffset)
2153 *EmittableOffset = Offset;
2156 int NewOff = Offset < 0 ? MinOff : MaxOff;
2157 if (EmittableOffset)
2158 *EmittableOffset = NewOff;
2159 Offset = (Offset - NewOff) * Scale;
2161 if (OutUseUnscaledOp)
2162 *OutUseUnscaledOp = useUnscaledOp;
2164 *OutUnscaledOp = UnscaledOp;
2165 return AArch64FrameOffsetCanUpdate |
2166 (Offset == 0 ? AArch64FrameOffsetIsLegal : 0);
2169 bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
2170 unsigned FrameReg, int &Offset,
2171 const AArch64InstrInfo *TII) {
2172 unsigned Opcode = MI.getOpcode();
2173 unsigned ImmIdx = FrameRegIdx + 1;
2175 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
2176 Offset += MI.getOperand(ImmIdx).getImm();
2177 emitFrameOffset(*MI.getParent(), MI, MI.getDebugLoc(),
2178 MI.getOperand(0).getReg(), FrameReg, Offset, TII,
2179 MachineInstr::NoFlags, (Opcode == AArch64::ADDSXri));
2180 MI.eraseFromParent();
2186 unsigned UnscaledOp;
2188 int Status = isAArch64FrameOffsetLegal(MI, Offset, &UseUnscaledOp,
2189 &UnscaledOp, &NewOffset);
2190 if (Status & AArch64FrameOffsetCanUpdate) {
2191 if (Status & AArch64FrameOffsetIsLegal)
2192 // Replace the FrameIndex with FrameReg.
2193 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2195 MI.setDesc(TII->get(UnscaledOp));
2197 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
2204 void AArch64InstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
2205 NopInst.setOpcode(AArch64::HINT);
2206 NopInst.addOperand(MCOperand::CreateImm(0));
2208 /// useMachineCombiner - return true when a target supports MachineCombiner
2209 bool AArch64InstrInfo::useMachineCombiner(void) const {
2210 // AArch64 supports the combiner
2214 // True when Opc sets flag
2215 static bool isCombineInstrSettingFlag(unsigned Opc) {
2217 case AArch64::ADDSWrr:
2218 case AArch64::ADDSWri:
2219 case AArch64::ADDSXrr:
2220 case AArch64::ADDSXri:
2221 case AArch64::SUBSWrr:
2222 case AArch64::SUBSXrr:
2223 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
2224 case AArch64::SUBSWri:
2225 case AArch64::SUBSXri:
2233 // 32b Opcodes that can be combined with a MUL
2234 static bool isCombineInstrCandidate32(unsigned Opc) {
2236 case AArch64::ADDWrr:
2237 case AArch64::ADDWri:
2238 case AArch64::SUBWrr:
2239 case AArch64::ADDSWrr:
2240 case AArch64::ADDSWri:
2241 case AArch64::SUBSWrr:
2242 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
2243 case AArch64::SUBWri:
2244 case AArch64::SUBSWri:
2252 // 64b Opcodes that can be combined with a MUL
2253 static bool isCombineInstrCandidate64(unsigned Opc) {
2255 case AArch64::ADDXrr:
2256 case AArch64::ADDXri:
2257 case AArch64::SUBXrr:
2258 case AArch64::ADDSXrr:
2259 case AArch64::ADDSXri:
2260 case AArch64::SUBSXrr:
2261 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
2262 case AArch64::SUBXri:
2263 case AArch64::SUBSXri:
2271 // Opcodes that can be combined with a MUL
2272 static bool isCombineInstrCandidate(unsigned Opc) {
2273 return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
2276 static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
2277 unsigned MulOpc, unsigned ZeroReg) {
2278 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2279 MachineInstr *MI = nullptr;
2280 // We need a virtual register definition.
2281 if (MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg()))
2282 MI = MRI.getUniqueVRegDef(MO.getReg());
2283 // And it needs to be in the trace (otherwise, it won't have a depth).
2284 if (!MI || MI->getParent() != &MBB || (unsigned)MI->getOpcode() != MulOpc)
2287 assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
2288 MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
2289 MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
2291 // The third input reg must be zero.
2292 if (MI->getOperand(3).getReg() != ZeroReg)
2295 // Must only used by the user we combine with.
2296 if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2302 /// hasPattern - return true when there is potentially a faster code sequence
2303 /// for an instruction chain ending in \p Root. All potential patterns are
2305 /// in the \p Pattern vector. Pattern should be sorted in priority order since
2306 /// the pattern evaluator stops checking as soon as it finds a faster sequence.
2308 bool AArch64InstrInfo::hasPattern(
2310 SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Pattern) const {
2311 unsigned Opc = Root.getOpcode();
2312 MachineBasicBlock &MBB = *Root.getParent();
2315 if (!isCombineInstrCandidate(Opc))
2317 if (isCombineInstrSettingFlag(Opc)) {
2318 int Cmp_NZCV = Root.findRegisterDefOperandIdx(AArch64::NZCV, true);
2319 // When NZCV is live bail out.
2322 unsigned NewOpc = convertFlagSettingOpcode(&Root);
2323 // When opcode can't change bail out.
2324 // CHECKME: do we miss any cases for opcode conversion?
2333 case AArch64::ADDWrr:
2334 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
2335 "ADDWrr does not have register operands");
2336 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2338 Pattern.push_back(MachineCombinerPattern::MC_MULADDW_OP1);
2341 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDWrrr,
2343 Pattern.push_back(MachineCombinerPattern::MC_MULADDW_OP2);
2347 case AArch64::ADDXrr:
2348 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2350 Pattern.push_back(MachineCombinerPattern::MC_MULADDX_OP1);
2353 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDXrrr,
2355 Pattern.push_back(MachineCombinerPattern::MC_MULADDX_OP2);
2359 case AArch64::SUBWrr:
2360 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2362 Pattern.push_back(MachineCombinerPattern::MC_MULSUBW_OP1);
2365 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDWrrr,
2367 Pattern.push_back(MachineCombinerPattern::MC_MULSUBW_OP2);
2371 case AArch64::SUBXrr:
2372 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2374 Pattern.push_back(MachineCombinerPattern::MC_MULSUBX_OP1);
2377 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDXrrr,
2379 Pattern.push_back(MachineCombinerPattern::MC_MULSUBX_OP2);
2383 case AArch64::ADDWri:
2384 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2386 Pattern.push_back(MachineCombinerPattern::MC_MULADDWI_OP1);
2390 case AArch64::ADDXri:
2391 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2393 Pattern.push_back(MachineCombinerPattern::MC_MULADDXI_OP1);
2397 case AArch64::SUBWri:
2398 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2400 Pattern.push_back(MachineCombinerPattern::MC_MULSUBWI_OP1);
2404 case AArch64::SUBXri:
2405 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2407 Pattern.push_back(MachineCombinerPattern::MC_MULSUBXI_OP1);
2415 /// genMadd - Generate madd instruction and combine mul and add.
2419 /// ==> MADD R,A,B,C
2420 /// \param Root is the ADD instruction
2421 /// \param [out] InsInstrs is a vector of machine instructions and will
2422 /// contain the generated madd instruction
2423 /// \param IdxMulOpd is index of operand in Root that is the result of
2424 /// the MUL. In the example above IdxMulOpd is 1.
2425 /// \param MaddOpc the opcode fo the madd instruction
2426 static MachineInstr *genMadd(MachineFunction &MF, MachineRegisterInfo &MRI,
2427 const TargetInstrInfo *TII, MachineInstr &Root,
2428 SmallVectorImpl<MachineInstr *> &InsInstrs,
2429 unsigned IdxMulOpd, unsigned MaddOpc) {
2430 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
2432 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
2433 MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
2434 MachineOperand R = Root.getOperand(0);
2435 MachineOperand A = MUL->getOperand(1);
2436 MachineOperand B = MUL->getOperand(2);
2437 MachineOperand C = Root.getOperand(IdxOtherOpd);
2438 MachineInstrBuilder MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc))
2444 InsInstrs.push_back(MIB);
2448 /// genMaddR - Generate madd instruction and combine mul and add using
2449 /// an extra virtual register
2450 /// Example - an ADD intermediate needs to be stored in a register:
2453 /// ==> ORR V, ZR, Imm
2454 /// ==> MADD R,A,B,V
2455 /// \param Root is the ADD instruction
2456 /// \param [out] InsInstrs is a vector of machine instructions and will
2457 /// contain the generated madd instruction
2458 /// \param IdxMulOpd is index of operand in Root that is the result of
2459 /// the MUL. In the example above IdxMulOpd is 1.
2460 /// \param MaddOpc the opcode fo the madd instruction
2461 /// \param VR is a virtual register that holds the value of an ADD operand
2462 /// (V in the example above).
2463 static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
2464 const TargetInstrInfo *TII, MachineInstr &Root,
2465 SmallVectorImpl<MachineInstr *> &InsInstrs,
2466 unsigned IdxMulOpd, unsigned MaddOpc,
2468 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
2470 MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
2471 MachineOperand R = Root.getOperand(0);
2472 MachineOperand A = MUL->getOperand(1);
2473 MachineOperand B = MUL->getOperand(2);
2474 MachineInstrBuilder MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc))
2480 InsInstrs.push_back(MIB);
2483 /// genAlternativeCodeSequence - when hasPattern() finds a pattern
2484 /// this function generates the instructions that could replace the
2485 /// original code sequence
2486 void AArch64InstrInfo::genAlternativeCodeSequence(
2487 MachineInstr &Root, MachineCombinerPattern::MC_PATTERN Pattern,
2488 SmallVectorImpl<MachineInstr *> &InsInstrs,
2489 SmallVectorImpl<MachineInstr *> &DelInstrs,
2490 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
2491 MachineBasicBlock &MBB = *Root.getParent();
2492 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2493 MachineFunction &MF = *MBB.getParent();
2494 const TargetInstrInfo *TII = MF.getTarget().getSubtargetImpl()->getInstrInfo();
2502 case MachineCombinerPattern::MC_MULADDW_OP1:
2503 case MachineCombinerPattern::MC_MULADDX_OP1:
2507 // --- Create(MADD);
2508 Opc = Pattern == MachineCombinerPattern::MC_MULADDW_OP1 ? AArch64::MADDWrrr
2509 : AArch64::MADDXrrr;
2510 MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 1, Opc);
2512 case MachineCombinerPattern::MC_MULADDW_OP2:
2513 case MachineCombinerPattern::MC_MULADDX_OP2:
2517 // --- Create(MADD);
2518 Opc = Pattern == MachineCombinerPattern::MC_MULADDW_OP2 ? AArch64::MADDWrrr
2519 : AArch64::MADDXrrr;
2520 MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 2, Opc);
2522 case MachineCombinerPattern::MC_MULADDWI_OP1:
2523 case MachineCombinerPattern::MC_MULADDXI_OP1:
2526 // ==> ORR V, ZR, Imm
2528 // --- Create(MADD);
2530 const TargetRegisterClass *RC =
2531 MRI.getRegClass(Root.getOperand(1).getReg());
2532 unsigned NewVR = MRI.createVirtualRegister(RC);
2533 unsigned BitSize, OrrOpc, ZeroReg;
2534 if (Pattern == MachineCombinerPattern::MC_MULADDWI_OP1) {
2536 OrrOpc = AArch64::ORRWri;
2537 ZeroReg = AArch64::WZR;
2538 Opc = AArch64::MADDWrrr;
2540 OrrOpc = AArch64::ORRXri;
2542 ZeroReg = AArch64::XZR;
2543 Opc = AArch64::MADDXrrr;
2545 uint64_t Imm = Root.getOperand(2).getImm();
2547 if (Root.getOperand(3).isImm()) {
2548 unsigned val = Root.getOperand(3).getImm();
2551 uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize);
2554 if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
2555 MachineInstrBuilder MIB1 =
2556 BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc))
2557 .addOperand(MachineOperand::CreateReg(NewVR, true))
2560 InsInstrs.push_back(MIB1);
2561 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2562 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR);
2566 case MachineCombinerPattern::MC_MULSUBW_OP1:
2567 case MachineCombinerPattern::MC_MULSUBX_OP1: {
2571 // ==> MADD R,A,B,V // = -C + A*B
2572 // --- Create(MADD);
2573 const TargetRegisterClass *RC =
2574 MRI.getRegClass(Root.getOperand(1).getReg());
2575 unsigned NewVR = MRI.createVirtualRegister(RC);
2576 unsigned SubOpc, ZeroReg;
2577 if (Pattern == MachineCombinerPattern::MC_MULSUBW_OP1) {
2578 SubOpc = AArch64::SUBWrr;
2579 ZeroReg = AArch64::WZR;
2580 Opc = AArch64::MADDWrrr;
2582 SubOpc = AArch64::SUBXrr;
2583 ZeroReg = AArch64::XZR;
2584 Opc = AArch64::MADDXrrr;
2587 MachineInstrBuilder MIB1 =
2588 BuildMI(MF, Root.getDebugLoc(), TII->get(SubOpc))
2589 .addOperand(MachineOperand::CreateReg(NewVR, true))
2591 .addOperand(Root.getOperand(2));
2592 InsInstrs.push_back(MIB1);
2593 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2594 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR);
2596 case MachineCombinerPattern::MC_MULSUBW_OP2:
2597 case MachineCombinerPattern::MC_MULSUBX_OP2:
2600 // ==> MSUB R,A,B,C (computes C - A*B)
2601 // --- Create(MSUB);
2602 Opc = Pattern == MachineCombinerPattern::MC_MULSUBW_OP2 ? AArch64::MSUBWrrr
2603 : AArch64::MSUBXrrr;
2604 MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 2, Opc);
2606 case MachineCombinerPattern::MC_MULSUBWI_OP1:
2607 case MachineCombinerPattern::MC_MULSUBXI_OP1: {
2610 // ==> ORR V, ZR, -Imm
2611 // ==> MADD R,A,B,V // = -Imm + A*B
2612 // --- Create(MADD);
2613 const TargetRegisterClass *RC =
2614 MRI.getRegClass(Root.getOperand(1).getReg());
2615 unsigned NewVR = MRI.createVirtualRegister(RC);
2616 unsigned BitSize, OrrOpc, ZeroReg;
2617 if (Pattern == MachineCombinerPattern::MC_MULSUBWI_OP1) {
2619 OrrOpc = AArch64::ORRWri;
2620 ZeroReg = AArch64::WZR;
2621 Opc = AArch64::MADDWrrr;
2623 OrrOpc = AArch64::ORRXri;
2625 ZeroReg = AArch64::XZR;
2626 Opc = AArch64::MADDXrrr;
2628 int Imm = Root.getOperand(2).getImm();
2629 if (Root.getOperand(3).isImm()) {
2630 unsigned val = Root.getOperand(3).getImm();
2633 uint64_t UImm = -Imm << (64 - BitSize) >> (64 - BitSize);
2635 if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
2636 MachineInstrBuilder MIB1 =
2637 BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc))
2638 .addOperand(MachineOperand::CreateReg(NewVR, true))
2641 InsInstrs.push_back(MIB1);
2642 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2643 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR);
2647 // Record MUL and ADD/SUB for deletion
2648 DelInstrs.push_back(MUL);
2649 DelInstrs.push_back(&Root);