1 //===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ----------=//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of
11 // multiple and add / sub instructions) when special VMLx hazards are detected.
13 //===----------------------------------------------------------------------===//
15 #define DEBUG_TYPE "mlx-expansion"
17 #include "ARMBaseInstrInfo.h"
18 #include "llvm/CodeGen/MachineInstr.h"
19 #include "llvm/CodeGen/MachineInstrBuilder.h"
20 #include "llvm/CodeGen/MachineFunctionPass.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/Target/TargetRegisterInfo.h"
23 #include "llvm/ADT/Statistic.h"
24 #include "llvm/Support/CommandLine.h"
25 #include "llvm/Support/Debug.h"
26 #include "llvm/Support/raw_ostream.h"
30 ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden);
31 static cl::opt<unsigned>
32 ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden);
34 STATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded");
37 struct MLxExpansion : public MachineFunctionPass {
39 MLxExpansion() : MachineFunctionPass(ID) {}
41 virtual bool runOnMachineFunction(MachineFunction &Fn);
43 virtual const char *getPassName() const {
44 return "ARM MLA / MLS expansion pass";
48 const ARMBaseInstrInfo *TII;
49 const TargetRegisterInfo *TRI;
50 MachineRegisterInfo *MRI;
53 MachineInstr* LastMIs[4];
56 void pushStack(MachineInstr *MI);
57 MachineInstr *getAccDefMI(MachineInstr *MI) const;
58 unsigned getDefReg(MachineInstr *MI) const;
59 bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
60 bool FindMLxHazard(MachineInstr *MI) const;
61 void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
62 unsigned MulOpc, unsigned AddSubOpc,
63 bool NegAcc, bool HasLane);
64 bool ExpandFPMLxInstructions(MachineBasicBlock &MBB);
66 char MLxExpansion::ID = 0;
69 void MLxExpansion::clearStack() {
70 std::fill(LastMIs, LastMIs + 4, (MachineInstr*)0);
74 void MLxExpansion::pushStack(MachineInstr *MI) {
80 MachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const {
81 // Look past COPY and INSERT_SUBREG instructions to find the
82 // real definition MI. This is important for _sfp instructions.
83 unsigned Reg = MI->getOperand(1).getReg();
84 if (TargetRegisterInfo::isPhysicalRegister(Reg))
87 MachineBasicBlock *MBB = MI->getParent();
88 MachineInstr *DefMI = MRI->getVRegDef(Reg);
90 if (DefMI->getParent() != MBB)
92 if (DefMI->isCopyLike()) {
93 Reg = DefMI->getOperand(1).getReg();
94 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
95 DefMI = MRI->getVRegDef(Reg);
98 } else if (DefMI->isInsertSubreg()) {
99 Reg = DefMI->getOperand(2).getReg();
100 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
101 DefMI = MRI->getVRegDef(Reg);
110 unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
111 unsigned Reg = MI->getOperand(0).getReg();
112 if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
113 !MRI->hasOneNonDBGUse(Reg))
116 MachineBasicBlock *MBB = MI->getParent();
117 MachineInstr *UseMI = &*MRI->use_nodbg_begin(Reg);
118 if (UseMI->getParent() != MBB)
121 while (UseMI->isCopy() || UseMI->isInsertSubreg()) {
122 Reg = UseMI->getOperand(0).getReg();
123 if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
124 !MRI->hasOneNonDBGUse(Reg))
126 UseMI = &*MRI->use_nodbg_begin(Reg);
127 if (UseMI->getParent() != MBB)
134 bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
135 const TargetInstrDesc &TID = MI->getDesc();
136 // FIXME: Detect integer instructions properly.
137 unsigned Domain = TID.TSFlags & ARMII::DomainMask;
138 if (Domain == ARMII::DomainVFP) {
139 unsigned Opcode = TID.getOpcode();
140 if (Opcode == ARM::VSTRS || Opcode == ARM::VSTRD ||
141 Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
143 } else if (Domain == ARMII::DomainNEON) {
144 if (TID.mayStore() || TID.mayLoad())
150 return MI->readsRegister(Reg, TRI);
155 bool MLxExpansion::FindMLxHazard(MachineInstr *MI) const {
156 if (NumExpand >= ExpandLimit)
162 MachineInstr *DefMI = getAccDefMI(MI);
163 if (TII->isFpMLxInstruction(DefMI->getOpcode()))
165 // r3 = vmla r0, r1, r2
166 // takes 16 - 17 cycles
171 // takes about 14 - 15 cycles even with vmul stalling for 4 cycles.
174 // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the
175 // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall
176 // preserves the in-order retirement of the instructions.
177 // Look at the next few instructions, if *most* of them can cause hazards,
178 // then the scheduler can't *fix* this, we'd better break up the VMLA.
179 for (unsigned i = 1; i <= 4; ++i) {
180 int Idx = ((int)MIIdx - i + 4) % 4;
181 MachineInstr *NextMI = LastMIs[Idx];
185 if (TII->canCauseFpMLxStall(NextMI->getOpcode()))
188 // Look for VMLx RAW hazard.
189 if (hasRAWHazard(getDefReg(MI), NextMI))
196 /// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair
197 /// of MUL + ADD / SUB instructions.
199 MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
200 unsigned MulOpc, unsigned AddSubOpc,
201 bool NegAcc, bool HasLane) {
202 unsigned DstReg = MI->getOperand(0).getReg();
203 bool DstDead = MI->getOperand(0).isDead();
204 unsigned AccReg = MI->getOperand(1).getReg();
205 unsigned Src1Reg = MI->getOperand(2).getReg();
206 unsigned Src2Reg = MI->getOperand(3).getReg();
207 bool Src1Kill = MI->getOperand(2).isKill();
208 bool Src2Kill = MI->getOperand(3).isKill();
209 unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0;
210 unsigned NextOp = HasLane ? 5 : 4;
211 ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm();
212 unsigned PredReg = MI->getOperand(++NextOp).getReg();
214 const TargetInstrDesc &TID1 = TII->get(MulOpc);
215 const TargetInstrDesc &TID2 = TII->get(AddSubOpc);
216 unsigned TmpReg = MRI->createVirtualRegister(TID1.getRegClass(0, TRI));
218 MachineInstrBuilder MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), TID1, TmpReg)
219 .addReg(Src1Reg, getKillRegState(Src1Kill))
220 .addReg(Src2Reg, getKillRegState(Src2Kill));
223 MIB.addImm(Pred).addReg(PredReg);
225 MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), TID2)
226 .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead));
229 bool AccKill = MRI->hasOneNonDBGUse(AccReg);
230 MIB.addReg(TmpReg, getKillRegState(true))
231 .addReg(AccReg, getKillRegState(AccKill));
233 MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true));
235 MIB.addImm(Pred).addReg(PredReg);
238 dbgs() << "Expanding: " << *MI;
240 MachineBasicBlock::iterator MII = MI;
241 MII = llvm::prior(MII);
242 MachineInstr &MI2 = *MII;
243 MII = llvm::prior(MII);
244 MachineInstr &MI1 = *MII;
245 dbgs() << " " << MI1;
246 dbgs() << " " << MI2;
249 MI->eraseFromParent();
253 bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) {
254 bool Changed = false;
259 MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend();
261 MachineInstr *MI = &*MII;
263 if (MI->isLabel() || MI->isImplicitDef() || MI->isCopy()) {
268 const TargetInstrDesc &TID = MI->getDesc();
269 if (TID.isBarrier()) {
276 unsigned Domain = TID.TSFlags & ARMII::DomainMask;
277 if (Domain == ARMII::DomainGeneral) {
279 // Assume dual issues of non-VFP / NEON instructions.
284 unsigned MulOpc, AddSubOpc;
285 bool NegAcc, HasLane;
286 if (!TII->isFpMLxInstruction(TID.getOpcode(),
287 MulOpc, AddSubOpc, NegAcc, HasLane) ||
291 ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane);
292 E = MBB.rend(); // May have changed if MI was the 1st instruction.
304 bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
305 TII = static_cast<const ARMBaseInstrInfo*>(Fn.getTarget().getInstrInfo());
306 TRI = Fn.getTarget().getRegisterInfo();
307 MRI = &Fn.getRegInfo();
309 bool Modified = false;
310 for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E;
312 MachineBasicBlock &MBB = *MFI;
313 Modified |= ExpandFPMLxInstructions(MBB);
319 FunctionPass *llvm::createMLxExpansionPass() {
320 return new MLxExpansion();