1 //===-- MachineCSE.cpp - Machine Common Subexpression Elimination Pass ----===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass performs global common subexpression elimination on machine
11 // instructions using a scoped hash table based value numbering scheme. It
12 // must be run while the machine function is still in SSA form.
14 //===----------------------------------------------------------------------===//
16 #include "llvm/CodeGen/Passes.h"
17 #include "llvm/ADT/DenseMap.h"
18 #include "llvm/ADT/ScopedHashTable.h"
19 #include "llvm/ADT/SmallSet.h"
20 #include "llvm/ADT/Statistic.h"
21 #include "llvm/Analysis/AliasAnalysis.h"
22 #include "llvm/CodeGen/MachineDominators.h"
23 #include "llvm/CodeGen/MachineInstr.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/Support/Debug.h"
26 #include "llvm/Support/RecyclingAllocator.h"
27 #include "llvm/Target/TargetInstrInfo.h"
28 #include "llvm/Target/TargetSubtargetInfo.h"
31 #define DEBUG_TYPE "machine-cse"
33 STATISTIC(NumCoalesces, "Number of copies coalesced");
34 STATISTIC(NumCSEs, "Number of common subexpression eliminated");
35 STATISTIC(NumPhysCSEs,
36 "Number of physreg referencing common subexpr eliminated");
37 STATISTIC(NumCrossBBCSEs,
38 "Number of cross-MBB physreg referencing CS eliminated");
39 STATISTIC(NumCommutes, "Number of copies coalesced after commuting");
42 class MachineCSE : public MachineFunctionPass {
43 const TargetInstrInfo *TII;
44 const TargetRegisterInfo *TRI;
46 MachineDominatorTree *DT;
47 MachineRegisterInfo *MRI;
49 static char ID; // Pass identification
50 MachineCSE() : MachineFunctionPass(ID), LookAheadLimit(5), CurrVN(0) {
51 initializeMachineCSEPass(*PassRegistry::getPassRegistry());
54 bool runOnMachineFunction(MachineFunction &MF) override;
56 void getAnalysisUsage(AnalysisUsage &AU) const override {
58 MachineFunctionPass::getAnalysisUsage(AU);
59 AU.addRequired<AliasAnalysis>();
60 AU.addPreservedID(MachineLoopInfoID);
61 AU.addRequired<MachineDominatorTree>();
62 AU.addPreserved<MachineDominatorTree>();
65 void releaseMemory() override {
71 const unsigned LookAheadLimit;
72 typedef RecyclingAllocator<BumpPtrAllocator,
73 ScopedHashTableVal<MachineInstr*, unsigned> > AllocatorTy;
74 typedef ScopedHashTable<MachineInstr*, unsigned,
75 MachineInstrExpressionTrait, AllocatorTy> ScopedHTType;
76 typedef ScopedHTType::ScopeTy ScopeType;
77 DenseMap<MachineBasicBlock*, ScopeType*> ScopeMap;
79 SmallVector<MachineInstr*, 64> Exps;
82 bool PerformTrivialCopyPropagation(MachineInstr *MI,
83 MachineBasicBlock *MBB);
84 bool isPhysDefTriviallyDead(unsigned Reg,
85 MachineBasicBlock::const_iterator I,
86 MachineBasicBlock::const_iterator E) const;
87 bool hasLivePhysRegDefUses(const MachineInstr *MI,
88 const MachineBasicBlock *MBB,
89 SmallSet<unsigned,8> &PhysRefs,
90 SmallVectorImpl<unsigned> &PhysDefs,
91 bool &PhysUseDef) const;
92 bool PhysRegDefsReach(MachineInstr *CSMI, MachineInstr *MI,
93 SmallSet<unsigned,8> &PhysRefs,
94 SmallVectorImpl<unsigned> &PhysDefs,
95 bool &NonLocal) const;
96 bool isCSECandidate(MachineInstr *MI);
97 bool isProfitableToCSE(unsigned CSReg, unsigned Reg,
98 MachineInstr *CSMI, MachineInstr *MI);
99 void EnterScope(MachineBasicBlock *MBB);
100 void ExitScope(MachineBasicBlock *MBB);
101 bool ProcessBlock(MachineBasicBlock *MBB);
102 void ExitScopeIfDone(MachineDomTreeNode *Node,
103 DenseMap<MachineDomTreeNode*, unsigned> &OpenChildren);
104 bool PerformCSE(MachineDomTreeNode *Node);
106 } // end anonymous namespace
108 char MachineCSE::ID = 0;
109 char &llvm::MachineCSEID = MachineCSE::ID;
110 INITIALIZE_PASS_BEGIN(MachineCSE, "machine-cse",
111 "Machine Common Subexpression Elimination", false, false)
112 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
113 INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
114 INITIALIZE_PASS_END(MachineCSE, "machine-cse",
115 "Machine Common Subexpression Elimination", false, false)
117 /// The source register of a COPY machine instruction can be propagated to all
118 /// its users, and this propagation could increase the probability of finding
119 /// common subexpressions. If the COPY has only one user, the COPY itself can
121 bool MachineCSE::PerformTrivialCopyPropagation(MachineInstr *MI,
122 MachineBasicBlock *MBB) {
123 bool Changed = false;
124 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
125 MachineOperand &MO = MI->getOperand(i);
126 if (!MO.isReg() || !MO.isUse())
128 unsigned Reg = MO.getReg();
129 if (!TargetRegisterInfo::isVirtualRegister(Reg))
131 bool OnlyOneUse = MRI->hasOneNonDBGUse(Reg);
132 MachineInstr *DefMI = MRI->getVRegDef(Reg);
133 if (!DefMI->isCopy())
135 unsigned SrcReg = DefMI->getOperand(1).getReg();
136 if (!TargetRegisterInfo::isVirtualRegister(SrcReg))
138 if (DefMI->getOperand(0).getSubReg())
140 // FIXME: We should trivially coalesce subregister copies to expose CSE
141 // opportunities on instructions with truncated operands (see
142 // cse-add-with-overflow.ll). This can be done here as follows:
144 // RC = TRI->getMatchingSuperRegClass(MRI->getRegClass(SrcReg), RC,
146 // MO.substVirtReg(SrcReg, SrcSubReg, *TRI);
148 // The 2-addr pass has been updated to handle coalesced subregs. However,
149 // some machine-specific code still can't handle it.
150 // To handle it properly we also need a way find a constrained subregister
151 // class given a super-reg class and subreg index.
152 if (DefMI->getOperand(1).getSubReg())
154 const TargetRegisterClass *RC = MRI->getRegClass(Reg);
155 if (!MRI->constrainRegClass(SrcReg, RC))
157 DEBUG(dbgs() << "Coalescing: " << *DefMI);
158 DEBUG(dbgs() << "*** to: " << *MI);
159 // Propagate SrcReg of copies to MI.
161 MRI->clearKillFlags(SrcReg);
162 // Coalesce single use copies.
164 DefMI->eraseFromParent();
174 MachineCSE::isPhysDefTriviallyDead(unsigned Reg,
175 MachineBasicBlock::const_iterator I,
176 MachineBasicBlock::const_iterator E) const {
177 unsigned LookAheadLeft = LookAheadLimit;
178 while (LookAheadLeft) {
179 // Skip over dbg_value's.
180 while (I != E && I->isDebugValue())
184 // Reached end of block, register is obviously dead.
187 bool SeenDef = false;
188 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
189 const MachineOperand &MO = I->getOperand(i);
190 if (MO.isRegMask() && MO.clobbersPhysReg(Reg))
192 if (!MO.isReg() || !MO.getReg())
194 if (!TRI->regsOverlap(MO.getReg(), Reg))
202 // See a def of Reg (or an alias) before encountering any use, it's
212 /// hasLivePhysRegDefUses - Return true if the specified instruction read/write
213 /// physical registers (except for dead defs of physical registers). It also
214 /// returns the physical register def by reference if it's the only one and the
215 /// instruction does not uses a physical register.
216 bool MachineCSE::hasLivePhysRegDefUses(const MachineInstr *MI,
217 const MachineBasicBlock *MBB,
218 SmallSet<unsigned,8> &PhysRefs,
219 SmallVectorImpl<unsigned> &PhysDefs,
220 bool &PhysUseDef) const{
221 // First, add all uses to PhysRefs.
222 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
223 const MachineOperand &MO = MI->getOperand(i);
224 if (!MO.isReg() || MO.isDef())
226 unsigned Reg = MO.getReg();
229 if (TargetRegisterInfo::isVirtualRegister(Reg))
231 // Reading constant physregs is ok.
232 if (!MRI->isConstantPhysReg(Reg, *MBB->getParent()))
233 for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
234 PhysRefs.insert(*AI);
237 // Next, collect all defs into PhysDefs. If any is already in PhysRefs
238 // (which currently contains only uses), set the PhysUseDef flag.
240 MachineBasicBlock::const_iterator I = MI; I = std::next(I);
241 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
242 const MachineOperand &MO = MI->getOperand(i);
243 if (!MO.isReg() || !MO.isDef())
245 unsigned Reg = MO.getReg();
248 if (TargetRegisterInfo::isVirtualRegister(Reg))
250 // Check against PhysRefs even if the def is "dead".
251 if (PhysRefs.count(Reg))
253 // If the def is dead, it's ok. But the def may not marked "dead". That's
254 // common since this pass is run before livevariables. We can scan
255 // forward a few instructions and check if it is obviously dead.
256 if (!MO.isDead() && !isPhysDefTriviallyDead(Reg, I, MBB->end()))
257 PhysDefs.push_back(Reg);
260 // Finally, add all defs to PhysRefs as well.
261 for (unsigned i = 0, e = PhysDefs.size(); i != e; ++i)
262 for (MCRegAliasIterator AI(PhysDefs[i], TRI, true); AI.isValid(); ++AI)
263 PhysRefs.insert(*AI);
265 return !PhysRefs.empty();
268 bool MachineCSE::PhysRegDefsReach(MachineInstr *CSMI, MachineInstr *MI,
269 SmallSet<unsigned,8> &PhysRefs,
270 SmallVectorImpl<unsigned> &PhysDefs,
271 bool &NonLocal) const {
272 // For now conservatively returns false if the common subexpression is
273 // not in the same basic block as the given instruction. The only exception
274 // is if the common subexpression is in the sole predecessor block.
275 const MachineBasicBlock *MBB = MI->getParent();
276 const MachineBasicBlock *CSMBB = CSMI->getParent();
278 bool CrossMBB = false;
280 if (MBB->pred_size() != 1 || *MBB->pred_begin() != CSMBB)
283 for (unsigned i = 0, e = PhysDefs.size(); i != e; ++i) {
284 if (MRI->isAllocatable(PhysDefs[i]) || MRI->isReserved(PhysDefs[i]))
285 // Avoid extending live range of physical registers if they are
286 //allocatable or reserved.
291 MachineBasicBlock::const_iterator I = CSMI; I = std::next(I);
292 MachineBasicBlock::const_iterator E = MI;
293 MachineBasicBlock::const_iterator EE = CSMBB->end();
294 unsigned LookAheadLeft = LookAheadLimit;
295 while (LookAheadLeft) {
296 // Skip over dbg_value's.
297 while (I != E && I != EE && I->isDebugValue())
301 assert(CrossMBB && "Reaching end-of-MBB without finding MI?");
313 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
314 const MachineOperand &MO = I->getOperand(i);
315 // RegMasks go on instructions like calls that clobber lots of physregs.
316 // Don't attempt to CSE across such an instruction.
319 if (!MO.isReg() || !MO.isDef())
321 unsigned MOReg = MO.getReg();
322 if (TargetRegisterInfo::isVirtualRegister(MOReg))
324 if (PhysRefs.count(MOReg))
335 bool MachineCSE::isCSECandidate(MachineInstr *MI) {
336 if (MI->isPosition() || MI->isPHI() || MI->isImplicitDef() || MI->isKill() ||
337 MI->isInlineAsm() || MI->isDebugValue())
341 if (MI->isCopyLike())
344 // Ignore stuff that we obviously can't move.
345 if (MI->mayStore() || MI->isCall() || MI->isTerminator() ||
346 MI->hasUnmodeledSideEffects())
350 // Okay, this instruction does a load. As a refinement, we allow the target
351 // to decide whether the loaded value is actually a constant. If so, we can
352 // actually use it as a load.
353 if (!MI->isInvariantLoad(AA))
354 // FIXME: we should be able to hoist loads with no other side effects if
355 // there are no other instructions which can change memory in this loop.
356 // This is a trivial form of alias analysis.
362 /// isProfitableToCSE - Return true if it's profitable to eliminate MI with a
363 /// common expression that defines Reg.
364 bool MachineCSE::isProfitableToCSE(unsigned CSReg, unsigned Reg,
365 MachineInstr *CSMI, MachineInstr *MI) {
366 // FIXME: Heuristics that works around the lack the live range splitting.
368 // If CSReg is used at all uses of Reg, CSE should not increase register
369 // pressure of CSReg.
370 bool MayIncreasePressure = true;
371 if (TargetRegisterInfo::isVirtualRegister(CSReg) &&
372 TargetRegisterInfo::isVirtualRegister(Reg)) {
373 MayIncreasePressure = false;
374 SmallPtrSet<MachineInstr*, 8> CSUses;
375 for (MachineInstr &MI : MRI->use_nodbg_instructions(CSReg)) {
378 for (MachineInstr &MI : MRI->use_nodbg_instructions(Reg)) {
379 if (!CSUses.count(&MI)) {
380 MayIncreasePressure = true;
385 if (!MayIncreasePressure) return true;
387 // Heuristics #1: Don't CSE "cheap" computation if the def is not local or in
388 // an immediate predecessor. We don't want to increase register pressure and
389 // end up causing other computation to be spilled.
390 if (TII->isAsCheapAsAMove(MI)) {
391 MachineBasicBlock *CSBB = CSMI->getParent();
392 MachineBasicBlock *BB = MI->getParent();
393 if (CSBB != BB && !CSBB->isSuccessor(BB))
397 // Heuristics #2: If the expression doesn't not use a vr and the only use
398 // of the redundant computation are copies, do not cse.
399 bool HasVRegUse = false;
400 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
401 const MachineOperand &MO = MI->getOperand(i);
402 if (MO.isReg() && MO.isUse() &&
403 TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
409 bool HasNonCopyUse = false;
410 for (MachineInstr &MI : MRI->use_nodbg_instructions(Reg)) {
412 if (!MI.isCopyLike()) {
413 HasNonCopyUse = true;
421 // Heuristics #3: If the common subexpression is used by PHIs, do not reuse
422 // it unless the defined value is already used in the BB of the new use.
424 SmallPtrSet<MachineBasicBlock*, 4> CSBBs;
425 for (MachineInstr &MI : MRI->use_nodbg_instructions(CSReg)) {
426 HasPHI |= MI.isPHI();
427 CSBBs.insert(MI.getParent());
432 return CSBBs.count(MI->getParent());
435 void MachineCSE::EnterScope(MachineBasicBlock *MBB) {
436 DEBUG(dbgs() << "Entering: " << MBB->getName() << '\n');
437 ScopeType *Scope = new ScopeType(VNT);
438 ScopeMap[MBB] = Scope;
441 void MachineCSE::ExitScope(MachineBasicBlock *MBB) {
442 DEBUG(dbgs() << "Exiting: " << MBB->getName() << '\n');
443 DenseMap<MachineBasicBlock*, ScopeType*>::iterator SI = ScopeMap.find(MBB);
444 assert(SI != ScopeMap.end());
449 bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) {
450 bool Changed = false;
452 SmallVector<std::pair<unsigned, unsigned>, 8> CSEPairs;
453 SmallVector<unsigned, 2> ImplicitDefsToUpdate;
454 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E; ) {
455 MachineInstr *MI = &*I;
458 if (!isCSECandidate(MI))
461 bool FoundCSE = VNT.count(MI);
463 // Using trivial copy propagation to find more CSE opportunities.
464 if (PerformTrivialCopyPropagation(MI, MBB)) {
467 // After coalescing MI itself may become a copy.
468 if (MI->isCopyLike())
471 // Try again to see if CSE is possible.
472 FoundCSE = VNT.count(MI);
476 // Commute commutable instructions.
477 bool Commuted = false;
478 if (!FoundCSE && MI->isCommutable()) {
479 MachineInstr *NewMI = TII->commuteInstruction(MI);
482 FoundCSE = VNT.count(NewMI);
484 // New instruction. It doesn't need to be kept.
485 NewMI->eraseFromParent();
487 } else if (!FoundCSE)
488 // MI was changed but it didn't help, commute it back!
489 (void)TII->commuteInstruction(MI);
493 // If the instruction defines physical registers and the values *may* be
494 // used, then it's not safe to replace it with a common subexpression.
495 // It's also not safe if the instruction uses physical registers.
496 bool CrossMBBPhysDef = false;
497 SmallSet<unsigned, 8> PhysRefs;
498 SmallVector<unsigned, 2> PhysDefs;
499 bool PhysUseDef = false;
500 if (FoundCSE && hasLivePhysRegDefUses(MI, MBB, PhysRefs,
501 PhysDefs, PhysUseDef)) {
504 // ... Unless the CS is local or is in the sole predecessor block
505 // and it also defines the physical register which is not clobbered
506 // in between and the physical register uses were not clobbered.
507 // This can never be the case if the instruction both uses and
508 // defines the same physical register, which was detected above.
510 unsigned CSVN = VNT.lookup(MI);
511 MachineInstr *CSMI = Exps[CSVN];
512 if (PhysRegDefsReach(CSMI, MI, PhysRefs, PhysDefs, CrossMBBPhysDef))
518 VNT.insert(MI, CurrVN++);
523 // Found a common subexpression, eliminate it.
524 unsigned CSVN = VNT.lookup(MI);
525 MachineInstr *CSMI = Exps[CSVN];
526 DEBUG(dbgs() << "Examining: " << *MI);
527 DEBUG(dbgs() << "*** Found a common subexpression: " << *CSMI);
529 // Check if it's profitable to perform this CSE.
531 unsigned NumDefs = MI->getDesc().getNumDefs() +
532 MI->getDesc().getNumImplicitDefs();
534 for (unsigned i = 0, e = MI->getNumOperands(); NumDefs && i != e; ++i) {
535 MachineOperand &MO = MI->getOperand(i);
536 if (!MO.isReg() || !MO.isDef())
538 unsigned OldReg = MO.getReg();
539 unsigned NewReg = CSMI->getOperand(i).getReg();
541 // Go through implicit defs of CSMI and MI, if a def is not dead at MI,
542 // we should make sure it is not dead at CSMI.
543 if (MO.isImplicit() && !MO.isDead() && CSMI->getOperand(i).isDead())
544 ImplicitDefsToUpdate.push_back(i);
545 if (OldReg == NewReg) {
550 assert(TargetRegisterInfo::isVirtualRegister(OldReg) &&
551 TargetRegisterInfo::isVirtualRegister(NewReg) &&
552 "Do not CSE physical register defs!");
554 if (!isProfitableToCSE(NewReg, OldReg, CSMI, MI)) {
555 DEBUG(dbgs() << "*** Not profitable, avoid CSE!\n");
560 // Don't perform CSE if the result of the old instruction cannot exist
561 // within the register class of the new instruction.
562 const TargetRegisterClass *OldRC = MRI->getRegClass(OldReg);
563 if (!MRI->constrainRegClass(NewReg, OldRC)) {
564 DEBUG(dbgs() << "*** Not the same register class, avoid CSE!\n");
569 CSEPairs.push_back(std::make_pair(OldReg, NewReg));
573 // Actually perform the elimination.
575 for (unsigned i = 0, e = CSEPairs.size(); i != e; ++i) {
576 MRI->replaceRegWith(CSEPairs[i].first, CSEPairs[i].second);
577 MRI->clearKillFlags(CSEPairs[i].second);
580 // Go through implicit defs of CSMI and MI, if a def is not dead at MI,
581 // we should make sure it is not dead at CSMI.
582 for (unsigned i = 0, e = ImplicitDefsToUpdate.size(); i != e; ++i)
583 CSMI->getOperand(ImplicitDefsToUpdate[i]).setIsDead(false);
585 if (CrossMBBPhysDef) {
586 // Add physical register defs now coming in from a predecessor to MBB
588 while (!PhysDefs.empty()) {
589 unsigned LiveIn = PhysDefs.pop_back_val();
590 if (!MBB->isLiveIn(LiveIn))
591 MBB->addLiveIn(LiveIn);
596 MI->eraseFromParent();
598 if (!PhysRefs.empty())
604 VNT.insert(MI, CurrVN++);
608 ImplicitDefsToUpdate.clear();
614 /// ExitScopeIfDone - Destroy scope for the MBB that corresponds to the given
615 /// dominator tree node if its a leaf or all of its children are done. Walk
616 /// up the dominator tree to destroy ancestors which are now done.
618 MachineCSE::ExitScopeIfDone(MachineDomTreeNode *Node,
619 DenseMap<MachineDomTreeNode*, unsigned> &OpenChildren) {
620 if (OpenChildren[Node])
624 ExitScope(Node->getBlock());
626 // Now traverse upwards to pop ancestors whose offsprings are all done.
627 while (MachineDomTreeNode *Parent = Node->getIDom()) {
628 unsigned Left = --OpenChildren[Parent];
631 ExitScope(Parent->getBlock());
636 bool MachineCSE::PerformCSE(MachineDomTreeNode *Node) {
637 SmallVector<MachineDomTreeNode*, 32> Scopes;
638 SmallVector<MachineDomTreeNode*, 8> WorkList;
639 DenseMap<MachineDomTreeNode*, unsigned> OpenChildren;
643 // Perform a DFS walk to determine the order of visit.
644 WorkList.push_back(Node);
646 Node = WorkList.pop_back_val();
647 Scopes.push_back(Node);
648 const std::vector<MachineDomTreeNode*> &Children = Node->getChildren();
649 unsigned NumChildren = Children.size();
650 OpenChildren[Node] = NumChildren;
651 for (unsigned i = 0; i != NumChildren; ++i) {
652 MachineDomTreeNode *Child = Children[i];
653 WorkList.push_back(Child);
655 } while (!WorkList.empty());
658 bool Changed = false;
659 for (unsigned i = 0, e = Scopes.size(); i != e; ++i) {
660 MachineDomTreeNode *Node = Scopes[i];
661 MachineBasicBlock *MBB = Node->getBlock();
663 Changed |= ProcessBlock(MBB);
664 // If it's a leaf node, it's done. Traverse upwards to pop ancestors.
665 ExitScopeIfDone(Node, OpenChildren);
671 bool MachineCSE::runOnMachineFunction(MachineFunction &MF) {
672 if (skipOptnoneFunction(*MF.getFunction()))
675 TII = MF.getSubtarget().getInstrInfo();
676 TRI = MF.getSubtarget().getRegisterInfo();
677 MRI = &MF.getRegInfo();
678 AA = &getAnalysis<AliasAnalysis>();
679 DT = &getAnalysis<MachineDominatorTree>();
680 return PerformCSE(DT->getRootNode());