X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FVirtRegRewriter.cpp;h=14ee69da4c2a871f107d7b83158dedd9c9d2c0f0;hb=09aa3f0ef35d9241c92439d74b8d5e9a81d814c2;hp=7f0412cf2d6db436778e4aac620e6c3d1fde14d9;hpb=bd6cb4ba1f743f840da844b44aa0ec92e56777ee;p=oota-llvm.git diff --git a/lib/CodeGen/VirtRegRewriter.cpp b/lib/CodeGen/VirtRegRewriter.cpp index 7f0412cf2d6..14ee69da4c2 100644 --- a/lib/CodeGen/VirtRegRewriter.cpp +++ b/lib/CodeGen/VirtRegRewriter.cpp @@ -22,8 +22,8 @@ #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" -#include using namespace llvm; STATISTIC(NumDSE , "Number of dead stores elided"); @@ -67,23 +67,16 @@ VirtRegRewriter::~VirtRegRewriter() {} /// Note that operands may be added, so the MO reference is no longer valid. static void substitutePhysReg(MachineOperand &MO, unsigned Reg, const TargetRegisterInfo &TRI) { - if (unsigned SubIdx = MO.getSubReg()) { - // Insert the physical subreg and reset the subreg field. - MO.setReg(TRI.getSubReg(Reg, SubIdx)); - MO.setSubReg(0); - - // Any def, dead, and kill flags apply to the full virtual register, so they - // also apply to the full physical register. Add imp-def/dead and imp-kill - // as needed. + if (MO.getSubReg()) { + MO.substPhysReg(Reg, TRI); + + // Any kill flags apply to the full virtual register, so they also apply to + // the full physical register. + // We assume that partial defs have already been decorated with a super-reg + // operand by LiveIntervals. MachineInstr &MI = *MO.getParent(); - if (MO.isDef()) - if (MO.isDead()) - MI.addRegisterDead(Reg, &TRI, /*AddIfNotFound=*/ true); - else - MI.addRegisterDefined(Reg, &TRI); - else if (!MO.isUndef() && - (MO.isKill() || - MI.isRegTiedToDefOperand(&MO-&MI.getOperand(0)))) + if (MO.isUse() && !MO.isUndef() && + (MO.isKill() || MI.isRegTiedToDefOperand(&MO-&MI.getOperand(0)))) MI.addRegisterKilled(Reg, &TRI, /*AddIfNotFound=*/ true); } else { MO.setReg(Reg); @@ -304,7 +297,7 @@ ComputeReloadLoc(MachineBasicBlock::iterator const InsertLoc, const TargetLowering *TL = MF.getTarget().getTargetLowering(); if (!TL->isTypeLegal(TL->getPointerTy())) - // Believe it or not, this is true on PIC16. + // Believe it or not, this is true on 16-bit targets like PIC16. return InsertLoc; const TargetRegisterClass *ptrRegClass = @@ -460,7 +453,7 @@ public: /// blocks each of which is a successor of the specified BB and has no other /// predecessor. static void findSinglePredSuccessor(MachineBasicBlock *MBB, - SmallVectorImpl &Succs) { + SmallVectorImpl &Succs){ for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(), SE = MBB->succ_end(); SI != SE; ++SI) { MachineBasicBlock *SuccMBB = *SI; @@ -667,8 +660,7 @@ static void ReMaterialize(MachineBasicBlock &MBB, assert(TID.getNumDefs() == 1 && "Don't know how to remat instructions that define > 1 values!"); #endif - TII->reMaterialize(MBB, MII, DestReg, - ReMatDefMI->getOperand(0).getSubReg(), ReMatDefMI, TRI); + TII->reMaterialize(MBB, MII, DestReg, 0, ReMatDefMI, *TRI); MachineInstr *NewMI = prior(MII); for (unsigned i = 0, e = NewMI->getNumOperands(); i != e; ++i) { MachineOperand &MO = NewMI->getOperand(i); @@ -769,7 +761,7 @@ void AvailableSpills::AddAvailableRegsToLiveIn(MachineBasicBlock &MBB, I = PhysRegsAvailable.begin(), E = PhysRegsAvailable.end(); I != E; ++I) { unsigned Reg = I->first; - const TargetRegisterClass* RC = TRI->getPhysicalRegisterRegClass(Reg); + const TargetRegisterClass* RC = TRI->getMinimalPhysRegClass(Reg); // FIXME: A temporary workaround. We can't reuse available value if it's // not safe to move the def of the virtual register's class. e.g. // X86::RFP* register classes. Do not add it as a live-in. @@ -853,8 +845,8 @@ unsigned ReuseInfo::GetRegForReload(const TargetRegisterClass *RC, // Yup, use the reload register that we didn't use before. unsigned NewReg = Op.AssignedPhysReg; Rejected.insert(PhysReg); - return GetRegForReload(RC, NewReg, MF, MI, Spills, MaybeDeadStores, Rejected, - RegKills, KillOps, VRM); + return GetRegForReload(RC, NewReg, MF, MI, Spills, MaybeDeadStores, + Rejected, RegKills, KillOps, VRM); } else { // Otherwise, we might also have a problem if a previously reused // value aliases the new register. If so, codegen the previous reload @@ -907,7 +899,7 @@ unsigned ReuseInfo::GetRegForReload(const TargetRegisterClass *RC, TRI, VRM); } else { TII->loadRegFromStackSlot(*MBB, InsertLoc, NewPhysReg, - NewOp.StackSlotOrReMat, AliasRC); + NewOp.StackSlotOrReMat, AliasRC, TRI); MachineInstr *LoadMI = prior(InsertLoc); VRM.addSpillSlotUse(NewOp.StackSlotOrReMat, LoadMI); // Any stores to this stack slot are not dead anymore. @@ -1022,7 +1014,7 @@ static unsigned FindFreeRegister(MachineBasicBlock::iterator MII, for (unsigned i = 0, e = Kills.size(); i != e; ++i) { unsigned Kill = Kills[i]; if (!Defs[Kill] && !Uses[Kill] && - TRI->getPhysicalRegisterRegClass(Kill) == RC) + RC->contains(Kill)) return Kill; } for (unsigned i = 0, e = LocalUses.size(); i != e; ++i) { @@ -1118,6 +1110,12 @@ private: bool InsertSpills(MachineInstr *MI); + void ProcessUses(MachineInstr &MI, AvailableSpills &Spills, + std::vector &MaybeDeadStores, + BitVector &RegKills, + ReuseInfo &ReusedOperands, + std::vector &KillOps); + void RewriteMBB(LiveIntervals *LIs, AvailableSpills &Spills, BitVector &RegKills, std::vector &KillOps); @@ -1265,7 +1263,7 @@ OptimizeByUnfold2(unsigned VirtReg, int SS, ComputeReloadLoc(MII, MBB->begin(), PhysReg, TRI, false, SS, TII, MF); // Load from SS to the spare physical register. - TII->loadRegFromStackSlot(*MBB, MII, PhysReg, SS, RC); + TII->loadRegFromStackSlot(*MBB, MII, PhysReg, SS, RC, TRI); // This invalidates Phys. Spills.ClobberPhysReg(PhysReg); // Remember it's available. @@ -1308,7 +1306,7 @@ OptimizeByUnfold2(unsigned VirtReg, int SS, } while (FoldsStackSlotModRef(*NextMII, SS, PhysReg, TII, TRI, *VRM)); // Store the value back into SS. - TII->storeRegToStackSlot(*MBB, NextMII, PhysReg, true, SS, RC); + TII->storeRegToStackSlot(*MBB, NextMII, PhysReg, true, SS, RC, TRI); MachineInstr *StoreMI = prior(NextMII); VRM->addSpillSlotUse(SS, StoreMI); VRM->virtFolded(VirtReg, StoreMI, VirtRegMap::isMod); @@ -1410,25 +1408,25 @@ OptimizeByUnfold(MachineBasicBlock::iterator &MII, if (TII->unfoldMemoryOperand(MF, &MI, UnfoldVR, false, false, NewMIs)) { assert(NewMIs.size() == 1); MachineInstr *NewMI = NewMIs.back(); + MBB->insert(MII, NewMI); NewMIs.clear(); int Idx = NewMI->findRegisterUseOperandIdx(VirtReg, false); assert(Idx != -1); SmallVector Ops; Ops.push_back(Idx); - MachineInstr *FoldedMI = TII->foldMemoryOperand(MF, NewMI, Ops, SS); + MachineInstr *FoldedMI = TII->foldMemoryOperand(NewMI, Ops, SS); + NewMI->eraseFromParent(); if (FoldedMI) { VRM->addSpillSlotUse(SS, FoldedMI); if (!VRM->hasPhys(UnfoldVR)) VRM->assignVirt2Phys(UnfoldVR, UnfoldPR); VRM->virtFolded(VirtReg, FoldedMI, VirtRegMap::isRef); - MII = MBB->insert(MII, FoldedMI); + MII = FoldedMI; InvalidateKills(MI, TRI, RegKills, KillOps); VRM->RemoveMachineInstrFromMaps(&MI); MBB->erase(&MI); - MF.DeleteMachineInstr(NewMI); return true; } - MF.DeleteMachineInstr(NewMI); } } @@ -1480,7 +1478,6 @@ CommuteToFoldReload(MachineBasicBlock::iterator &MII, if (MII == MBB->begin() || !MII->killsRegister(SrcReg)) return false; - MachineFunction &MF = *MBB->getParent(); MachineInstr &MI = *MII; MachineBasicBlock::iterator DefMII = prior(MII); MachineInstr *DefMI = DefMII; @@ -1511,11 +1508,12 @@ CommuteToFoldReload(MachineBasicBlock::iterator &MII, MachineInstr *CommutedMI = TII->commuteInstruction(DefMI, true); if (!CommutedMI) return false; + MBB->insert(MII, CommutedMI); SmallVector Ops; Ops.push_back(NewDstIdx); - MachineInstr *FoldedMI = TII->foldMemoryOperand(MF, CommutedMI, Ops, SS); + MachineInstr *FoldedMI = TII->foldMemoryOperand(CommutedMI, Ops, SS); // Not needed since foldMemoryOperand returns new MI. - MF.DeleteMachineInstr(CommutedMI); + CommutedMI->eraseFromParent(); if (!FoldedMI) return false; @@ -1523,12 +1521,12 @@ CommuteToFoldReload(MachineBasicBlock::iterator &MII, VRM->virtFolded(VirtReg, FoldedMI, VirtRegMap::isRef); // Insert new def MI and spill MI. const TargetRegisterClass* RC = MRI->getRegClass(VirtReg); - TII->storeRegToStackSlot(*MBB, &MI, NewReg, true, SS, RC); + TII->storeRegToStackSlot(*MBB, &MI, NewReg, true, SS, RC, TRI); MII = prior(MII); MachineInstr *StoreMI = MII; VRM->addSpillSlotUse(SS, StoreMI); VRM->virtFolded(VirtReg, StoreMI, VirtRegMap::isMod); - MII = MBB->insert(MII, FoldedMI); // Update MII to backtrack. + MII = FoldedMI; // Update MII to backtrack. // Delete all 3 old instructions. InvalidateKills(*ReloadMI, TRI, RegKills, KillOps); @@ -1566,7 +1564,8 @@ SpillRegToStackSlot(MachineBasicBlock::iterator &MII, std::vector &KillOps) { MachineBasicBlock::iterator oldNextMII = llvm::next(MII); - TII->storeRegToStackSlot(*MBB, llvm::next(MII), PhysReg, true, StackSlot, RC); + TII->storeRegToStackSlot(*MBB, llvm::next(MII), PhysReg, true, StackSlot, RC, + TRI); MachineInstr *StoreMI = prior(oldNextMII); VRM->addSpillSlotUse(StackSlot, StoreMI); DEBUG(dbgs() << "Store:\t" << *StoreMI); @@ -1703,13 +1702,13 @@ bool LocalRewriter::InsertEmergencySpills(MachineInstr *MI) { std::vector &EmSpills = VRM->getEmergencySpills(MI); for (unsigned i = 0, e = EmSpills.size(); i != e; ++i) { unsigned PhysReg = EmSpills[i]; - const TargetRegisterClass *RC = TRI->getPhysicalRegisterRegClass(PhysReg); + const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(PhysReg); assert(RC && "Unable to determine register class!"); int SS = VRM->getEmergencySpillSlot(RC); if (UsedSS.count(SS)) llvm_unreachable("Need to spill more than one physical registers!"); UsedSS.insert(SS); - TII->storeRegToStackSlot(*MBB, MII, PhysReg, true, SS, RC); + TII->storeRegToStackSlot(*MBB, MII, PhysReg, true, SS, RC, TRI); MachineInstr *StoreMI = prior(MII); VRM->addSpillSlotUse(SS, StoreMI); @@ -1718,7 +1717,7 @@ bool LocalRewriter::InsertEmergencySpills(MachineInstr *MI) { ComputeReloadLoc(llvm::next(MII), MBB->begin(), PhysReg, TRI, false, SS, TII, *MBB->getParent()); - TII->loadRegFromStackSlot(*MBB, InsertLoc, PhysReg, SS, RC); + TII->loadRegFromStackSlot(*MBB, InsertLoc, PhysReg, SS, RC, TRI); MachineInstr *LoadMI = prior(InsertLoc); VRM->addSpillSlotUse(SS, LoadMI); @@ -1758,7 +1757,6 @@ bool LocalRewriter::InsertRestores(MachineInstr *MI, bool DoReMat = VRM->isReMaterialized(VirtReg); int SSorRMId = DoReMat ? VRM->getReMatId(VirtReg) : VRM->getStackSlot(VirtReg); - const TargetRegisterClass* RC = MRI->getRegClass(VirtReg); unsigned InReg = Spills.getSpillSlotOrReMatPhysReg(SSorRMId); if (InReg == Phys) { // If the value is already available in the expected register, save @@ -1792,19 +1790,16 @@ bool LocalRewriter::InsertRestores(MachineInstr *MI, MachineBasicBlock::iterator InsertLoc = ComputeReloadLoc(MII, MBB->begin(), Phys, TRI, DoReMat, SSorRMId, TII, *MBB->getParent()); - - TII->copyRegToReg(*MBB, InsertLoc, Phys, InReg, RC, RC); + MachineInstr *CopyMI = BuildMI(*MBB, InsertLoc, MI->getDebugLoc(), + TII->get(TargetOpcode::COPY), Phys) + .addReg(InReg, RegState::Kill); // This invalidates Phys. Spills.ClobberPhysReg(Phys); // Remember it's available. Spills.addAvailable(SSorRMId, Phys); - // Mark is killed. - MachineInstr *CopyMI = prior(InsertLoc); CopyMI->setAsmPrinterFlag(MachineInstr::ReloadReuse); - MachineOperand *KillOpnd = CopyMI->findRegisterUseOperand(InReg); - KillOpnd->setIsKill(); UpdateKills(*CopyMI, TRI, RegKills, KillOps); DEBUG(dbgs() << '\t' << *CopyMI); @@ -1821,7 +1816,7 @@ bool LocalRewriter::InsertRestores(MachineInstr *MI, ReMaterialize(*MBB, InsertLoc, Phys, VirtReg, TII, TRI, *VRM); } else { const TargetRegisterClass* RC = MRI->getRegClass(VirtReg); - TII->loadRegFromStackSlot(*MBB, InsertLoc, Phys, SSorRMId, RC); + TII->loadRegFromStackSlot(*MBB, InsertLoc, Phys, SSorRMId, RC, TRI); MachineInstr *LoadMI = prior(InsertLoc); VRM->addSpillSlotUse(SSorRMId, LoadMI); ++NumLoads; @@ -1839,7 +1834,7 @@ bool LocalRewriter::InsertRestores(MachineInstr *MI, return true; } -/// InsertEmergencySpills - Insert spills after MI if requested by VRM. Return +/// InsertSpills - Insert spills after MI if requested by VRM. Return /// true if spills were inserted. bool LocalRewriter::InsertSpills(MachineInstr *MI) { if (!VRM->isSpillPt(MI)) @@ -1857,7 +1852,7 @@ bool LocalRewriter::InsertSpills(MachineInstr *MI) { int StackSlot = VRM->getStackSlot(VirtReg); MachineBasicBlock::iterator oldNextMII = llvm::next(MII); TII->storeRegToStackSlot(*MBB, llvm::next(MII), Phys, isKill, StackSlot, - RC); + RC, TRI); MachineInstr *StoreMI = prior(oldNextMII); VRM->addSpillSlotUse(StackSlot, StoreMI); DEBUG(dbgs() << "Store:\t" << *StoreMI); @@ -1867,8 +1862,350 @@ bool LocalRewriter::InsertSpills(MachineInstr *MI) { } +/// ProcessUses - Process all of MI's spilled operands and all available +/// operands. +void LocalRewriter::ProcessUses(MachineInstr &MI, AvailableSpills &Spills, + std::vector &MaybeDeadStores, + BitVector &RegKills, + ReuseInfo &ReusedOperands, + std::vector &KillOps) { + // Clear kill info. + SmallSet KilledMIRegs; + SmallVector VirtUseOps; + for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { + MachineOperand &MO = MI.getOperand(i); + if (!MO.isReg() || MO.getReg() == 0) + continue; // Ignore non-register operands. + + unsigned VirtReg = MO.getReg(); + if (TargetRegisterInfo::isPhysicalRegister(VirtReg)) { + // Ignore physregs for spilling, but remember that it is used by this + // function. + MRI->setPhysRegUsed(VirtReg); + continue; + } + + // We want to process implicit virtual register uses first. + if (MO.isImplicit()) + // If the virtual register is implicitly defined, emit a implicit_def + // before so scavenger knows it's "defined". + // FIXME: This is a horrible hack done the by register allocator to + // remat a definition with virtual register operand. + VirtUseOps.insert(VirtUseOps.begin(), i); + else + VirtUseOps.push_back(i); + + // A partial def causes problems because the same operand both reads and + // writes the register. This rewriter is designed to rewrite uses and defs + // separately, so a partial def would already have been rewritten to a + // physreg by the time we get to processing defs. + // Add an implicit use operand to model the partial def. + if (MO.isDef() && MO.getSubReg() && MI.readsVirtualRegister(VirtReg) && + MI.findRegisterUseOperandIdx(VirtReg) == -1) { + VirtUseOps.insert(VirtUseOps.begin(), MI.getNumOperands()); + MI.addOperand(MachineOperand::CreateReg(VirtReg, + false, // isDef + true)); // isImplicit + DEBUG(dbgs() << "Partial redef: " << MI); + } + } + + // Process all of the spilled uses and all non spilled reg references. + SmallVector PotentialDeadStoreSlots; + KilledMIRegs.clear(); + for (unsigned j = 0, e = VirtUseOps.size(); j != e; ++j) { + unsigned i = VirtUseOps[j]; + unsigned VirtReg = MI.getOperand(i).getReg(); + assert(TargetRegisterInfo::isVirtualRegister(VirtReg) && + "Not a virtual register?"); + + unsigned SubIdx = MI.getOperand(i).getSubReg(); + if (VRM->isAssignedReg(VirtReg)) { + // This virtual register was assigned a physreg! + unsigned Phys = VRM->getPhys(VirtReg); + MRI->setPhysRegUsed(Phys); + if (MI.getOperand(i).isDef()) + ReusedOperands.markClobbered(Phys); + substitutePhysReg(MI.getOperand(i), Phys, *TRI); + if (VRM->isImplicitlyDefined(VirtReg)) + // FIXME: Is this needed? + BuildMI(*MBB, &MI, MI.getDebugLoc(), + TII->get(TargetOpcode::IMPLICIT_DEF), Phys); + continue; + } + + // This virtual register is now known to be a spilled value. + if (!MI.getOperand(i).isUse()) + continue; // Handle defs in the loop below (handle use&def here though) + + bool AvoidReload = MI.getOperand(i).isUndef(); + // Check if it is defined by an implicit def. It should not be spilled. + // Note, this is for correctness reason. e.g. + // 8 %reg1024 = IMPLICIT_DEF + // 12 %reg1024 = INSERT_SUBREG %reg1024, %reg1025, 2 + // The live range [12, 14) are not part of the r1024 live interval since + // it's defined by an implicit def. It will not conflicts with live + // interval of r1025. Now suppose both registers are spilled, you can + // easily see a situation where both registers are reloaded before + // the INSERT_SUBREG and both target registers that would overlap. + bool DoReMat = VRM->isReMaterialized(VirtReg); + int SSorRMId = DoReMat + ? VRM->getReMatId(VirtReg) : VRM->getStackSlot(VirtReg); + int ReuseSlot = SSorRMId; + + // Check to see if this stack slot is available. + unsigned PhysReg = Spills.getSpillSlotOrReMatPhysReg(SSorRMId); + + // If this is a sub-register use, make sure the reuse register is in the + // right register class. For example, for x86 not all of the 32-bit + // registers have accessible sub-registers. + // Similarly so for EXTRACT_SUBREG. Consider this: + // EDI = op + // MOV32_mr fi#1, EDI + // ... + // = EXTRACT_SUBREG fi#1 + // fi#1 is available in EDI, but it cannot be reused because it's not in + // the right register file. + if (PhysReg && !AvoidReload && SubIdx) { + const TargetRegisterClass* RC = MRI->getRegClass(VirtReg); + if (!RC->contains(PhysReg)) + PhysReg = 0; + } + + if (PhysReg && !AvoidReload) { + // This spilled operand might be part of a two-address operand. If this + // is the case, then changing it will necessarily require changing the + // def part of the instruction as well. However, in some cases, we + // aren't allowed to modify the reused register. If none of these cases + // apply, reuse it. + bool CanReuse = true; + bool isTied = MI.isRegTiedToDefOperand(i); + if (isTied) { + // Okay, we have a two address operand. We can reuse this physreg as + // long as we are allowed to clobber the value and there isn't an + // earlier def that has already clobbered the physreg. + CanReuse = !ReusedOperands.isClobbered(PhysReg) && + Spills.canClobberPhysReg(PhysReg); + } + // If this is an asm, and a PhysReg alias is used elsewhere as an + // earlyclobber operand, we can't also use it as an input. + if (MI.isInlineAsm()) { + for (unsigned k = 0, e = MI.getNumOperands(); k != e; ++k) { + MachineOperand &MOk = MI.getOperand(k); + if (MOk.isReg() && MOk.isEarlyClobber() && + TRI->regsOverlap(MOk.getReg(), PhysReg)) { + CanReuse = false; + DEBUG(dbgs() << "Not reusing physreg " << TRI->getName(PhysReg) + << " for vreg" << VirtReg << ": " << MOk << '\n'); + break; + } + } + } + + if (CanReuse) { + // If this stack slot value is already available, reuse it! + if (ReuseSlot > VirtRegMap::MAX_STACK_SLOT) + DEBUG(dbgs() << "Reusing RM#" + << ReuseSlot-VirtRegMap::MAX_STACK_SLOT-1); + else + DEBUG(dbgs() << "Reusing SS#" << ReuseSlot); + DEBUG(dbgs() << " from physreg " + << TRI->getName(PhysReg) << " for vreg" + << VirtReg <<" instead of reloading into physreg " + << TRI->getName(VRM->getPhys(VirtReg)) << '\n'); + unsigned RReg = SubIdx ? TRI->getSubReg(PhysReg, SubIdx) : PhysReg; + MI.getOperand(i).setReg(RReg); + MI.getOperand(i).setSubReg(0); + + // The only technical detail we have is that we don't know that + // PhysReg won't be clobbered by a reloaded stack slot that occurs + // later in the instruction. In particular, consider 'op V1, V2'. + // If V1 is available in physreg R0, we would choose to reuse it + // here, instead of reloading it into the register the allocator + // indicated (say R1). However, V2 might have to be reloaded + // later, and it might indicate that it needs to live in R0. When + // this occurs, we need to have information available that + // indicates it is safe to use R1 for the reload instead of R0. + // + // To further complicate matters, we might conflict with an alias, + // or R0 and R1 might not be compatible with each other. In this + // case, we actually insert a reload for V1 in R1, ensuring that + // we can get at R0 or its alias. + ReusedOperands.addReuse(i, ReuseSlot, PhysReg, + VRM->getPhys(VirtReg), VirtReg); + if (isTied) + // Only mark it clobbered if this is a use&def operand. + ReusedOperands.markClobbered(PhysReg); + ++NumReused; + + if (MI.getOperand(i).isKill() && + ReuseSlot <= VirtRegMap::MAX_STACK_SLOT) { + + // The store of this spilled value is potentially dead, but we + // won't know for certain until we've confirmed that the re-use + // above is valid, which means waiting until the other operands + // are processed. For now we just track the spill slot, we'll + // remove it after the other operands are processed if valid. + + PotentialDeadStoreSlots.push_back(ReuseSlot); + } + + // Mark is isKill if it's there no other uses of the same virtual + // register and it's not a two-address operand. IsKill will be + // unset if reg is reused. + if (!isTied && KilledMIRegs.count(VirtReg) == 0) { + MI.getOperand(i).setIsKill(); + KilledMIRegs.insert(VirtReg); + } + + continue; + } // CanReuse + + // Otherwise we have a situation where we have a two-address instruction + // whose mod/ref operand needs to be reloaded. This reload is already + // available in some register "PhysReg", but if we used PhysReg as the + // operand to our 2-addr instruction, the instruction would modify + // PhysReg. This isn't cool if something later uses PhysReg and expects + // to get its initial value. + // + // To avoid this problem, and to avoid doing a load right after a store, + // we emit a copy from PhysReg into the designated register for this + // operand. + // + // This case also applies to an earlyclobber'd PhysReg. + unsigned DesignatedReg = VRM->getPhys(VirtReg); + assert(DesignatedReg && "Must map virtreg to physreg!"); + + // Note that, if we reused a register for a previous operand, the + // register we want to reload into might not actually be + // available. If this occurs, use the register indicated by the + // reuser. + if (ReusedOperands.hasReuses()) + DesignatedReg = ReusedOperands. + GetRegForReload(VirtReg, DesignatedReg, &MI, Spills, + MaybeDeadStores, RegKills, KillOps, *VRM); + + // If the mapped designated register is actually the physreg we have + // incoming, we don't need to inserted a dead copy. + if (DesignatedReg == PhysReg) { + // If this stack slot value is already available, reuse it! + if (ReuseSlot > VirtRegMap::MAX_STACK_SLOT) + DEBUG(dbgs() << "Reusing RM#" + << ReuseSlot-VirtRegMap::MAX_STACK_SLOT-1); + else + DEBUG(dbgs() << "Reusing SS#" << ReuseSlot); + DEBUG(dbgs() << " from physreg " << TRI->getName(PhysReg) + << " for vreg" << VirtReg + << " instead of reloading into same physreg.\n"); + unsigned RReg = SubIdx ? TRI->getSubReg(PhysReg, SubIdx) : PhysReg; + MI.getOperand(i).setReg(RReg); + MI.getOperand(i).setSubReg(0); + ReusedOperands.markClobbered(RReg); + ++NumReused; + continue; + } + + MRI->setPhysRegUsed(DesignatedReg); + ReusedOperands.markClobbered(DesignatedReg); + + // Back-schedule reloads and remats. + MachineBasicBlock::iterator InsertLoc = + ComputeReloadLoc(&MI, MBB->begin(), PhysReg, TRI, DoReMat, + SSorRMId, TII, *MBB->getParent()); + MachineInstr *CopyMI = BuildMI(*MBB, InsertLoc, MI.getDebugLoc(), + TII->get(TargetOpcode::COPY), + DesignatedReg).addReg(PhysReg); + CopyMI->setAsmPrinterFlag(MachineInstr::ReloadReuse); + UpdateKills(*CopyMI, TRI, RegKills, KillOps); + + // This invalidates DesignatedReg. + Spills.ClobberPhysReg(DesignatedReg); + + Spills.addAvailable(ReuseSlot, DesignatedReg); + unsigned RReg = + SubIdx ? TRI->getSubReg(DesignatedReg, SubIdx) : DesignatedReg; + MI.getOperand(i).setReg(RReg); + MI.getOperand(i).setSubReg(0); + DEBUG(dbgs() << '\t' << *prior(InsertLoc)); + ++NumReused; + continue; + } // if (PhysReg) + + // Otherwise, reload it and remember that we have it. + PhysReg = VRM->getPhys(VirtReg); + assert(PhysReg && "Must map virtreg to physreg!"); + + // Note that, if we reused a register for a previous operand, the + // register we want to reload into might not actually be + // available. If this occurs, use the register indicated by the + // reuser. + if (ReusedOperands.hasReuses()) + PhysReg = ReusedOperands.GetRegForReload(VirtReg, PhysReg, &MI, + Spills, MaybeDeadStores, RegKills, KillOps, *VRM); + + MRI->setPhysRegUsed(PhysReg); + ReusedOperands.markClobbered(PhysReg); + if (AvoidReload) + ++NumAvoided; + else { + // Back-schedule reloads and remats. + MachineBasicBlock::iterator InsertLoc = + ComputeReloadLoc(MI, MBB->begin(), PhysReg, TRI, DoReMat, + SSorRMId, TII, *MBB->getParent()); + + if (DoReMat) { + ReMaterialize(*MBB, InsertLoc, PhysReg, VirtReg, TII, TRI, *VRM); + } else { + const TargetRegisterClass* RC = MRI->getRegClass(VirtReg); + TII->loadRegFromStackSlot(*MBB, InsertLoc, PhysReg, SSorRMId, RC,TRI); + MachineInstr *LoadMI = prior(InsertLoc); + VRM->addSpillSlotUse(SSorRMId, LoadMI); + ++NumLoads; + DistanceMap.insert(std::make_pair(LoadMI, DistanceMap.size())); + } + // This invalidates PhysReg. + Spills.ClobberPhysReg(PhysReg); + + // Any stores to this stack slot are not dead anymore. + if (!DoReMat) + MaybeDeadStores[SSorRMId] = NULL; + Spills.addAvailable(SSorRMId, PhysReg); + // Assumes this is the last use. IsKill will be unset if reg is reused + // unless it's a two-address operand. + if (!MI.isRegTiedToDefOperand(i) && + KilledMIRegs.count(VirtReg) == 0) { + MI.getOperand(i).setIsKill(); + KilledMIRegs.insert(VirtReg); + } + + UpdateKills(*prior(InsertLoc), TRI, RegKills, KillOps); + DEBUG(dbgs() << '\t' << *prior(InsertLoc)); + } + unsigned RReg = SubIdx ? TRI->getSubReg(PhysReg, SubIdx) : PhysReg; + MI.getOperand(i).setReg(RReg); + MI.getOperand(i).setSubReg(0); + } + + // Ok - now we can remove stores that have been confirmed dead. + for (unsigned j = 0, e = PotentialDeadStoreSlots.size(); j != e; ++j) { + // This was the last use and the spilled value is still available + // for reuse. That means the spill was unnecessary! + int PDSSlot = PotentialDeadStoreSlots[j]; + MachineInstr* DeadStore = MaybeDeadStores[PDSSlot]; + if (DeadStore) { + DEBUG(dbgs() << "Removed dead store:\t" << *DeadStore); + InvalidateKills(*DeadStore, TRI, RegKills, KillOps); + VRM->RemoveMachineInstrFromMaps(DeadStore); + MBB->erase(DeadStore); + MaybeDeadStores[PDSSlot] = NULL; + ++NumDSE; + } + } + +} + /// rewriteMBB - Keep track of which spills are available even after the -/// register allocator is done with them. If possible, avid reloading vregs. +/// register allocator is done with them. If possible, avoid reloading vregs. void LocalRewriter::RewriteMBB(LiveIntervals *LIs, AvailableSpills &Spills, BitVector &RegKills, @@ -1891,8 +2228,10 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, // ReMatDefs - These are rematerializable def MIs which are not deleted. SmallSet ReMatDefs; - // Clear kill info. - SmallSet KilledMIRegs; + // Keep track of the registers we have already spilled in case there are + // multiple defs of the same register in MI. + SmallSet SpilledMIRegs; + RegKills.reset(); KillOps.clear(); KillOps.resize(TRI->getNumRegs(), NULL); @@ -1913,7 +2252,6 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, if (InsertSpills(MII)) NextMII = llvm::next(MII); - VirtRegMap::MI2VirtMapTy::const_iterator I, End; bool Erased = false; bool BackTracked = false; MachineInstr &MI = *MII; @@ -1925,309 +2263,8 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, /// ReusedOperands - Keep track of operand reuse in case we need to undo /// reuse. ReuseInfo ReusedOperands(MI, TRI); - SmallVector VirtUseOps; - for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { - MachineOperand &MO = MI.getOperand(i); - if (!MO.isReg() || MO.getReg() == 0) - continue; // Ignore non-register operands. - - unsigned VirtReg = MO.getReg(); - if (TargetRegisterInfo::isPhysicalRegister(VirtReg)) { - // Ignore physregs for spilling, but remember that it is used by this - // function. - MRI->setPhysRegUsed(VirtReg); - continue; - } - - // We want to process implicit virtual register uses first. - if (MO.isImplicit()) - // If the virtual register is implicitly defined, emit a implicit_def - // before so scavenger knows it's "defined". - // FIXME: This is a horrible hack done the by register allocator to - // remat a definition with virtual register operand. - VirtUseOps.insert(VirtUseOps.begin(), i); - else - VirtUseOps.push_back(i); - } - - // Process all of the spilled uses and all non spilled reg references. - SmallVector PotentialDeadStoreSlots; - KilledMIRegs.clear(); - for (unsigned j = 0, e = VirtUseOps.size(); j != e; ++j) { - unsigned i = VirtUseOps[j]; - unsigned VirtReg = MI.getOperand(i).getReg(); - assert(TargetRegisterInfo::isVirtualRegister(VirtReg) && - "Not a virtual register?"); - - unsigned SubIdx = MI.getOperand(i).getSubReg(); - if (VRM->isAssignedReg(VirtReg)) { - // This virtual register was assigned a physreg! - unsigned Phys = VRM->getPhys(VirtReg); - MRI->setPhysRegUsed(Phys); - if (MI.getOperand(i).isDef()) - ReusedOperands.markClobbered(Phys); - substitutePhysReg(MI.getOperand(i), Phys, *TRI); - if (VRM->isImplicitlyDefined(VirtReg)) - // FIXME: Is this needed? - BuildMI(*MBB, &MI, MI.getDebugLoc(), - TII->get(TargetOpcode::IMPLICIT_DEF), Phys); - continue; - } - - // This virtual register is now known to be a spilled value. - if (!MI.getOperand(i).isUse()) - continue; // Handle defs in the loop below (handle use&def here though) - - bool AvoidReload = MI.getOperand(i).isUndef(); - // Check if it is defined by an implicit def. It should not be spilled. - // Note, this is for correctness reason. e.g. - // 8 %reg1024 = IMPLICIT_DEF - // 12 %reg1024 = INSERT_SUBREG %reg1024, %reg1025, 2 - // The live range [12, 14) are not part of the r1024 live interval since - // it's defined by an implicit def. It will not conflicts with live - // interval of r1025. Now suppose both registers are spilled, you can - // easily see a situation where both registers are reloaded before - // the INSERT_SUBREG and both target registers that would overlap. - bool DoReMat = VRM->isReMaterialized(VirtReg); - int SSorRMId = DoReMat - ? VRM->getReMatId(VirtReg) : VRM->getStackSlot(VirtReg); - int ReuseSlot = SSorRMId; - - // Check to see if this stack slot is available. - unsigned PhysReg = Spills.getSpillSlotOrReMatPhysReg(SSorRMId); - - // If this is a sub-register use, make sure the reuse register is in the - // right register class. For example, for x86 not all of the 32-bit - // registers have accessible sub-registers. - // Similarly so for EXTRACT_SUBREG. Consider this: - // EDI = op - // MOV32_mr fi#1, EDI - // ... - // = EXTRACT_SUBREG fi#1 - // fi#1 is available in EDI, but it cannot be reused because it's not in - // the right register file. - if (PhysReg && !AvoidReload && (SubIdx || MI.isExtractSubreg())) { - const TargetRegisterClass* RC = MRI->getRegClass(VirtReg); - if (!RC->contains(PhysReg)) - PhysReg = 0; - } - - if (PhysReg && !AvoidReload) { - // This spilled operand might be part of a two-address operand. If this - // is the case, then changing it will necessarily require changing the - // def part of the instruction as well. However, in some cases, we - // aren't allowed to modify the reused register. If none of these cases - // apply, reuse it. - bool CanReuse = true; - bool isTied = MI.isRegTiedToDefOperand(i); - if (isTied) { - // Okay, we have a two address operand. We can reuse this physreg as - // long as we are allowed to clobber the value and there isn't an - // earlier def that has already clobbered the physreg. - CanReuse = !ReusedOperands.isClobbered(PhysReg) && - Spills.canClobberPhysReg(PhysReg); - } - - if (CanReuse) { - // If this stack slot value is already available, reuse it! - if (ReuseSlot > VirtRegMap::MAX_STACK_SLOT) - DEBUG(dbgs() << "Reusing RM#" - << ReuseSlot-VirtRegMap::MAX_STACK_SLOT-1); - else - DEBUG(dbgs() << "Reusing SS#" << ReuseSlot); - DEBUG(dbgs() << " from physreg " - << TRI->getName(PhysReg) << " for vreg" - << VirtReg <<" instead of reloading into physreg " - << TRI->getName(VRM->getPhys(VirtReg)) << '\n'); - unsigned RReg = SubIdx ? TRI->getSubReg(PhysReg, SubIdx) : PhysReg; - MI.getOperand(i).setReg(RReg); - MI.getOperand(i).setSubReg(0); - - // The only technical detail we have is that we don't know that - // PhysReg won't be clobbered by a reloaded stack slot that occurs - // later in the instruction. In particular, consider 'op V1, V2'. - // If V1 is available in physreg R0, we would choose to reuse it - // here, instead of reloading it into the register the allocator - // indicated (say R1). However, V2 might have to be reloaded - // later, and it might indicate that it needs to live in R0. When - // this occurs, we need to have information available that - // indicates it is safe to use R1 for the reload instead of R0. - // - // To further complicate matters, we might conflict with an alias, - // or R0 and R1 might not be compatible with each other. In this - // case, we actually insert a reload for V1 in R1, ensuring that - // we can get at R0 or its alias. - ReusedOperands.addReuse(i, ReuseSlot, PhysReg, - VRM->getPhys(VirtReg), VirtReg); - if (isTied) - // Only mark it clobbered if this is a use&def operand. - ReusedOperands.markClobbered(PhysReg); - ++NumReused; - - if (MI.getOperand(i).isKill() && - ReuseSlot <= VirtRegMap::MAX_STACK_SLOT) { - - // The store of this spilled value is potentially dead, but we - // won't know for certain until we've confirmed that the re-use - // above is valid, which means waiting until the other operands - // are processed. For now we just track the spill slot, we'll - // remove it after the other operands are processed if valid. - - PotentialDeadStoreSlots.push_back(ReuseSlot); - } - - // Mark is isKill if it's there no other uses of the same virtual - // register and it's not a two-address operand. IsKill will be - // unset if reg is reused. - if (!isTied && KilledMIRegs.count(VirtReg) == 0) { - MI.getOperand(i).setIsKill(); - KilledMIRegs.insert(VirtReg); - } - - continue; - } // CanReuse - - // Otherwise we have a situation where we have a two-address instruction - // whose mod/ref operand needs to be reloaded. This reload is already - // available in some register "PhysReg", but if we used PhysReg as the - // operand to our 2-addr instruction, the instruction would modify - // PhysReg. This isn't cool if something later uses PhysReg and expects - // to get its initial value. - // - // To avoid this problem, and to avoid doing a load right after a store, - // we emit a copy from PhysReg into the designated register for this - // operand. - unsigned DesignatedReg = VRM->getPhys(VirtReg); - assert(DesignatedReg && "Must map virtreg to physreg!"); - - // Note that, if we reused a register for a previous operand, the - // register we want to reload into might not actually be - // available. If this occurs, use the register indicated by the - // reuser. - if (ReusedOperands.hasReuses()) - DesignatedReg = ReusedOperands. - GetRegForReload(VirtReg, DesignatedReg, &MI, Spills, - MaybeDeadStores, RegKills, KillOps, *VRM); - - // If the mapped designated register is actually the physreg we have - // incoming, we don't need to inserted a dead copy. - if (DesignatedReg == PhysReg) { - // If this stack slot value is already available, reuse it! - if (ReuseSlot > VirtRegMap::MAX_STACK_SLOT) - DEBUG(dbgs() << "Reusing RM#" - << ReuseSlot-VirtRegMap::MAX_STACK_SLOT-1); - else - DEBUG(dbgs() << "Reusing SS#" << ReuseSlot); - DEBUG(dbgs() << " from physreg " << TRI->getName(PhysReg) - << " for vreg" << VirtReg - << " instead of reloading into same physreg.\n"); - unsigned RReg = SubIdx ? TRI->getSubReg(PhysReg, SubIdx) : PhysReg; - MI.getOperand(i).setReg(RReg); - MI.getOperand(i).setSubReg(0); - ReusedOperands.markClobbered(RReg); - ++NumReused; - continue; - } - - const TargetRegisterClass* RC = MRI->getRegClass(VirtReg); - MRI->setPhysRegUsed(DesignatedReg); - ReusedOperands.markClobbered(DesignatedReg); - - // Back-schedule reloads and remats. - MachineBasicBlock::iterator InsertLoc = - ComputeReloadLoc(&MI, MBB->begin(), PhysReg, TRI, DoReMat, - SSorRMId, TII, MF); - - TII->copyRegToReg(*MBB, InsertLoc, DesignatedReg, PhysReg, RC, RC); - - MachineInstr *CopyMI = prior(InsertLoc); - CopyMI->setAsmPrinterFlag(MachineInstr::ReloadReuse); - UpdateKills(*CopyMI, TRI, RegKills, KillOps); - - // This invalidates DesignatedReg. - Spills.ClobberPhysReg(DesignatedReg); - - Spills.addAvailable(ReuseSlot, DesignatedReg); - unsigned RReg = - SubIdx ? TRI->getSubReg(DesignatedReg, SubIdx) : DesignatedReg; - MI.getOperand(i).setReg(RReg); - MI.getOperand(i).setSubReg(0); - DEBUG(dbgs() << '\t' << *prior(MII)); - ++NumReused; - continue; - } // if (PhysReg) - - // Otherwise, reload it and remember that we have it. - PhysReg = VRM->getPhys(VirtReg); - assert(PhysReg && "Must map virtreg to physreg!"); - - // Note that, if we reused a register for a previous operand, the - // register we want to reload into might not actually be - // available. If this occurs, use the register indicated by the - // reuser. - if (ReusedOperands.hasReuses()) - PhysReg = ReusedOperands.GetRegForReload(VirtReg, PhysReg, &MI, - Spills, MaybeDeadStores, RegKills, KillOps, *VRM); - - MRI->setPhysRegUsed(PhysReg); - ReusedOperands.markClobbered(PhysReg); - if (AvoidReload) - ++NumAvoided; - else { - // Back-schedule reloads and remats. - MachineBasicBlock::iterator InsertLoc = - ComputeReloadLoc(MII, MBB->begin(), PhysReg, TRI, DoReMat, - SSorRMId, TII, MF); - - if (DoReMat) { - ReMaterialize(*MBB, InsertLoc, PhysReg, VirtReg, TII, TRI, *VRM); - } else { - const TargetRegisterClass* RC = MRI->getRegClass(VirtReg); - TII->loadRegFromStackSlot(*MBB, InsertLoc, PhysReg, SSorRMId, RC); - MachineInstr *LoadMI = prior(InsertLoc); - VRM->addSpillSlotUse(SSorRMId, LoadMI); - ++NumLoads; - DistanceMap.insert(std::make_pair(LoadMI, DistanceMap.size())); - } - // This invalidates PhysReg. - Spills.ClobberPhysReg(PhysReg); - - // Any stores to this stack slot are not dead anymore. - if (!DoReMat) - MaybeDeadStores[SSorRMId] = NULL; - Spills.addAvailable(SSorRMId, PhysReg); - // Assumes this is the last use. IsKill will be unset if reg is reused - // unless it's a two-address operand. - if (!MI.isRegTiedToDefOperand(i) && - KilledMIRegs.count(VirtReg) == 0) { - MI.getOperand(i).setIsKill(); - KilledMIRegs.insert(VirtReg); - } - - UpdateKills(*prior(InsertLoc), TRI, RegKills, KillOps); - DEBUG(dbgs() << '\t' << *prior(InsertLoc)); - } - unsigned RReg = SubIdx ? TRI->getSubReg(PhysReg, SubIdx) : PhysReg; - MI.getOperand(i).setReg(RReg); - MI.getOperand(i).setSubReg(0); - } - - // Ok - now we can remove stores that have been confirmed dead. - for (unsigned j = 0, e = PotentialDeadStoreSlots.size(); j != e; ++j) { - // This was the last use and the spilled value is still available - // for reuse. That means the spill was unnecessary! - int PDSSlot = PotentialDeadStoreSlots[j]; - MachineInstr* DeadStore = MaybeDeadStores[PDSSlot]; - if (DeadStore) { - DEBUG(dbgs() << "Removed dead store:\t" << *DeadStore); - InvalidateKills(*DeadStore, TRI, RegKills, KillOps); - VRM->RemoveMachineInstrFromMaps(DeadStore); - MBB->erase(DeadStore); - MaybeDeadStores[PDSSlot] = NULL; - ++NumDSE; - } - } + ProcessUses(MI, Spills, MaybeDeadStores, RegKills, ReusedOperands, KillOps); DEBUG(dbgs() << '\t' << MI); @@ -2235,15 +2272,22 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, // If we have folded references to memory operands, make sure we clear all // physical registers that may contain the value of the spilled virtual // register + + // Copy the folded virts to a small vector, we may change MI2VirtMap. + SmallVector, 4> FoldedVirts; + // C++0x FTW! + for (std::pair FVRange = + VRM->getFoldedVirts(&MI); + FVRange.first != FVRange.second; ++FVRange.first) + FoldedVirts.push_back(FVRange.first->second); + SmallSet FoldedSS; - for (tie(I, End) = VRM->getFoldedVirts(&MI); I != End; ) { - unsigned VirtReg = I->second.first; - VirtRegMap::ModRef MR = I->second.second; + for (unsigned FVI = 0, FVE = FoldedVirts.size(); FVI != FVE; ++FVI) { + unsigned VirtReg = FoldedVirts[FVI].first; + VirtRegMap::ModRef MR = FoldedVirts[FVI].second; DEBUG(dbgs() << "Folded vreg: " << VirtReg << " MR: " << MR); - // MI2VirtMap be can updated which invalidate the iterator. - // Increment the iterator first. - ++I; int SS = VRM->getStackSlot(VirtReg); if (SS == VirtRegMap::NO_STACK_SLOT) continue; @@ -2261,26 +2305,16 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, if (unsigned InReg = Spills.getSpillSlotOrReMatPhysReg(SS)) { DEBUG(dbgs() << "Promoted Load To Copy: " << MI); if (DestReg != InReg) { - const TargetRegisterClass *RC = MRI->getRegClass(VirtReg); - TII->copyRegToReg(*MBB, &MI, DestReg, InReg, RC, RC); MachineOperand *DefMO = MI.findRegisterDefOperand(DestReg); - unsigned SubIdx = DefMO->getSubReg(); + MachineInstr *CopyMI = BuildMI(*MBB, &MI, MI.getDebugLoc(), + TII->get(TargetOpcode::COPY)) + .addReg(DestReg, RegState::Define, DefMO->getSubReg()) + .addReg(InReg, RegState::Kill); // Revisit the copy so we make sure to notice the effects of the // operation on the destreg (either needing to RA it if it's // virtual or needing to clobber any values if it's physical). - NextMII = &MI; - --NextMII; // backtrack to the copy. + NextMII = CopyMI; NextMII->setAsmPrinterFlag(MachineInstr::ReloadReuse); - // Propagate the sub-register index over. - if (SubIdx) { - DefMO = NextMII->findRegisterDefOperand(DestReg); - DefMO->setSubReg(SubIdx); - } - - // Mark is killed. - MachineOperand *KillOpnd = NextMII->findRegisterUseOperand(InReg); - KillOpnd->setIsKill(); - BackTracked = true; } else { DEBUG(dbgs() << "Removing now-noop copy: " << MI); @@ -2299,7 +2333,7 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, unsigned PhysReg = Spills.getSpillSlotOrReMatPhysReg(SS); SmallVector NewMIs; if (PhysReg && - TII->unfoldMemoryOperand(MF, &MI, PhysReg, false, false, NewMIs)) { + TII->unfoldMemoryOperand(MF, &MI, PhysReg, false, false, NewMIs)){ MBB->insert(MII, NewMIs[0]); InvalidateKills(MI, TRI, RegKills, KillOps); VRM->RemoveMachineInstrFromMaps(&MI); @@ -2408,6 +2442,7 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, } // Process all of the spilled defs. + SpilledMIRegs.clear(); for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { MachineOperand &MO = MI.getOperand(i); if (!(MO.isReg() && MO.getReg() && MO.isDef())) @@ -2420,20 +2455,17 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, // Also check if it's copying from an "undef", if so, we can't // eliminate this or else the undef marker is lost and it will // confuses the scavenger. This is extremely rare. - unsigned Src, Dst, SrcSR, DstSR; - if (TII->isMoveInstr(MI, Src, Dst, SrcSR, DstSR) && Src == Dst && - !MI.findRegisterUseOperand(Src)->isUndef()) { + if (MI.isIdentityCopy() && !MI.getOperand(1).isUndef() && + MI.getNumOperands() == 2) { ++NumDCE; DEBUG(dbgs() << "Removing now-noop copy: " << MI); SmallVector KillRegs; InvalidateKills(MI, TRI, RegKills, KillOps, &KillRegs); if (MO.isDead() && !KillRegs.empty()) { // Source register or an implicit super/sub-register use is killed. - assert(KillRegs[0] == Dst || - TRI->isSubRegister(KillRegs[0], Dst) || - TRI->isSuperRegister(KillRegs[0], Dst)); + assert(TRI->regsOverlap(KillRegs[0], MI.getOperand(0).getReg())); // Last def is now dead. - TransferDeadness(Src, RegKills, KillOps); + TransferDeadness(MI.getOperand(1).getReg(), RegKills, KillOps); } VRM->RemoveMachineInstrFromMaps(&MI); MBB->erase(&MI); @@ -2500,7 +2532,7 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, MI.getOperand(i).setReg(RReg); MI.getOperand(i).setSubReg(0); - if (!MO.isDead()) { + if (!MO.isDead() && SpilledMIRegs.insert(VirtReg)) { MachineInstr *&LastStore = MaybeDeadStores[StackSlot]; SpillRegToStackSlot(MII, -1, PhysReg, StackSlot, RC, true, LastStore, Spills, ReMatDefs, RegKills, KillOps); @@ -2508,18 +2540,15 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, // Check to see if this is a noop copy. If so, eliminate the // instruction before considering the dest reg to be changed. - { - unsigned Src, Dst, SrcSR, DstSR; - if (TII->isMoveInstr(MI, Src, Dst, SrcSR, DstSR) && Src == Dst) { - ++NumDCE; - DEBUG(dbgs() << "Removing now-noop copy: " << MI); - InvalidateKills(MI, TRI, RegKills, KillOps); - VRM->RemoveMachineInstrFromMaps(&MI); - MBB->erase(&MI); - Erased = true; - UpdateKills(*LastStore, TRI, RegKills, KillOps); - goto ProcessNextInst; - } + if (MI.isIdentityCopy()) { + ++NumDCE; + DEBUG(dbgs() << "Removing now-noop copy: " << MI); + InvalidateKills(MI, TRI, RegKills, KillOps); + VRM->RemoveMachineInstrFromMaps(&MI); + MBB->erase(&MI); + Erased = true; + UpdateKills(*LastStore, TRI, RegKills, KillOps); + goto ProcessNextInst; } } }