X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FInlineSpiller.cpp;h=b63611b94c1c1d403d38d9160a6dac9fe77d837e;hb=cccfd194ecf8e1c6fb203ec3f96aca8cfe9e0484;hp=f56cb4822ec874c339762595c4319f2b57f85420;hpb=c1d22d8adbd40c3e5d704fdae90f9ed2089bb67e;p=oota-llvm.git diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp index f56cb4822ec..b63611b94c1 100644 --- a/lib/CodeGen/InlineSpiller.cpp +++ b/lib/CodeGen/InlineSpiller.cpp @@ -16,6 +16,8 @@ #include "Spiller.h" #include "LiveRangeEdit.h" #include "VirtRegMap.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/TinyPtrVector.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveStackAnalysis.h" @@ -26,11 +28,27 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; +STATISTIC(NumSpilledRanges, "Number of spilled live ranges"); +STATISTIC(NumSnippets, "Number of spilled snippets"); +STATISTIC(NumSpills, "Number of spills inserted"); +STATISTIC(NumSpillsRemoved, "Number of spills removed"); +STATISTIC(NumReloads, "Number of reloads inserted"); +STATISTIC(NumReloadsRemoved, "Number of reloads removed"); +STATISTIC(NumFolded, "Number of folded stack accesses"); +STATISTIC(NumFoldedLoads, "Number of folded loads"); +STATISTIC(NumRemats, "Number of rematerialized defs for spilling"); +STATISTIC(NumOmitReloadSpill, "Number of omitted spills of reloads"); +STATISTIC(NumHoists, "Number of hoisted spills"); + +static cl::opt DisableHoisting("disable-spill-hoist", cl::Hidden, + cl::desc("Disable inline spill hoisting")); + namespace { class InlineSpiller : public Spiller { MachineFunctionPass &Pass; @@ -62,26 +80,49 @@ class InlineSpiller : public Spiller { // Values that failed to remat at some point. SmallPtrSet UsedValues; +public: // Information about a value that was defined by a copy from a sibling // register. struct SibValueInfo { // True when all reaching defs were reloads: No spill is necessary. bool AllDefsAreReloads; + // True when value is defined by an original PHI not from splitting. + bool DefByOrigPHI; + + // True when the COPY defining this value killed its source. + bool KillsSource; + // The preferred register to spill. unsigned SpillReg; // The value of SpillReg that should be spilled. VNInfo *SpillVNI; + // The block where SpillVNI should be spilled. Currently, this must be the + // block containing SpillVNI->def. + MachineBasicBlock *SpillMBB; + // A defining instruction that is not a sibling copy or a reload, or NULL. // This can be used as a template for rematerialization. MachineInstr *DefMI; + // List of values that depend on this one. These values are actually the + // same, but live range splitting has placed them in different registers, + // or SSA update needed to insert PHI-defs to preserve SSA form. This is + // copies of the current value and phi-kills. Usually only phi-kills cause + // more than one dependent value. + TinyPtrVector Deps; + SibValueInfo(unsigned Reg, VNInfo *VNI) - : AllDefsAreReloads(false), SpillReg(Reg), SpillVNI(VNI), DefMI(0) {} + : AllDefsAreReloads(true), DefByOrigPHI(false), KillsSource(false), + SpillReg(Reg), SpillVNI(VNI), SpillMBB(0), DefMI(0) {} + + // Returns true when a def has been found. + bool hasDef() const { return DefByOrigPHI || DefMI; } }; +private: // Values in RegsToSpill defined by sibling copies. typedef DenseMap SibValueMap; SibValueMap SibValues; @@ -121,6 +162,7 @@ private: bool isSibling(unsigned Reg); MachineInstr *traceSiblingValue(unsigned, VNInfo*, VNInfo*); + void propagateSiblingValue(SibValueMap::iterator, VNInfo *VNI = 0); void analyzeSiblingValues(); bool hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI); @@ -134,11 +176,13 @@ private: bool foldMemoryOperand(MachineBasicBlock::iterator MI, const SmallVectorImpl &Ops, MachineInstr *LoadMI = 0); - void insertReload(LiveInterval &NewLI, MachineBasicBlock::iterator MI); + void insertReload(LiveInterval &NewLI, SlotIndex, + MachineBasicBlock::iterator MI); void insertSpill(LiveInterval &NewLI, const LiveInterval &OldLI, - MachineBasicBlock::iterator MI); + SlotIndex, MachineBasicBlock::iterator MI); void spillAroundUses(unsigned Reg); + void spillAll(); }; } @@ -165,11 +209,7 @@ Spiller *createInlineSpiller(MachineFunctionPass &pass, /// isFullCopyOf - If MI is a COPY to or from Reg, return the other register, /// otherwise return 0. static unsigned isFullCopyOf(const MachineInstr *MI, unsigned Reg) { - if (!MI->isCopy()) - return 0; - if (MI->getOperand(0).getSubReg() != 0) - return 0; - if (MI->getOperand(1).getSubReg() != 0) + if (!MI->isFullCopy()) return 0; if (MI->getOperand(0).getReg() == Reg) return MI->getOperand(1).getReg(); @@ -245,10 +285,11 @@ void InlineSpiller::collectRegsToSpill() { if (!isSnippet(SnipLI)) continue; SnippetCopies.insert(MI); - if (!isRegToSpill(SnipReg)) - RegsToSpill.push_back(SnipReg); - + if (isRegToSpill(SnipReg)) + continue; + RegsToSpill.push_back(SnipReg); DEBUG(dbgs() << "\talso spill snippet " << SnipLI << '\n'); + ++NumSnippets; } } @@ -270,6 +311,156 @@ bool InlineSpiller::isSibling(unsigned Reg) { VRM.getOriginal(Reg) == Original; } +#ifndef NDEBUG +static raw_ostream &operator<<(raw_ostream &OS, + const InlineSpiller::SibValueInfo &SVI) { + OS << "spill " << PrintReg(SVI.SpillReg) << ':' + << SVI.SpillVNI->id << '@' << SVI.SpillVNI->def; + if (SVI.SpillMBB) + OS << " in BB#" << SVI.SpillMBB->getNumber(); + if (SVI.AllDefsAreReloads) + OS << " all-reloads"; + if (SVI.DefByOrigPHI) + OS << " orig-phi"; + if (SVI.KillsSource) + OS << " kill"; + OS << " deps["; + for (unsigned i = 0, e = SVI.Deps.size(); i != e; ++i) + OS << ' ' << SVI.Deps[i]->id << '@' << SVI.Deps[i]->def; + OS << " ]"; + if (SVI.DefMI) + OS << " def: " << *SVI.DefMI; + else + OS << '\n'; + return OS; +} +#endif + +/// propagateSiblingValue - Propagate the value in SVI to dependents if it is +/// known. Otherwise remember the dependency for later. +/// +/// @param SVI SibValues entry to propagate. +/// @param VNI Dependent value, or NULL to propagate to all saved dependents. +void InlineSpiller::propagateSiblingValue(SibValueMap::iterator SVI, + VNInfo *VNI) { + // When VNI is non-NULL, add it to SVI's deps, and only propagate to that. + TinyPtrVector FirstDeps; + if (VNI) { + FirstDeps.push_back(VNI); + SVI->second.Deps.push_back(VNI); + } + + // Has the value been completely determined yet? If not, defer propagation. + if (!SVI->second.hasDef()) + return; + + // Work list of values to propagate. It would be nice to use a SetVector + // here, but then we would be forced to use a SmallSet. + SmallVector WorkList(1, SVI); + SmallPtrSet WorkSet; + + do { + SVI = WorkList.pop_back_val(); + WorkSet.erase(SVI->first); + TinyPtrVector *Deps = VNI ? &FirstDeps : &SVI->second.Deps; + VNI = 0; + + SibValueInfo &SV = SVI->second; + if (!SV.SpillMBB) + SV.SpillMBB = LIS.getMBBFromIndex(SV.SpillVNI->def); + + DEBUG(dbgs() << " prop to " << Deps->size() << ": " + << SVI->first->id << '@' << SVI->first->def << ":\t" << SV); + + assert(SV.hasDef() && "Propagating undefined value"); + + // Should this value be propagated as a preferred spill candidate? We don't + // propagate values of registers that are about to spill. + bool PropSpill = !DisableHoisting && !isRegToSpill(SV.SpillReg); + unsigned SpillDepth = ~0u; + + for (TinyPtrVector::iterator DepI = Deps->begin(), + DepE = Deps->end(); DepI != DepE; ++DepI) { + SibValueMap::iterator DepSVI = SibValues.find(*DepI); + assert(DepSVI != SibValues.end() && "Dependent value not in SibValues"); + SibValueInfo &DepSV = DepSVI->second; + if (!DepSV.SpillMBB) + DepSV.SpillMBB = LIS.getMBBFromIndex(DepSV.SpillVNI->def); + + bool Changed = false; + + // Propagate defining instruction. + if (!DepSV.hasDef()) { + Changed = true; + DepSV.DefMI = SV.DefMI; + DepSV.DefByOrigPHI = SV.DefByOrigPHI; + } + + // Propagate AllDefsAreReloads. For PHI values, this computes an AND of + // all predecessors. + if (!SV.AllDefsAreReloads && DepSV.AllDefsAreReloads) { + Changed = true; + DepSV.AllDefsAreReloads = false; + } + + // Propagate best spill value. + if (PropSpill && SV.SpillVNI != DepSV.SpillVNI) { + if (SV.SpillMBB == DepSV.SpillMBB) { + // DepSV is in the same block. Hoist when dominated. + if (DepSV.KillsSource && SV.SpillVNI->def < DepSV.SpillVNI->def) { + // This is an alternative def earlier in the same MBB. + // Hoist the spill as far as possible in SpillMBB. This can ease + // register pressure: + // + // x = def + // y = use x + // s = copy x + // + // Hoisting the spill of s to immediately after the def removes the + // interference between x and y: + // + // x = def + // spill x + // y = use x + // + // This hoist only helps when the DepSV copy kills its source. + Changed = true; + DepSV.SpillReg = SV.SpillReg; + DepSV.SpillVNI = SV.SpillVNI; + DepSV.SpillMBB = SV.SpillMBB; + } + } else { + // DepSV is in a different block. + if (SpillDepth == ~0u) + SpillDepth = Loops.getLoopDepth(SV.SpillMBB); + + // Also hoist spills to blocks with smaller loop depth, but make sure + // that the new value dominates. Non-phi dependents are always + // dominated, phis need checking. + if ((Loops.getLoopDepth(DepSV.SpillMBB) > SpillDepth) && + (!DepSVI->first->isPHIDef() || + MDT.dominates(SV.SpillMBB, DepSV.SpillMBB))) { + Changed = true; + DepSV.SpillReg = SV.SpillReg; + DepSV.SpillVNI = SV.SpillVNI; + DepSV.SpillMBB = SV.SpillMBB; + } + } + } + + if (!Changed) + continue; + + // Something changed in DepSVI. Propagate to dependents. + if (WorkSet.insert(DepSVI->first)) + WorkList.push_back(DepSVI); + + DEBUG(dbgs() << " update " << DepSVI->first->id << '@' + << DepSVI->first->def << " to:\t" << DepSV); + } + } while (!WorkList.empty()); +} + /// traceSiblingValue - Trace a value that is about to be spilled back to the /// real defining instructions by looking through sibling copies. Always stay /// within the range of OrigVNI so the registers are known to carry the same @@ -282,59 +473,101 @@ bool InlineSpiller::isSibling(unsigned Reg) { /// MachineInstr *InlineSpiller::traceSiblingValue(unsigned UseReg, VNInfo *UseVNI, VNInfo *OrigVNI) { + // Check if a cached value already exists. + SibValueMap::iterator SVI; + bool Inserted; + tie(SVI, Inserted) = + SibValues.insert(std::make_pair(UseVNI, SibValueInfo(UseReg, UseVNI))); + if (!Inserted) { + DEBUG(dbgs() << "Cached value " << PrintReg(UseReg) << ':' + << UseVNI->id << '@' << UseVNI->def << ' ' << SVI->second); + return SVI->second.DefMI; + } + DEBUG(dbgs() << "Tracing value " << PrintReg(UseReg) << ':' << UseVNI->id << '@' << UseVNI->def << '\n'); - SmallPtrSet Visited; + + // List of (Reg, VNI) that have been inserted into SibValues, but need to be + // processed. SmallVector, 8> WorkList; WorkList.push_back(std::make_pair(UseReg, UseVNI)); - // Best spill candidate seen so far. This must dominate UseVNI. - SibValueInfo SVI(UseReg, UseVNI); - MachineBasicBlock *UseMBB = LIS.getMBBFromIndex(UseVNI->def); - unsigned SpillDepth = Loops.getLoopDepth(UseMBB); - bool SeenOrigPHI = false; // Original PHI met. - do { unsigned Reg; VNInfo *VNI; tie(Reg, VNI) = WorkList.pop_back_val(); - if (!Visited.insert(VNI)) - continue; + DEBUG(dbgs() << " " << PrintReg(Reg) << ':' << VNI->id << '@' << VNI->def + << ":\t"); - // Is this value a better spill candidate? - if (!isRegToSpill(Reg)) { - MachineBasicBlock *MBB = LIS.getMBBFromIndex(VNI->def); - if (MBB != UseMBB && MDT.dominates(MBB, UseMBB)) { - // This is a valid spill location dominating UseVNI. - // Prefer to spill at a smaller loop depth. - unsigned Depth = Loops.getLoopDepth(MBB); - if (Depth < SpillDepth) { - DEBUG(dbgs() << " spill depth " << Depth << ": " << PrintReg(Reg) - << ':' << VNI->id << '@' << VNI->def << '\n'); - SVI.SpillReg = Reg; - SVI.SpillVNI = VNI; - SpillDepth = Depth; - } - } - } + // First check if this value has already been computed. + SVI = SibValues.find(VNI); + assert(SVI != SibValues.end() && "Missing SibValues entry"); // Trace through PHI-defs created by live range splitting. if (VNI->isPHIDef()) { + // Stop at original PHIs. We don't know the value at the predecessors. if (VNI->def == OrigVNI->def) { - DEBUG(dbgs() << " orig phi value " << PrintReg(Reg) << ':' - << VNI->id << '@' << VNI->def << '\n'); - SeenOrigPHI = true; + DEBUG(dbgs() << "orig phi value\n"); + SVI->second.DefByOrigPHI = true; + SVI->second.AllDefsAreReloads = false; + propagateSiblingValue(SVI); continue; } - // Get values live-out of predecessors. + + // This is a PHI inserted by live range splitting. We could trace the + // live-out value from predecessor blocks, but that search can be very + // expensive if there are many predecessors and many more PHIs as + // generated by tail-dup when it sees an indirectbr. Instead, look at + // all the non-PHI defs that have the same value as OrigVNI. They must + // jointly dominate VNI->def. This is not optimal since VNI may actually + // be jointly dominated by a smaller subset of defs, so there is a change + // we will miss a AllDefsAreReloads optimization. + + // Separate all values dominated by OrigVNI into PHIs and non-PHIs. + SmallVector PHIs, NonPHIs; LiveInterval &LI = LIS.getInterval(Reg); - MachineBasicBlock *MBB = LIS.getMBBFromIndex(VNI->def); - for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), - PE = MBB->pred_end(); PI != PE; ++PI) { - VNInfo *PVNI = LI.getVNInfoAt(LIS.getMBBEndIdx(*PI).getPrevSlot()); - if (PVNI) - WorkList.push_back(std::make_pair(Reg, PVNI)); + LiveInterval &OrigLI = LIS.getInterval(Original); + + for (LiveInterval::vni_iterator VI = LI.vni_begin(), VE = LI.vni_end(); + VI != VE; ++VI) { + VNInfo *VNI2 = *VI; + if (VNI2->isUnused()) + continue; + if (!OrigLI.containsOneValue() && + OrigLI.getVNInfoAt(VNI2->def) != OrigVNI) + continue; + if (VNI2->isPHIDef() && VNI2->def != OrigVNI->def) + PHIs.push_back(VNI2); + else + NonPHIs.push_back(VNI2); } + DEBUG(dbgs() << "split phi value, checking " << PHIs.size() + << " phi-defs, and " << NonPHIs.size() + << " non-phi/orig defs\n"); + + // Create entries for all the PHIs. Don't add them to the worklist, we + // are processing all of them in one go here. + for (unsigned i = 0, e = PHIs.size(); i != e; ++i) + SibValues.insert(std::make_pair(PHIs[i], SibValueInfo(Reg, PHIs[i]))); + + // Add every PHI as a dependent of all the non-PHIs. + for (unsigned i = 0, e = NonPHIs.size(); i != e; ++i) { + VNInfo *NonPHI = NonPHIs[i]; + // Known value? Try an insertion. + tie(SVI, Inserted) = + SibValues.insert(std::make_pair(NonPHI, SibValueInfo(Reg, NonPHI))); + // Add all the PHIs as dependents of NonPHI. + for (unsigned pi = 0, pe = PHIs.size(); pi != pe; ++pi) + SVI->second.Deps.push_back(PHIs[pi]); + // This is the first time we see NonPHI, add it to the worklist. + if (Inserted) + WorkList.push_back(std::make_pair(Reg, NonPHI)); + else + // Propagate to all inserted PHIs, not just VNI. + propagateSiblingValue(SVI); + } + + // Next work list item. continue; } @@ -345,48 +578,49 @@ MachineInstr *InlineSpiller::traceSiblingValue(unsigned UseReg, VNInfo *UseVNI, if (unsigned SrcReg = isFullCopyOf(MI, Reg)) { if (isSibling(SrcReg)) { LiveInterval &SrcLI = LIS.getInterval(SrcReg); - VNInfo *SrcVNI = SrcLI.getVNInfoAt(VNI->def.getUseIndex()); - assert(SrcVNI && "Copy from non-existing value"); - DEBUG(dbgs() << " copy of " << PrintReg(SrcReg) << ':' - << SrcVNI->id << '@' << SrcVNI->def << '\n'); - WorkList.push_back(std::make_pair(SrcReg, SrcVNI)); + LiveRange *SrcLR = SrcLI.getLiveRangeContaining(VNI->def.getUseIndex()); + assert(SrcLR && "Copy from non-existing value"); + // Check if this COPY kills its source. + SVI->second.KillsSource = (SrcLR->end == VNI->def); + VNInfo *SrcVNI = SrcLR->valno; + DEBUG(dbgs() << "copy of " << PrintReg(SrcReg) << ':' + << SrcVNI->id << '@' << SrcVNI->def + << " kill=" << unsigned(SVI->second.KillsSource) << '\n'); + // Known sibling source value? Try an insertion. + tie(SVI, Inserted) = SibValues.insert(std::make_pair(SrcVNI, + SibValueInfo(SrcReg, SrcVNI))); + // This is the first time we see Src, add it to the worklist. + if (Inserted) + WorkList.push_back(std::make_pair(SrcReg, SrcVNI)); + propagateSiblingValue(SVI, VNI); + // Next work list item. continue; } } // Track reachable reloads. + SVI->second.DefMI = MI; + SVI->second.SpillMBB = MI->getParent(); int FI; if (Reg == TII.isLoadFromStackSlot(MI, FI) && FI == StackSlot) { - DEBUG(dbgs() << " reload " << PrintReg(Reg) << ':' - << VNI->id << "@" << VNI->def << '\n'); - SVI.AllDefsAreReloads = true; + DEBUG(dbgs() << "reload\n"); + propagateSiblingValue(SVI); + // Next work list item. continue; } - // We have an 'original' def. Don't record trivial cases. - if (VNI == UseVNI) { - DEBUG(dbgs() << "Not a sibling copy.\n"); - return MI; - } - // Potential remat candidate. - DEBUG(dbgs() << " def " << PrintReg(Reg) << ':' - << VNI->id << '@' << VNI->def << '\t' << *MI); - SVI.DefMI = MI; + DEBUG(dbgs() << "def " << *MI); + SVI->second.AllDefsAreReloads = false; + propagateSiblingValue(SVI); } while (!WorkList.empty()); - if (SeenOrigPHI || SVI.DefMI) - SVI.AllDefsAreReloads = false; - - DEBUG({ - if (SVI.AllDefsAreReloads) - dbgs() << "All defs are reloads.\n"; - else - dbgs() << "Prefer to spill " << PrintReg(SVI.SpillReg) << ':' - << SVI.SpillVNI->id << '@' << SVI.SpillVNI->def << '\n'; - }); - SibValues.insert(std::make_pair(UseVNI, SVI)); - return SVI.DefMI; + // Look up the value we were looking for. We already did this lokup at the + // top of the function, but SibValues may have been invalidated. + SVI = SibValues.find(UseVNI); + assert(SVI != SibValues.end() && "Didn't compute requested info"); + DEBUG(dbgs() << " traced to:\t" << SVI->second); + return SVI->second.DefMI; } /// analyzeSiblingValues - Trace values defined by sibling copies back to @@ -413,13 +647,16 @@ void InlineSpiller::analyzeSiblingValues() { // Check possible sibling copies. if (VNI->isPHIDef() || VNI->getCopy()) { VNInfo *OrigVNI = OrigLI.getVNInfoAt(VNI->def); + assert(OrigVNI && "Def outside original live range"); if (OrigVNI->def != VNI->def) DefMI = traceSiblingValue(Reg, VNI, OrigVNI); } if (!DefMI && !VNI->isPHIDef()) DefMI = LIS.getInstructionFromIndex(VNI->def); - if (DefMI) - Edit->checkRematerializable(VNI, DefMI, TII, AA); + if (DefMI && Edit->checkRematerializable(VNI, DefMI, TII, AA)) { + DEBUG(dbgs() << "Value " << PrintReg(Reg) << ':' << VNI->id << '@' + << VNI->def << " may remat from " << *DefMI); + } } } } @@ -430,7 +667,7 @@ bool InlineSpiller::hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI) { SlotIndex Idx = LIS.getInstructionIndex(CopyMI); VNInfo *VNI = SpillLI.getVNInfoAt(Idx.getDefIndex()); assert(VNI && VNI->def == Idx.getDefIndex() && "Not defined by copy"); - SibValueMap::const_iterator I = SibValues.find(VNI); + SibValueMap::iterator I = SibValues.find(VNI); if (I == SibValues.end()) return false; @@ -440,6 +677,20 @@ bool InlineSpiller::hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI) { if (!SVI.AllDefsAreReloads && SVI.SpillVNI == VNI) return false; + // SpillReg may have been deleted by remat and DCE. + if (!LIS.hasInterval(SVI.SpillReg)) { + DEBUG(dbgs() << "Stale interval: " << PrintReg(SVI.SpillReg) << '\n'); + SibValues.erase(I); + return false; + } + + LiveInterval &SibLI = LIS.getInterval(SVI.SpillReg); + if (!SibLI.containsValue(SVI.SpillVNI)) { + DEBUG(dbgs() << "Stale value: " << PrintReg(SVI.SpillReg) << '\n'); + SibValues.erase(I); + return false; + } + // Conservatively extend the stack slot range to the range of the original // value. We may be able to do better with stack slot coloring by being more // careful here. @@ -451,19 +702,23 @@ bool InlineSpiller::hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI) { << *StackInt << '\n'); // Already spilled everywhere. - if (SVI.AllDefsAreReloads) + if (SVI.AllDefsAreReloads) { + DEBUG(dbgs() << "\tno spill needed: " << SVI); + ++NumOmitReloadSpill; return true; - + } // We are going to spill SVI.SpillVNI immediately after its def, so clear out // any later spills of the same value. - eliminateRedundantSpills(LIS.getInterval(SVI.SpillReg), SVI.SpillVNI); + eliminateRedundantSpills(SibLI, SVI.SpillVNI); MachineBasicBlock *MBB = LIS.getMBBFromIndex(SVI.SpillVNI->def); MachineBasicBlock::iterator MII; if (SVI.SpillVNI->isPHIDef()) MII = MBB->SkipPHIsAndLabels(MBB->begin()); else { - MII = LIS.getInstructionFromIndex(SVI.SpillVNI->def); + MachineInstr *DefMI = LIS.getInstructionFromIndex(SVI.SpillVNI->def); + assert(DefMI && "Defining instruction disappeared"); + MII = DefMI; ++MII; } // Insert spill without kill flag immediately after def. @@ -473,6 +728,9 @@ bool InlineSpiller::hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI) { LIS.InsertMachineInstrInMaps(MII); VRM.addSpillSlotUse(StackSlot, MII); DEBUG(dbgs() << "\thoisted: " << SVI.SpillVNI->def << '\t' << *MII); + + ++NumSpills; + ++NumHoists; return true; } @@ -488,8 +746,8 @@ void InlineSpiller::eliminateRedundantSpills(LiveInterval &SLI, VNInfo *VNI) { LiveInterval *LI; tie(LI, VNI) = WorkList.pop_back_val(); unsigned Reg = LI->reg; - DEBUG(dbgs() << "Checking redundant spills for " << PrintReg(Reg) << ':' - << VNI->id << '@' << VNI->def << '\n'); + DEBUG(dbgs() << "Checking redundant spills for " + << VNI->id << '@' << VNI->def << " in " << *LI << '\n'); // Regs to spill are taken care of. if (isRegToSpill(Reg)) @@ -527,6 +785,8 @@ void InlineSpiller::eliminateRedundantSpills(LiveInterval &SLI, VNInfo *VNI) { // eliminateDeadDefs won't normally remove stores, so switch opcode. MI->setDesc(TII.get(TargetOpcode::KILL)); DeadDefs.push_back(MI); + ++NumSpillsRemoved; + --NumSpills; } } } while (!WorkList.empty()); @@ -574,13 +834,13 @@ void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) { bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineBasicBlock::iterator MI) { SlotIndex UseIdx = LIS.getInstructionIndex(MI).getUseIndex(); - VNInfo *ParentVNI = VirtReg.getVNInfoAt(UseIdx); + VNInfo *ParentVNI = VirtReg.getVNInfoAt(UseIdx.getBaseIndex()); if (!ParentVNI) { DEBUG(dbgs() << "\tadding flags: "); for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); - if (MO.isReg() && MO.isUse() && MO.getReg() == Edit->getReg()) + if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg) MO.setIsUndef(); } DEBUG(dbgs() << UseIdx << '\t' << *MI); @@ -601,11 +861,11 @@ bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, return false; } - // If the instruction also writes Edit->getReg(), it had better not require - // the same register for uses and defs. + // If the instruction also writes VirtReg.reg, it had better not require the + // same register for uses and defs. bool Reads, Writes; SmallVector Ops; - tie(Reads, Writes) = MI->readsWritesVirtualRegister(Edit->getReg(), &Ops); + tie(Reads, Writes) = MI->readsWritesVirtualRegister(VirtReg.reg, &Ops); if (Writes) { for (unsigned i = 0, e = Ops.size(); i != e; ++i) { MachineOperand &MO = MI->getOperand(Ops[i]); @@ -622,17 +882,14 @@ bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, if (RM.OrigMI->getDesc().canFoldAsLoad() && foldMemoryOperand(MI, Ops, RM.OrigMI)) { Edit->markRematerialized(RM.ParentVNI); + ++NumFoldedLoads; return true; } // Alocate a new register for the remat. - LiveInterval &NewLI = Edit->create(LIS, VRM); + LiveInterval &NewLI = Edit->createFrom(Original, LIS, VRM); NewLI.markNotSpillable(); - // Rematting for a copy: Set allocation hint to be the destination register. - if (MI->isCopy()) - MRI.setRegAllocationHint(NewLI.reg, 0, MI->getOperand(0).getReg()); - // Finally we can rematerialize OrigMI before MI. SlotIndex DefIdx = Edit->rematerializeAt(*MI->getParent(), MI, NewLI.reg, RM, LIS, TII, TRI); @@ -642,7 +899,7 @@ bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, // Replace operands for (unsigned i = 0, e = Ops.size(); i != e; ++i) { MachineOperand &MO = MI->getOperand(Ops[i]); - if (MO.isReg() && MO.isUse() && MO.getReg() == Edit->getReg()) { + if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg) { MO.setReg(NewLI.reg); MO.setIsKill(); } @@ -652,6 +909,7 @@ bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, VNInfo *DefVNI = NewLI.getNextValue(DefIdx, 0, LIS.getVNInfoAllocator()); NewLI.addRange(LiveRange(DefIdx, UseIdx.getDefIndex(), DefVNI)); DEBUG(dbgs() << "\tinterval: " << NewLI << '\n'); + ++NumRemats; return true; } @@ -718,13 +976,18 @@ void InlineSpiller::reMaterializeAll() { DEBUG(dbgs() << RegsToSpill.size() << " registers to spill after remat.\n"); } + +//===----------------------------------------------------------------------===// +// Spilling +//===----------------------------------------------------------------------===// + /// If MI is a load or store of StackSlot, it can be removed. bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, unsigned Reg) { int FI = 0; - unsigned InstrReg; - if (!(InstrReg = TII.isLoadFromStackSlot(MI, FI)) && - !(InstrReg = TII.isStoreToStackSlot(MI, FI))) - return false; + unsigned InstrReg = TII.isLoadFromStackSlot(MI, FI); + bool IsLoad = InstrReg; + if (!IsLoad) + InstrReg = TII.isStoreToStackSlot(MI, FI); // We have a stack access. Is it the right register and slot? if (InstrReg != Reg || FI != StackSlot) @@ -733,6 +996,15 @@ bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, unsigned Reg) { DEBUG(dbgs() << "Coalescing stack access: " << *MI); LIS.RemoveMachineInstrFromMaps(MI); MI->eraseFromParent(); + + if (IsLoad) { + ++NumReloadsRemoved; + --NumReloads; + } else { + ++NumSpillsRemoved; + --NumSpills; + } + return true; } @@ -744,6 +1016,7 @@ bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, unsigned Reg) { bool InlineSpiller::foldMemoryOperand(MachineBasicBlock::iterator MI, const SmallVectorImpl &Ops, MachineInstr *LoadMI) { + bool WasCopy = MI->isCopy(); // TargetInstrInfo::foldMemoryOperand only expects explicit, non-tied // operands. SmallVector FoldOps; @@ -773,14 +1046,20 @@ bool InlineSpiller::foldMemoryOperand(MachineBasicBlock::iterator MI, VRM.addSpillSlotUse(StackSlot, FoldMI); MI->eraseFromParent(); DEBUG(dbgs() << "\tfolded: " << *FoldMI); + if (!WasCopy) + ++NumFolded; + else if (Ops.front() == 0) + ++NumSpills; + else + ++NumReloads; return true; } /// insertReload - Insert a reload of NewLI.reg before MI. void InlineSpiller::insertReload(LiveInterval &NewLI, + SlotIndex Idx, MachineBasicBlock::iterator MI) { MachineBasicBlock &MBB = *MI->getParent(); - SlotIndex Idx = LIS.getInstructionIndex(MI).getDefIndex(); TII.loadRegFromStackSlot(MBB, MI, NewLI.reg, StackSlot, MRI.getRegClass(NewLI.reg), &TRI); --MI; // Point to load instruction. @@ -790,19 +1069,13 @@ void InlineSpiller::insertReload(LiveInterval &NewLI, VNInfo *LoadVNI = NewLI.getNextValue(LoadIdx, 0, LIS.getVNInfoAllocator()); NewLI.addRange(LiveRange(LoadIdx, Idx, LoadVNI)); + ++NumReloads; } /// insertSpill - Insert a spill of NewLI.reg after MI. void InlineSpiller::insertSpill(LiveInterval &NewLI, const LiveInterval &OldLI, - MachineBasicBlock::iterator MI) { + SlotIndex Idx, MachineBasicBlock::iterator MI) { MachineBasicBlock &MBB = *MI->getParent(); - - // Get the defined value. It could be an early clobber so keep the def index. - SlotIndex Idx = LIS.getInstructionIndex(MI).getDefIndex(); - VNInfo *VNI = OldLI.getVNInfoAt(Idx); - assert(VNI && VNI->def.getDefIndex() == Idx && "Inconsistent VNInfo"); - Idx = VNI->def; - TII.storeRegToStackSlot(MBB, ++MI, NewLI.reg, true, StackSlot, MRI.getRegClass(NewLI.reg), &TRI); --MI; // Point to store instruction. @@ -811,10 +1084,12 @@ void InlineSpiller::insertSpill(LiveInterval &NewLI, const LiveInterval &OldLI, DEBUG(dbgs() << "\tspilled: " << StoreIdx << '\t' << *MI); VNInfo *StoreVNI = NewLI.getNextValue(Idx, 0, LIS.getVNInfoAllocator()); NewLI.addRange(LiveRange(Idx, StoreIdx, StoreVNI)); + ++NumSpills; } /// spillAroundUses - insert spill code around each use of Reg. void InlineSpiller::spillAroundUses(unsigned Reg) { + DEBUG(dbgs() << "spillAroundUses " << PrintReg(Reg) << '\n'); LiveInterval &OldLI = LIS.getInterval(Reg); // Iterate over instructions using Reg. @@ -852,9 +1127,22 @@ void InlineSpiller::spillAroundUses(unsigned Reg) { SmallVector Ops; tie(Reads, Writes) = MI->readsWritesVirtualRegister(Reg, &Ops); + // Find the slot index where this instruction reads and writes OldLI. + // This is usually the def slot, except for tied early clobbers. + SlotIndex Idx = LIS.getInstructionIndex(MI).getDefIndex(); + if (VNInfo *VNI = OldLI.getVNInfoAt(Idx.getUseIndex())) + if (SlotIndex::isSameInstr(Idx, VNI->def)) + Idx = VNI->def; + // Check for a sibling copy. unsigned SibReg = isFullCopyOf(MI, Reg); if (SibReg && isSibling(SibReg)) { + // This may actually be a copy between snippets. + if (isRegToSpill(SibReg)) { + DEBUG(dbgs() << "Found new snippet copy: " << *MI); + SnippetCopies.insert(MI); + continue; + } if (Writes) { // Hoist the spill of a sib-reg copy. if (hoistSpill(OldLI, MI)) { @@ -865,7 +1153,6 @@ void InlineSpiller::spillAroundUses(unsigned Reg) { } } else { // This is a reload for a sib-reg copy. Drop spills downstream. - SlotIndex Idx = LIS.getInstructionIndex(MI).getDefIndex(); LiveInterval &SibLI = LIS.getInterval(SibReg); eliminateRedundantSpills(SibLI, SibLI.getVNInfoAt(Idx)); // The COPY will fold to a reload below. @@ -882,7 +1169,7 @@ void InlineSpiller::spillAroundUses(unsigned Reg) { NewLI.markNotSpillable(); if (Reads) - insertReload(NewLI, MI); + insertReload(NewLI, Idx, MI); // Rewrite instruction operands. bool hasLiveDef = false; @@ -897,40 +1184,18 @@ void InlineSpiller::spillAroundUses(unsigned Reg) { hasLiveDef = true; } } + DEBUG(dbgs() << "\trewrite: " << Idx << '\t' << *MI); // FIXME: Use a second vreg if instruction has no tied ops. if (Writes && hasLiveDef) - insertSpill(NewLI, OldLI, MI); + insertSpill(NewLI, OldLI, Idx, MI); DEBUG(dbgs() << "\tinterval: " << NewLI << '\n'); } } -void InlineSpiller::spill(LiveRangeEdit &edit) { - Edit = &edit; - assert(!TargetRegisterInfo::isStackSlot(edit.getReg()) - && "Trying to spill a stack slot."); - // Share a stack slot among all descendants of Original. - Original = VRM.getOriginal(edit.getReg()); - StackSlot = VRM.getStackSlot(Original); - StackInt = 0; - - DEBUG(dbgs() << "Inline spilling " - << MRI.getRegClass(edit.getReg())->getName() - << ':' << edit.getParent() << "\nFrom original " - << LIS.getInterval(Original) << '\n'); - assert(edit.getParent().isSpillable() && - "Attempting to spill already spilled value."); - assert(DeadDefs.empty() && "Previous spill didn't remove dead defs"); - - collectRegsToSpill(); - analyzeSiblingValues(); - reMaterializeAll(); - - // Remat may handle everything. - if (RegsToSpill.empty()) - return; - +/// spillAll - Spill all registers remaining after rematerialization. +void InlineSpiller::spillAll() { // Update LiveStacks now that we are committed to spilling. if (StackSlot == VirtRegMap::NO_STACK_SLOT) { StackSlot = VRM.assignVirt2StackSlot(Original); @@ -939,8 +1204,8 @@ void InlineSpiller::spill(LiveRangeEdit &edit) { } else StackInt = &LSS.getInterval(StackSlot); - if (Original != edit.getReg()) - VRM.assignVirt2StackSlot(edit.getReg(), StackSlot); + if (Original != Edit->getReg()) + VRM.assignVirt2StackSlot(Edit->getReg(), StackSlot); assert(StackInt->getNumValNums() == 1 && "Bad stack interval values"); for (unsigned i = 0, e = RegsToSpill.size(); i != e; ++i) @@ -959,15 +1224,47 @@ void InlineSpiller::spill(LiveRangeEdit &edit) { } // Finally delete the SnippetCopies. - for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(edit.getReg()); - MachineInstr *MI = RI.skipInstruction();) { - assert(SnippetCopies.count(MI) && "Remaining use wasn't a snippet copy"); - // FIXME: Do this with a LiveRangeEdit callback. - VRM.RemoveMachineInstrFromMaps(MI); - LIS.RemoveMachineInstrFromMaps(MI); - MI->eraseFromParent(); + for (unsigned i = 0, e = RegsToSpill.size(); i != e; ++i) { + for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(RegsToSpill[i]); + MachineInstr *MI = RI.skipInstruction();) { + assert(SnippetCopies.count(MI) && "Remaining use wasn't a snippet copy"); + // FIXME: Do this with a LiveRangeEdit callback. + VRM.RemoveMachineInstrFromMaps(MI); + LIS.RemoveMachineInstrFromMaps(MI); + MI->eraseFromParent(); + } } + // Delete all spilled registers. for (unsigned i = 0, e = RegsToSpill.size(); i != e; ++i) - edit.eraseVirtReg(RegsToSpill[i], LIS); + Edit->eraseVirtReg(RegsToSpill[i], LIS); +} + +void InlineSpiller::spill(LiveRangeEdit &edit) { + ++NumSpilledRanges; + Edit = &edit; + assert(!TargetRegisterInfo::isStackSlot(edit.getReg()) + && "Trying to spill a stack slot."); + // Share a stack slot among all descendants of Original. + Original = VRM.getOriginal(edit.getReg()); + StackSlot = VRM.getStackSlot(Original); + StackInt = 0; + + DEBUG(dbgs() << "Inline spilling " + << MRI.getRegClass(edit.getReg())->getName() + << ':' << edit.getParent() << "\nFrom original " + << LIS.getInterval(Original) << '\n'); + assert(edit.getParent().isSpillable() && + "Attempting to spill already spilled value."); + assert(DeadDefs.empty() && "Previous spill didn't remove dead defs"); + + collectRegsToSpill(); + analyzeSiblingValues(); + reMaterializeAll(); + + // Remat may handle everything. + if (!RegsToSpill.empty()) + spillAll(); + + Edit->calculateRegClassAndHint(MF, LIS, Loops); }