X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FRegAllocLinearScan.cpp;h=9e97d89c62e129ac31b6061b698737ce5f9927ad;hb=f648125be9385a0d4abfd5e77ea3dd40694c4c07;hp=63a99e461d33ee7401ab3850e64c3be5632cf4dc;hpb=c23197a26f34f559ea9797de51e187087c039c42;p=oota-llvm.git diff --git a/lib/CodeGen/RegAllocLinearScan.cpp b/lib/CodeGen/RegAllocLinearScan.cpp index 63a99e461d3..9e97d89c62e 100644 --- a/lib/CodeGen/RegAllocLinearScan.cpp +++ b/lib/CodeGen/RegAllocLinearScan.cpp @@ -16,6 +16,7 @@ #include "VirtRegRewriter.h" #include "Spiller.h" #include "llvm/Function.h" +#include "llvm/CodeGen/CalcSpillWeights.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveStackAnalysis.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -35,8 +36,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" #include #include #include @@ -61,8 +61,8 @@ PreSplitIntervals("pre-alloc-split", cl::init(false), cl::Hidden); static cl::opt -NewSpillFramework("new-spill-framework", - cl::desc("New spilling framework"), +TrivCoalesceEnds("trivial-coalesce-ends", + cl::desc("Attempt trivial coalescing of interval ends"), cl::init(false), cl::Hidden); static RegisterRegAlloc @@ -70,9 +70,31 @@ linearscanRegAlloc("linearscan", "linear scan register allocator", createLinearScanRegisterAllocator); namespace { - struct VISIBILITY_HIDDEN RALinScan : public MachineFunctionPass { + // When we allocate a register, add it to a fixed-size queue of + // registers to skip in subsequent allocations. This trades a small + // amount of register pressure and increased spills for flexibility in + // the post-pass scheduler. + // + // Note that in a the number of registers used for reloading spills + // will be one greater than the value of this option. + // + // One big limitation of this is that it doesn't differentiate between + // different register classes. So on x86-64, if there is xmm register + // pressure, it can caused fewer GPRs to be held in the queue. + static cl::opt + NumRecentlyUsedRegs("linearscan-skip-count", + cl::desc("Number of registers for linearscan to remember to skip."), + cl::init(0), + cl::Hidden); + + struct RALinScan : public MachineFunctionPass { static char ID; - RALinScan() : MachineFunctionPass(&ID) {} + RALinScan() : MachineFunctionPass(&ID) { + // Initialize the queue to record recently-used registers. + if (NumRecentlyUsedRegs > 0) + RecentRegs.resize(NumRecentlyUsedRegs, 0); + RecentNext = RecentRegs.begin(); + } typedef std::pair IntervalPtr; typedef SmallVector IntervalPtrs; @@ -138,18 +160,35 @@ namespace { std::auto_ptr spiller_; + // The queue of recently-used registers. + SmallVector RecentRegs; + SmallVector::iterator RecentNext; + + // Record that we just picked this register. + void recordRecentlyUsed(unsigned reg) { + assert(reg != 0 && "Recently used register is NOREG!"); + if (!RecentRegs.empty()) { + *RecentNext++ = reg; + if (RecentNext == RecentRegs.end()) + RecentNext = RecentRegs.begin(); + } + } + public: virtual const char* getPassName() const { return "Linear Scan Register Allocator"; } virtual void getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesCFG(); AU.addRequired(); + AU.addPreserved(); if (StrongPHIElim) AU.addRequiredID(StrongPHIEliminationID); // Make sure PassManager knows which analyses to make available // to coalescing and which analyses coalescing invalidates. AU.addRequiredTransitive(); + AU.addRequired(); if (PreSplitIntervals) AU.addRequiredID(PreAllocSplittingID); AU.addRequired(); @@ -165,6 +204,12 @@ namespace { /// runOnMachineFunction - register allocate the whole function bool runOnMachineFunction(MachineFunction&); + // Determine if we skip this register due to its being recently used. + bool isRecentlyUsed(unsigned reg) const { + return std::find(RecentRegs.begin(), RecentRegs.end(), reg) != + RecentRegs.end(); + } + private: /// linearScan - the linear scan algorithm void linearScan(); @@ -175,11 +220,11 @@ namespace { /// processActiveIntervals - expire old intervals and move non-overlapping /// ones to the inactive list. - void processActiveIntervals(unsigned CurPoint); + void processActiveIntervals(SlotIndex CurPoint); /// processInactiveIntervals - expire old intervals and move overlapping /// ones to the active list. - void processInactiveIntervals(unsigned CurPoint); + void processInactiveIntervals(SlotIndex CurPoint); /// hasNextReloadInterval - Return the next liveinterval that's being /// defined by a reload from the same SS as the specified one. @@ -232,7 +277,7 @@ namespace { bool Error = false; for (unsigned i = 0, e = tri_->getNumRegs(); i != e; ++i) { if (regUse_[i] != 0) { - cerr << tri_->getName(i) << " is still in use!\n"; + errs() << tri_->getName(i) << " is still in use!\n"; Error = true; } } @@ -297,15 +342,20 @@ namespace { template void printIntervals(const char* const str, ItTy i, ItTy e) const { - if (str) DOUT << str << " intervals:\n"; - for (; i != e; ++i) { - DOUT << "\t" << *i->first << " -> "; - unsigned reg = i->first->reg; - if (TargetRegisterInfo::isVirtualRegister(reg)) { - reg = vrm_->getPhys(reg); - } - DOUT << tri_->getName(reg) << '\n'; - } + DEBUG({ + if (str) + errs() << str << " intervals:\n"; + + for (; i != e; ++i) { + errs() << "\t" << *i->first << " -> "; + + unsigned reg = i->first->reg; + if (TargetRegisterInfo::isVirtualRegister(reg)) + reg = vrm_->getPhys(reg); + + errs() << tri_->getName(reg) << '\n'; + } + }); } }; char RALinScan::ID = 0; @@ -347,65 +397,71 @@ void RALinScan::ComputeRelatedRegClasses() { RelatedRegClasses.unionSets(I->second, OneClassForEachPhysReg[*AS]); } -/// attemptTrivialCoalescing - If a simple interval is defined by a copy, -/// try allocate the definition the same register as the source register -/// if the register is not defined during live time of the interval. This -/// eliminate a copy. This is used to coalesce copies which were not -/// coalesced away before allocation either due to dest and src being in -/// different register classes or because the coalescer was overly -/// conservative. +/// attemptTrivialCoalescing - If a simple interval is defined by a copy, try +/// allocate the definition the same register as the source register if the +/// register is not defined during live time of the interval. If the interval is +/// killed by a copy, try to use the destination register. This eliminates a +/// copy. This is used to coalesce copies which were not coalesced away before +/// allocation either due to dest and src being in different register classes or +/// because the coalescer was overly conservative. unsigned RALinScan::attemptTrivialCoalescing(LiveInterval &cur, unsigned Reg) { unsigned Preference = vrm_->getRegAllocPref(cur.reg); if ((Preference && Preference == Reg) || !cur.containsOneValue()) return Reg; - VNInfo *vni = cur.begin()->valno; - if (!vni->def || vni->isUnused() || !vni->isDefAccurate()) + // We cannot handle complicated live ranges. Simple linear stuff only. + if (cur.ranges.size() != 1) return Reg; - MachineInstr *CopyMI = li_->getInstructionFromIndex(vni->def); - unsigned SrcReg, DstReg, SrcSubReg, DstSubReg, PhysReg; - if (!CopyMI || - !tii_->isMoveInstr(*CopyMI, SrcReg, DstReg, SrcSubReg, DstSubReg)) + + const LiveRange &range = cur.ranges.front(); + + VNInfo *vni = range.valno; + if (vni->isUnused()) return Reg; - PhysReg = SrcReg; - if (TargetRegisterInfo::isVirtualRegister(SrcReg)) { - if (!vrm_->isAssignedReg(SrcReg)) + + unsigned CandReg; + { + MachineInstr *CopyMI; + unsigned SrcReg, DstReg, SrcSubReg, DstSubReg; + if (vni->def != SlotIndex() && vni->isDefAccurate() && + (CopyMI = li_->getInstructionFromIndex(vni->def)) && + tii_->isMoveInstr(*CopyMI, SrcReg, DstReg, SrcSubReg, DstSubReg)) + // Defined by a copy, try to extend SrcReg forward + CandReg = SrcReg; + else if (TrivCoalesceEnds && + (CopyMI = + li_->getInstructionFromIndex(range.end.getBaseIndex())) && + tii_->isMoveInstr(*CopyMI, SrcReg, DstReg, SrcSubReg, DstSubReg) && + cur.reg == SrcReg) + // Only used by a copy, try to extend DstReg backwards + CandReg = DstReg; + else return Reg; - PhysReg = vrm_->getPhys(SrcReg); } - if (Reg == PhysReg) + + if (TargetRegisterInfo::isVirtualRegister(CandReg)) { + if (!vrm_->isAssignedReg(CandReg)) + return Reg; + CandReg = vrm_->getPhys(CandReg); + } + if (Reg == CandReg) return Reg; const TargetRegisterClass *RC = mri_->getRegClass(cur.reg); - if (!RC->contains(PhysReg)) + if (!RC->contains(CandReg)) return Reg; - // Try to coalesce. - if (!li_->conflictsWithPhysRegDef(cur, *vrm_, PhysReg)) { - DOUT << "Coalescing: " << cur << " -> " << tri_->getName(PhysReg) - << '\n'; - vrm_->clearVirt(cur.reg); - vrm_->assignVirt2Phys(cur.reg, PhysReg); - - // Remove unnecessary kills since a copy does not clobber the register. - if (li_->hasInterval(SrcReg)) { - LiveInterval &SrcLI = li_->getInterval(SrcReg); - for (MachineRegisterInfo::reg_iterator I = mri_->reg_begin(cur.reg), - E = mri_->reg_end(); I != E; ++I) { - MachineOperand &O = I.getOperand(); - if (!O.isUse() || !O.isKill()) - continue; - MachineInstr *MI = &*I; - if (SrcLI.liveAt(li_->getDefIndex(li_->getInstructionIndex(MI)))) - O.setIsKill(false); - } - } + if (li_->conflictsWithPhysReg(cur, *vrm_, CandReg)) + return Reg; - ++NumCoalesce; - return PhysReg; - } + // Try to coalesce. + DEBUG(errs() << "Coalescing: " << cur << " -> " << tri_->getName(CandReg) + << '\n'); + vrm_->clearVirt(cur.reg); + vrm_->assignVirt2Phys(cur.reg, CandReg); - return Reg; + ++NumCoalesce; + return CandReg; } bool RALinScan::runOnMachineFunction(MachineFunction &fn) { @@ -434,9 +490,7 @@ bool RALinScan::runOnMachineFunction(MachineFunction &fn) { vrm_ = &getAnalysis(); if (!rewriter_.get()) rewriter_.reset(createVirtRegRewriter()); - if (NewSpillFramework) { - spiller_.reset(createSpiller(mf_, li_, ls_, vrm_)); - } + spiller_.reset(createSpiller(mf_, li_, loopInfo, vrm_)); initIntervalSets(); @@ -473,50 +527,60 @@ void RALinScan::initIntervalSets() for (LiveIntervals::iterator i = li_->begin(), e = li_->end(); i != e; ++i) { if (TargetRegisterInfo::isPhysicalRegister(i->second->reg)) { - mri_->setPhysRegUsed(i->second->reg); - fixed_.push_back(std::make_pair(i->second, i->second->begin())); - } else - unhandled_.push(i->second); + if (!i->second->empty()) { + mri_->setPhysRegUsed(i->second->reg); + fixed_.push_back(std::make_pair(i->second, i->second->begin())); + } + } else { + if (i->second->empty()) { + assignRegOrStackSlotAtInterval(i->second); + } + else + unhandled_.push(i->second); + } } } -void RALinScan::linearScan() -{ +void RALinScan::linearScan() { // linear scan algorithm - DOUT << "********** LINEAR SCAN **********\n"; - DOUT << "********** Function: " << mf_->getFunction()->getName() << '\n'; - - DEBUG(printIntervals("fixed", fixed_.begin(), fixed_.end())); + DEBUG({ + errs() << "********** LINEAR SCAN **********\n" + << "********** Function: " + << mf_->getFunction()->getName() << '\n'; + printIntervals("fixed", fixed_.begin(), fixed_.end()); + }); while (!unhandled_.empty()) { // pick the interval with the earliest start point LiveInterval* cur = unhandled_.top(); unhandled_.pop(); ++NumIters; - DOUT << "\n*** CURRENT ***: " << *cur << '\n'; + DEBUG(errs() << "\n*** CURRENT ***: " << *cur << '\n'); - if (!cur->empty()) { - processActiveIntervals(cur->beginNumber()); - processInactiveIntervals(cur->beginNumber()); + assert(!cur->empty() && "Empty interval in unhandled set."); - assert(TargetRegisterInfo::isVirtualRegister(cur->reg) && - "Can only allocate virtual registers!"); - } + processActiveIntervals(cur->beginIndex()); + processInactiveIntervals(cur->beginIndex()); + + assert(TargetRegisterInfo::isVirtualRegister(cur->reg) && + "Can only allocate virtual registers!"); // Allocating a virtual register. try to find a free // physical register or spill an interval (possibly this one) in order to // assign it one. assignRegOrStackSlotAtInterval(cur); - DEBUG(printIntervals("active", active_.begin(), active_.end())); - DEBUG(printIntervals("inactive", inactive_.begin(), inactive_.end())); + DEBUG({ + printIntervals("active", active_.begin(), active_.end()); + printIntervals("inactive", inactive_.begin(), inactive_.end()); + }); } // Expire any remaining active intervals while (!active_.empty()) { IntervalPtr &IP = active_.back(); unsigned reg = IP.first->reg; - DOUT << "\tinterval " << *IP.first << " expired\n"; + DEBUG(errs() << "\tinterval " << *IP.first << " expired\n"); assert(TargetRegisterInfo::isVirtualRegister(reg) && "Can only allocate virtual registers!"); reg = vrm_->getPhys(reg); @@ -525,9 +589,11 @@ void RALinScan::linearScan() } // Expire any remaining inactive intervals - DEBUG(for (IntervalPtrs::reverse_iterator - i = inactive_.rbegin(); i != inactive_.rend(); ++i) - DOUT << "\tinterval " << *i->first << " expired\n"); + DEBUG({ + for (IntervalPtrs::reverse_iterator + i = inactive_.rbegin(); i != inactive_.rend(); ++i) + errs() << "\tinterval " << *i->first << " expired\n"; + }); inactive_.clear(); // Add live-ins to every BB except for entry. Also perform trivial coalescing. @@ -562,7 +628,7 @@ void RALinScan::linearScan() } } - DOUT << *vrm_; + DEBUG(errs() << *vrm_); // Look for physical registers that end up not being allocated even though // register allocator had to spill other registers in its register class. @@ -574,9 +640,9 @@ void RALinScan::linearScan() /// processActiveIntervals - expire old intervals and move non-overlapping ones /// to the inactive list. -void RALinScan::processActiveIntervals(unsigned CurPoint) +void RALinScan::processActiveIntervals(SlotIndex CurPoint) { - DOUT << "\tprocessing active intervals:\n"; + DEBUG(errs() << "\tprocessing active intervals:\n"); for (unsigned i = 0, e = active_.size(); i != e; ++i) { LiveInterval *Interval = active_[i].first; @@ -586,7 +652,7 @@ void RALinScan::processActiveIntervals(unsigned CurPoint) IntervalPos = Interval->advanceTo(IntervalPos, CurPoint); if (IntervalPos == Interval->end()) { // Remove expired intervals. - DOUT << "\t\tinterval " << *Interval << " expired\n"; + DEBUG(errs() << "\t\tinterval " << *Interval << " expired\n"); assert(TargetRegisterInfo::isVirtualRegister(reg) && "Can only allocate virtual registers!"); reg = vrm_->getPhys(reg); @@ -599,7 +665,7 @@ void RALinScan::processActiveIntervals(unsigned CurPoint) } else if (IntervalPos->start > CurPoint) { // Move inactive intervals to inactive list. - DOUT << "\t\tinterval " << *Interval << " inactive\n"; + DEBUG(errs() << "\t\tinterval " << *Interval << " inactive\n"); assert(TargetRegisterInfo::isVirtualRegister(reg) && "Can only allocate virtual registers!"); reg = vrm_->getPhys(reg); @@ -620,9 +686,9 @@ void RALinScan::processActiveIntervals(unsigned CurPoint) /// processInactiveIntervals - expire old intervals and move overlapping /// ones to the active list. -void RALinScan::processInactiveIntervals(unsigned CurPoint) +void RALinScan::processInactiveIntervals(SlotIndex CurPoint) { - DOUT << "\tprocessing inactive intervals:\n"; + DEBUG(errs() << "\tprocessing inactive intervals:\n"); for (unsigned i = 0, e = inactive_.size(); i != e; ++i) { LiveInterval *Interval = inactive_[i].first; @@ -632,7 +698,7 @@ void RALinScan::processInactiveIntervals(unsigned CurPoint) IntervalPos = Interval->advanceTo(IntervalPos, CurPoint); if (IntervalPos == Interval->end()) { // remove expired intervals. - DOUT << "\t\tinterval " << *Interval << " expired\n"; + DEBUG(errs() << "\t\tinterval " << *Interval << " expired\n"); // Pop off the end of the list. inactive_[i] = inactive_.back(); @@ -640,7 +706,7 @@ void RALinScan::processInactiveIntervals(unsigned CurPoint) --i; --e; } else if (IntervalPos->start <= CurPoint) { // move re-activated intervals in active list - DOUT << "\t\tinterval " << *Interval << " active\n"; + DEBUG(errs() << "\t\tinterval " << *Interval << " active\n"); assert(TargetRegisterInfo::isVirtualRegister(reg) && "Can only allocate virtual registers!"); reg = vrm_->getPhys(reg); @@ -701,7 +767,7 @@ FindIntervalInVector(RALinScan::IntervalPtrs &IP, LiveInterval *LI) { return IP.end(); } -static void RevertVectorIteratorsTo(RALinScan::IntervalPtrs &V, unsigned Point){ +static void RevertVectorIteratorsTo(RALinScan::IntervalPtrs &V, SlotIndex Point){ for (unsigned i = 0, e = V.size(); i != e; ++i) { RALinScan::IntervalPtr &IP = V[i]; LiveInterval::iterator I = std::upper_bound(IP.first->begin(), @@ -727,7 +793,8 @@ static void addStackInterval(LiveInterval *cur, LiveStacks *ls_, if (SI.hasAtLeastOneValue()) VNI = SI.getValNumInfo(0); else - VNI = SI.getNextValue(0, 0, false, ls_->getVNInfoAllocator()); + VNI = SI.getNextValue(SlotIndex(), 0, false, + ls_->getVNInfoAllocator()); LiveInterval &RI = li_->getInterval(cur->reg); // FIXME: This may be overly conservative. @@ -766,10 +833,12 @@ void RALinScan::findIntervalsToSpill(LiveInterval *cur, float Conflicts[3] = { 0.0f, 0.0f, 0.0f }; SmallVector SLIs[3]; - DOUT << "\tConsidering " << NumCands << " candidates: "; - DEBUG(for (unsigned i = 0; i != NumCands; ++i) - DOUT << tri_->getName(Candidates[i].first) << " "; - DOUT << "\n";); + DEBUG({ + errs() << "\tConsidering " << NumCands << " candidates: "; + for (unsigned i = 0; i != NumCands; ++i) + errs() << tri_->getName(Candidates[i].first) << " "; + errs() << "\n"; + }); // Calculate the number of conflicts of each candidate. for (IntervalPtrs::iterator i = active_.begin(); i != active_.end(); ++i) { @@ -818,9 +887,15 @@ void RALinScan::findIntervalsToSpill(LiveInterval *cur, namespace { struct WeightCompare { + private: + const RALinScan &Allocator; + + public: + WeightCompare(const RALinScan &Alloc) : Allocator(Alloc) {} + typedef std::pair RegWeightPair; bool operator()(const RegWeightPair &LHS, const RegWeightPair &RHS) const { - return LHS.second < RHS.second; + return LHS.second < RHS.second && !Allocator.isRecentlyUsed(LHS.first); } }; } @@ -867,16 +942,15 @@ void RALinScan::UpgradeRegister(unsigned Reg) { namespace { struct LISorter { bool operator()(LiveInterval* A, LiveInterval* B) { - return A->beginNumber() < B->beginNumber(); + return A->beginIndex() < B->beginIndex(); } }; } /// assignRegOrStackSlotAtInterval - assign a register if one is available, or /// spill. -void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) -{ - DOUT << "\tallocating current interval: "; +void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) { + DEBUG(errs() << "\tallocating current interval: "); // This is an implicitly defined live interval, just assign any register. const TargetRegisterClass *RC = mri_->getRegClass(cur->reg); @@ -884,7 +958,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) unsigned physReg = vrm_->getRegAllocPref(cur->reg); if (!physReg) physReg = *RC->allocation_order_begin(*mf_); - DOUT << tri_->getName(physReg) << '\n'; + DEBUG(errs() << tri_->getName(physReg) << '\n'); // Note the register is not really in use. vrm_->assignVirt2Phys(cur->reg, physReg); return; @@ -893,7 +967,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) backUpRegUses(); std::vector > SpillWeightsToAdd; - unsigned StartPosition = cur->beginNumber(); + SlotIndex StartPosition = cur->beginIndex(); const TargetRegisterClass *RCLeader = RelatedRegClasses.getLeaderValue(RC); // If start of this live interval is defined by a move instruction and its @@ -903,7 +977,8 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) // one, e.g. X86::mov32to32_. These move instructions are not coalescable. if (!vrm_->getRegAllocPref(cur->reg) && cur->hasAtLeastOneValue()) { VNInfo *vni = cur->begin()->valno; - if (vni->def && !vni->isUnused() && vni->isDefAccurate()) { + if ((vni->def != SlotIndex()) && !vni->isUnused() && + vni->isDefAccurate()) { MachineInstr *CopyMI = li_->getInstructionFromIndex(vni->def); unsigned SrcReg, DstReg, SrcSubReg, DstSubReg; if (CopyMI && @@ -965,7 +1040,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) // Okay, this reg is on the fixed list. Check to see if we actually // conflict. LiveInterval *I = IP.first; - if (I->endNumber() > StartPosition) { + if (I->endIndex() > StartPosition) { LiveInterval::iterator II = I->advanceTo(IP.second, StartPosition); IP.second = II; if (II != I->begin() && II->start > StartPosition) @@ -990,7 +1065,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) const TargetRegisterClass *RegRC = OneClassForEachPhysReg[I->reg]; if (RelatedRegClasses.getLeaderValue(RegRC) == RCLeader && - I->endNumber() > StartPosition) { + I->endIndex() > StartPosition) { LiveInterval::iterator II = I->advanceTo(IP.second, StartPosition); IP.second = II; if (II != I->begin() && II->start > StartPosition) @@ -1017,7 +1092,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) // the free physical register and add this interval to the active // list. if (physReg) { - DOUT << tri_->getName(physReg) << '\n'; + DEBUG(errs() << tri_->getName(physReg) << '\n'); vrm_->assignVirt2Phys(cur->reg, physReg); addRegUse(physReg); active_.push_back(std::make_pair(cur, cur->begin())); @@ -1033,7 +1108,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) } return; } - DOUT << "no free registers\n"; + DEBUG(errs() << "no free registers\n"); // Compile the spill weights into an array that is better for scanning. std::vector SpillWeights(tri_->getNumRegs(), 0.0f); @@ -1051,7 +1126,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) updateSpillWeights(SpillWeights, reg, i->first->weight, RC); } - DOUT << "\tassigning stack slot at interval "<< *cur << ":\n"; + DEBUG(errs() << "\tassigning stack slot at interval "<< *cur << ":\n"); // Find a register to spill. float minWeight = HUGE_VALF; @@ -1064,7 +1139,8 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) e = RC->allocation_order_end(*mf_); i != e; ++i) { unsigned reg = *i; float regWeight = SpillWeights[reg]; - if (minWeight > regWeight) + // Skip recently allocated registers. + if (minWeight > regWeight && !isRecentlyUsed(reg)) Found = true; RegsWeights.push_back(std::make_pair(reg, regWeight)); } @@ -1082,7 +1158,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) } // Sort all potential spill candidates by weight. - std::sort(RegsWeights.begin(), RegsWeights.end(), WeightCompare()); + std::sort(RegsWeights.begin(), RegsWeights.end(), WeightCompare(*this)); minReg = RegsWeights[0].first; minWeight = RegsWeights[0].second; if (minWeight == HUGE_VALF) { @@ -1104,6 +1180,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) DowngradedRegs.clear(); assignRegOrStackSlotAtInterval(cur); } else { + assert(false && "Ran out of registers during register allocation!"); llvm_report_error("Ran out of registers during register allocation!"); } return; @@ -1118,24 +1195,23 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) --LastCandidate; } - DOUT << "\t\tregister(s) with min weight(s): "; - DEBUG(for (unsigned i = 0; i != LastCandidate; ++i) - DOUT << tri_->getName(RegsWeights[i].first) - << " (" << RegsWeights[i].second << ")\n"); + DEBUG({ + errs() << "\t\tregister(s) with min weight(s): "; + + for (unsigned i = 0; i != LastCandidate; ++i) + errs() << tri_->getName(RegsWeights[i].first) + << " (" << RegsWeights[i].second << ")\n"; + }); // If the current has the minimum weight, we need to spill it and // add any added intervals back to unhandled, and restart // linearscan. if (cur->weight != HUGE_VALF && cur->weight <= minWeight) { - DOUT << "\t\t\tspilling(c): " << *cur << '\n'; + DEBUG(errs() << "\t\t\tspilling(c): " << *cur << '\n'); SmallVector spillIs; std::vector added; - if (!NewSpillFramework) { - added = li_->addIntervalsForSpills(*cur, spillIs, loopInfo, *vrm_); - } else { - added = spiller_->spill(cur); - } + added = spiller_->spill(cur, spillIs); std::sort(added.begin(), added.end(), LISorter()); addStackInterval(cur, ls_, li_, mri_, *vrm_); @@ -1155,14 +1231,14 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) LiveInterval *ReloadLi = added[i]; if (ReloadLi->weight == HUGE_VALF && li_->getApproximateInstructionCount(*ReloadLi) == 0) { - unsigned ReloadIdx = ReloadLi->beginNumber(); + SlotIndex ReloadIdx = ReloadLi->beginIndex(); MachineBasicBlock *ReloadMBB = li_->getMBBFromIndex(ReloadIdx); int ReloadSS = vrm_->getStackSlot(ReloadLi->reg); if (LastReloadMBB == ReloadMBB && LastReloadSS == ReloadSS) { // Last reload of same SS is in the same MBB. We want to try to // allocate both reloads the same register and make sure the reg // isn't clobbered in between if at all possible. - assert(LastReload->beginNumber() < ReloadIdx); + assert(LastReload->beginIndex() < ReloadIdx); NextReloadMap.insert(std::make_pair(LastReload->reg, ReloadLi->reg)); } LastReloadMBB = ReloadMBB; @@ -1197,9 +1273,9 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) // The earliest start of a Spilled interval indicates up to where // in handled we need to roll back + assert(!spillIs.empty() && "No spill intervals?"); + SlotIndex earliestStart = spillIs[0]->beginIndex(); - LiveInterval *earliestStartInterval = cur; - // Spill live intervals of virtual regs mapped to the physical register we // want to clear (and its aliases). We only spill those that overlap with the // current interval as the rest do not affect its allocation. we also keep @@ -1209,25 +1285,18 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) while (!spillIs.empty()) { LiveInterval *sli = spillIs.back(); spillIs.pop_back(); - DOUT << "\t\t\tspilling(a): " << *sli << '\n'; - earliestStartInterval = - (earliestStartInterval->beginNumber() < sli->beginNumber()) ? - earliestStartInterval : sli; + DEBUG(errs() << "\t\t\tspilling(a): " << *sli << '\n'); + if (sli->beginIndex() < earliestStart) + earliestStart = sli->beginIndex(); std::vector newIs; - if (!NewSpillFramework) { - newIs = li_->addIntervalsForSpills(*sli, spillIs, loopInfo, *vrm_); - } else { - newIs = spiller_->spill(sli); - } + newIs = spiller_->spill(sli, spillIs, &earliestStart); addStackInterval(sli, ls_, li_, mri_, *vrm_); std::copy(newIs.begin(), newIs.end(), std::back_inserter(added)); spilled.insert(sli->reg); } - unsigned earliestStart = earliestStartInterval->beginNumber(); - - DOUT << "\t\trolling back to: " << earliestStart << '\n'; + DEBUG(errs() << "\t\trolling back to: " << earliestStart << '\n'); // Scan handled in reverse order up to the earliest start of a // spilled live interval and undo each one, restoring the state of @@ -1235,9 +1304,9 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) while (!handled_.empty()) { LiveInterval* i = handled_.back(); // If this interval starts before t we are done. - if (i->beginNumber() < earliestStart) + if (!i->empty() && i->beginIndex() < earliestStart) break; - DOUT << "\t\t\tundo changes for: " << *i << '\n'; + DEBUG(errs() << "\t\t\tundo changes for: " << *i << '\n'); handled_.pop_back(); // When undoing a live interval allocation we must know if it is active or @@ -1286,8 +1355,8 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) for (unsigned i = 0, e = handled_.size(); i != e; ++i) { LiveInterval *HI = handled_[i]; if (!HI->expiredAt(earliestStart) && - HI->expiredAt(cur->beginNumber())) { - DOUT << "\t\t\tundo changes for: " << *HI << '\n'; + HI->expiredAt(cur->beginIndex())) { + DEBUG(errs() << "\t\t\tundo changes for: " << *HI << '\n'); active_.push_back(std::make_pair(HI, HI->begin())); assert(!TargetRegisterInfo::isPhysicalRegister(HI->reg)); addRegUse(vrm_->getPhys(HI->reg)); @@ -1306,14 +1375,14 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur) LiveInterval *ReloadLi = added[i]; if (ReloadLi->weight == HUGE_VALF && li_->getApproximateInstructionCount(*ReloadLi) == 0) { - unsigned ReloadIdx = ReloadLi->beginNumber(); + SlotIndex ReloadIdx = ReloadLi->beginIndex(); MachineBasicBlock *ReloadMBB = li_->getMBBFromIndex(ReloadIdx); int ReloadSS = vrm_->getStackSlot(ReloadLi->reg); if (LastReloadMBB == ReloadMBB && LastReloadSS == ReloadSS) { // Last reload of same SS is in the same MBB. We want to try to // allocate both reloads the same register and make sure the reg // isn't clobbered in between if at all possible. - assert(LastReload->beginNumber() < ReloadIdx); + assert(LastReload->beginIndex() < ReloadIdx); NextReloadMap.insert(std::make_pair(LastReload->reg, ReloadLi->reg)); } LastReloadMBB = ReloadMBB; @@ -1349,7 +1418,8 @@ unsigned RALinScan::getFreePhysReg(LiveInterval* cur, // Ignore "downgraded" registers. if (SkipDGRegs && DowngradedRegs.count(Reg)) continue; - if (isRegAvail(Reg)) { + // Skip recently allocated registers. + if (isRegAvail(Reg) && !isRecentlyUsed(Reg)) { FreeReg = Reg; if (FreeReg < inactiveCounts.size()) FreeRegInactiveCount = inactiveCounts[FreeReg]; @@ -1361,9 +1431,12 @@ unsigned RALinScan::getFreePhysReg(LiveInterval* cur, // If there are no free regs, or if this reg has the max inactive count, // return this register. - if (FreeReg == 0 || FreeRegInactiveCount == MaxInactiveCount) + if (FreeReg == 0 || FreeRegInactiveCount == MaxInactiveCount) { + // Remember what register we picked so we can skip it next time. + if (FreeReg != 0) recordRecentlyUsed(FreeReg); return FreeReg; - + } + // Continue scanning the registers, looking for the one with the highest // inactive count. Alkis found that this reduced register pressure very // slightly on X86 (in rev 1.94 of this file), though this should probably be @@ -1374,7 +1447,7 @@ unsigned RALinScan::getFreePhysReg(LiveInterval* cur, if (SkipDGRegs && DowngradedRegs.count(Reg)) continue; if (isRegAvail(Reg) && Reg < inactiveCounts.size() && - FreeRegInactiveCount < inactiveCounts[Reg]) { + FreeRegInactiveCount < inactiveCounts[Reg] && !isRecentlyUsed(Reg)) { FreeReg = Reg; FreeRegInactiveCount = inactiveCounts[Reg]; if (FreeRegInactiveCount == MaxInactiveCount) @@ -1382,6 +1455,9 @@ unsigned RALinScan::getFreePhysReg(LiveInterval* cur, } } + // Remember what register we picked so we can skip it next time. + recordRecentlyUsed(FreeReg); + return FreeReg; } @@ -1416,7 +1492,7 @@ unsigned RALinScan::getFreePhysReg(LiveInterval *cur) { // available first. unsigned Preference = vrm_->getRegAllocPref(cur->reg); if (Preference) { - DOUT << "(preferred: " << tri_->getName(Preference) << ") "; + DEBUG(errs() << "(preferred: " << tri_->getName(Preference) << ") "); if (isRegAvail(Preference) && RC->contains(Preference)) return Preference;