1 //===-- RegAllocGreedy.cpp - greedy register allocator --------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the RAGreedy function pass for register allocation in
13 //===----------------------------------------------------------------------===//
15 #define DEBUG_TYPE "regalloc"
16 #include "AllocationOrder.h"
17 #include "LiveIntervalUnion.h"
18 #include "LiveRangeEdit.h"
19 #include "RegAllocBase.h"
21 #include "SpillPlacement.h"
23 #include "VirtRegMap.h"
24 #include "VirtRegRewriter.h"
25 #include "llvm/Analysis/AliasAnalysis.h"
26 #include "llvm/Function.h"
27 #include "llvm/PassAnalysisSupport.h"
28 #include "llvm/CodeGen/CalcSpillWeights.h"
29 #include "llvm/CodeGen/EdgeBundles.h"
30 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
31 #include "llvm/CodeGen/LiveStackAnalysis.h"
32 #include "llvm/CodeGen/MachineDominators.h"
33 #include "llvm/CodeGen/MachineFunctionPass.h"
34 #include "llvm/CodeGen/MachineLoopInfo.h"
35 #include "llvm/CodeGen/MachineLoopRanges.h"
36 #include "llvm/CodeGen/MachineRegisterInfo.h"
37 #include "llvm/CodeGen/Passes.h"
38 #include "llvm/CodeGen/RegAllocRegistry.h"
39 #include "llvm/CodeGen/RegisterCoalescer.h"
40 #include "llvm/Target/TargetOptions.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Support/raw_ostream.h"
44 #include "llvm/Support/Timer.h"
48 static RegisterRegAlloc greedyRegAlloc("greedy", "greedy register allocator",
49 createGreedyRegisterAllocator);
52 class RAGreedy : public MachineFunctionPass, public RegAllocBase {
55 BitVector ReservedRegs;
60 MachineDominatorTree *DomTree;
61 MachineLoopInfo *Loops;
62 MachineLoopRanges *LoopRanges;
64 SpillPlacement *SpillPlacer;
67 std::auto_ptr<Spiller> SpillerInstance;
68 std::auto_ptr<SplitAnalysis> SA;
72 /// All basic blocks where the current register is live.
73 SmallVector<SpillPlacement::BlockConstraint, 8> SpillConstraints;
75 /// Additional information about basic blocks where the current variable is
76 /// live. Such a block will look like one of these templates:
78 /// 1. | o---x | Internal to block. Variable is only live in this block.
79 /// 2. |---x | Live-in, kill.
80 /// 3. | o---| Def, live-out.
81 /// 4. |---x o---| Live-in, kill, def, live-out.
82 /// 5. |---o---o---| Live-through with uses or defs.
83 /// 6. |-----------| Live-through without uses. Transparent.
86 MachineBasicBlock *MBB;
87 SlotIndex FirstUse; ///< First instr using current reg.
88 SlotIndex LastUse; ///< Last instr using current reg.
89 SlotIndex Kill; ///< Interval end point inside block.
90 SlotIndex Def; ///< Interval start point inside block.
91 bool Uses; ///< Current reg has uses or defs in block.
92 bool LiveThrough; ///< Live in whole block (Templ 5. or 6. above).
93 bool LiveIn; ///< Current reg is live in.
94 bool LiveOut; ///< Current reg is live out.
96 // Per-interference pattern scratch data.
97 bool OverlapEntry; ///< Interference overlaps entering interval.
98 bool OverlapExit; ///< Interference overlaps exiting interval.
101 /// Basic blocks where var is live. This array is parallel to
102 /// SpillConstraints.
103 SmallVector<BlockInfo, 8> LiveBlocks;
108 /// Return the pass name.
109 virtual const char* getPassName() const {
110 return "Greedy Register Allocator";
113 /// RAGreedy analysis usage.
114 virtual void getAnalysisUsage(AnalysisUsage &AU) const;
116 virtual void releaseMemory();
118 virtual Spiller &spiller() { return *SpillerInstance; }
120 virtual float getPriority(LiveInterval *LI);
122 virtual unsigned selectOrSplit(LiveInterval&,
123 SmallVectorImpl<LiveInterval*>&);
125 /// Perform register allocation.
126 virtual bool runOnMachineFunction(MachineFunction &mf);
131 bool checkUncachedInterference(LiveInterval&, unsigned);
132 LiveInterval *getSingleInterference(LiveInterval&, unsigned);
133 bool reassignVReg(LiveInterval &InterferingVReg, unsigned OldPhysReg);
134 bool reassignInterferences(LiveInterval &VirtReg, unsigned PhysReg);
135 float calcInterferenceWeight(LiveInterval&, unsigned);
136 void calcLiveBlockInfo(LiveInterval&);
137 float calcInterferenceInfo(LiveInterval&, unsigned);
138 float calcGlobalSplitCost(const BitVector&);
139 void splitAroundRegion(LiveInterval&, unsigned, const BitVector&,
140 SmallVectorImpl<LiveInterval*>&);
142 unsigned tryReassign(LiveInterval&, AllocationOrder&);
143 unsigned tryRegionSplit(LiveInterval&, AllocationOrder&,
144 SmallVectorImpl<LiveInterval*>&);
145 unsigned trySplit(LiveInterval&, AllocationOrder&,
146 SmallVectorImpl<LiveInterval*>&);
147 unsigned trySpillInterferences(LiveInterval&, AllocationOrder&,
148 SmallVectorImpl<LiveInterval*>&);
150 } // end anonymous namespace
152 char RAGreedy::ID = 0;
154 FunctionPass* llvm::createGreedyRegisterAllocator() {
155 return new RAGreedy();
158 RAGreedy::RAGreedy(): MachineFunctionPass(ID) {
159 initializeSlotIndexesPass(*PassRegistry::getPassRegistry());
160 initializeLiveIntervalsPass(*PassRegistry::getPassRegistry());
161 initializeSlotIndexesPass(*PassRegistry::getPassRegistry());
162 initializeStrongPHIEliminationPass(*PassRegistry::getPassRegistry());
163 initializeRegisterCoalescerAnalysisGroup(*PassRegistry::getPassRegistry());
164 initializeCalculateSpillWeightsPass(*PassRegistry::getPassRegistry());
165 initializeLiveStacksPass(*PassRegistry::getPassRegistry());
166 initializeMachineDominatorTreePass(*PassRegistry::getPassRegistry());
167 initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry());
168 initializeMachineLoopRangesPass(*PassRegistry::getPassRegistry());
169 initializeVirtRegMapPass(*PassRegistry::getPassRegistry());
170 initializeEdgeBundlesPass(*PassRegistry::getPassRegistry());
171 initializeSpillPlacementPass(*PassRegistry::getPassRegistry());
174 void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
175 AU.setPreservesCFG();
176 AU.addRequired<AliasAnalysis>();
177 AU.addPreserved<AliasAnalysis>();
178 AU.addRequired<LiveIntervals>();
179 AU.addRequired<SlotIndexes>();
180 AU.addPreserved<SlotIndexes>();
182 AU.addRequiredID(StrongPHIEliminationID);
183 AU.addRequiredTransitive<RegisterCoalescer>();
184 AU.addRequired<CalculateSpillWeights>();
185 AU.addRequired<LiveStacks>();
186 AU.addPreserved<LiveStacks>();
187 AU.addRequired<MachineDominatorTree>();
188 AU.addPreserved<MachineDominatorTree>();
189 AU.addRequired<MachineLoopInfo>();
190 AU.addPreserved<MachineLoopInfo>();
191 AU.addRequired<MachineLoopRanges>();
192 AU.addPreserved<MachineLoopRanges>();
193 AU.addRequired<VirtRegMap>();
194 AU.addPreserved<VirtRegMap>();
195 AU.addRequired<EdgeBundles>();
196 AU.addRequired<SpillPlacement>();
197 MachineFunctionPass::getAnalysisUsage(AU);
200 void RAGreedy::releaseMemory() {
201 SpillerInstance.reset(0);
202 RegAllocBase::releaseMemory();
205 float RAGreedy::getPriority(LiveInterval *LI) {
206 float Priority = LI->weight;
208 // Prioritize hinted registers so they are allocated first.
209 std::pair<unsigned, unsigned> Hint;
210 if (Hint.first || Hint.second) {
211 // The hint can be target specific, a virtual register, or a physreg.
214 // Prefer physreg hints above anything else.
215 if (Hint.first == 0 && TargetRegisterInfo::isPhysicalRegister(Hint.second))
222 //===----------------------------------------------------------------------===//
223 // Register Reassignment
224 //===----------------------------------------------------------------------===//
226 // Check interference without using the cache.
227 bool RAGreedy::checkUncachedInterference(LiveInterval &VirtReg,
229 for (const unsigned *AliasI = TRI->getOverlaps(PhysReg); *AliasI; ++AliasI) {
230 LiveIntervalUnion::Query subQ(&VirtReg, &PhysReg2LiveUnion[*AliasI]);
231 if (subQ.checkInterference())
237 /// getSingleInterference - Return the single interfering virtual register
238 /// assigned to PhysReg. Return 0 if more than one virtual register is
240 LiveInterval *RAGreedy::getSingleInterference(LiveInterval &VirtReg,
242 // Check physreg and aliases.
243 LiveInterval *Interference = 0;
244 for (const unsigned *AliasI = TRI->getOverlaps(PhysReg); *AliasI; ++AliasI) {
245 LiveIntervalUnion::Query &Q = query(VirtReg, *AliasI);
246 if (Q.checkInterference()) {
249 Q.collectInterferingVRegs(1);
250 if (!Q.seenAllInterferences())
252 Interference = Q.interferingVRegs().front();
258 // Attempt to reassign this virtual register to a different physical register.
260 // FIXME: we are not yet caching these "second-level" interferences discovered
261 // in the sub-queries. These interferences can change with each call to
262 // selectOrSplit. However, we could implement a "may-interfere" cache that
263 // could be conservatively dirtied when we reassign or split.
265 // FIXME: This may result in a lot of alias queries. We could summarize alias
266 // live intervals in their parent register's live union, but it's messy.
267 bool RAGreedy::reassignVReg(LiveInterval &InterferingVReg,
268 unsigned WantedPhysReg) {
269 assert(TargetRegisterInfo::isVirtualRegister(InterferingVReg.reg) &&
270 "Can only reassign virtual registers");
271 assert(TRI->regsOverlap(WantedPhysReg, VRM->getPhys(InterferingVReg.reg)) &&
272 "inconsistent phys reg assigment");
274 AllocationOrder Order(InterferingVReg.reg, *VRM, ReservedRegs);
275 while (unsigned PhysReg = Order.next()) {
276 // Don't reassign to a WantedPhysReg alias.
277 if (TRI->regsOverlap(PhysReg, WantedPhysReg))
280 if (checkUncachedInterference(InterferingVReg, PhysReg))
283 // Reassign the interfering virtual reg to this physical reg.
284 unsigned OldAssign = VRM->getPhys(InterferingVReg.reg);
285 DEBUG(dbgs() << "reassigning: " << InterferingVReg << " from " <<
286 TRI->getName(OldAssign) << " to " << TRI->getName(PhysReg) << '\n');
287 PhysReg2LiveUnion[OldAssign].extract(InterferingVReg);
288 VRM->clearVirt(InterferingVReg.reg);
289 VRM->assignVirt2Phys(InterferingVReg.reg, PhysReg);
290 PhysReg2LiveUnion[PhysReg].unify(InterferingVReg);
297 /// reassignInterferences - Reassign all interferences to different physical
298 /// registers such that Virtreg can be assigned to PhysReg.
299 /// Currently this only works with a single interference.
300 /// @param VirtReg Currently unassigned virtual register.
301 /// @param PhysReg Physical register to be cleared.
302 /// @return True on success, false if nothing was changed.
303 bool RAGreedy::reassignInterferences(LiveInterval &VirtReg, unsigned PhysReg) {
304 LiveInterval *InterferingVReg = getSingleInterference(VirtReg, PhysReg);
305 if (!InterferingVReg)
307 if (TargetRegisterInfo::isPhysicalRegister(InterferingVReg->reg))
309 return reassignVReg(*InterferingVReg, PhysReg);
312 /// tryReassign - Try to reassign interferences to different physregs.
313 /// @param VirtReg Currently unassigned virtual register.
314 /// @param Order Physregs to try.
315 /// @return Physreg to assign VirtReg, or 0.
316 unsigned RAGreedy::tryReassign(LiveInterval &VirtReg, AllocationOrder &Order) {
317 NamedRegionTimer T("Reassign", TimerGroupName, TimePassesIsEnabled);
319 while (unsigned PhysReg = Order.next())
320 if (reassignInterferences(VirtReg, PhysReg))
326 //===----------------------------------------------------------------------===//
328 //===----------------------------------------------------------------------===//
330 /// calcLiveBlockInfo - Fill the LiveBlocks array with information about blocks
331 /// where VirtReg is live.
332 /// The SpillConstraints array is minimally initialized with MBB->getNumber().
333 void RAGreedy::calcLiveBlockInfo(LiveInterval &VirtReg) {
335 SpillConstraints.clear();
337 assert(!VirtReg.empty() && "Cannot allocate an empty interval");
338 LiveInterval::const_iterator LVI = VirtReg.begin();
339 LiveInterval::const_iterator LVE = VirtReg.end();
341 SmallVectorImpl<SlotIndex>::const_iterator UseI, UseE;
342 UseI = SA->UseSlots.begin();
343 UseE = SA->UseSlots.end();
345 // Loop over basic blocks where VirtReg is live.
346 MachineFunction::iterator MFI = Indexes->getMBBFromIndex(LVI->start);
348 // Block constraints depend on the interference pattern.
349 // Just allocate them here, don't compute anything.
350 SpillPlacement::BlockConstraint BC;
351 BC.Number = MFI->getNumber();
352 SpillConstraints.push_back(BC);
356 SlotIndex Start, Stop;
357 tie(Start, Stop) = Indexes->getMBBRange(BI.MBB);
359 // LVI is the first live segment overlapping MBB.
360 BI.LiveIn = LVI->start <= Start;
364 // Find the first and last uses in the block.
365 BI.Uses = SA->hasUses(MFI);
366 if (BI.Uses && UseI != UseE) {
368 assert(BI.FirstUse >= Start);
370 while (UseI != UseE && *UseI < Stop);
371 BI.LastUse = UseI[-1];
372 assert(BI.LastUse < Stop);
375 // Look for gaps in the live range.
378 while (LVI->end < Stop) {
379 SlotIndex LastStop = LVI->end;
380 if (++LVI == LVE || LVI->start >= Stop) {
385 if (LastStop < LVI->start) {
392 // Don't set LiveThrough when the block has a gap.
393 BI.LiveThrough = !hasGap && BI.LiveIn && BI.LiveOut;
394 LiveBlocks.push_back(BI);
396 // LVI is now at LVE or LVI->end >= Stop.
400 // Live segment ends exactly at Stop. Move to the next segment.
401 if (LVI->end == Stop && ++LVI == LVE)
404 // Pick the next basic block.
405 if (LVI->start < Stop)
408 MFI = Indexes->getMBBFromIndex(LVI->start);
412 /// calcInterferenceInfo - Compute per-block outgoing and ingoing constraints
413 /// when considering interference from PhysReg. Also compute an optimistic local
414 /// cost of this interference pattern.
416 /// The final cost of a split is the local cost + global cost of preferences
417 /// broken by SpillPlacement.
419 float RAGreedy::calcInterferenceInfo(LiveInterval &VirtReg, unsigned PhysReg) {
420 // Reset interference dependent info.
421 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
422 BlockInfo &BI = LiveBlocks[i];
423 SpillPlacement::BlockConstraint &BC = SpillConstraints[i];
424 BC.Entry = (BI.Uses && BI.LiveIn) ?
425 SpillPlacement::PrefReg : SpillPlacement::DontCare;
426 BC.Exit = (BI.Uses && BI.LiveOut) ?
427 SpillPlacement::PrefReg : SpillPlacement::DontCare;
428 BI.OverlapEntry = BI.OverlapExit = false;
431 // Add interference info from each PhysReg alias.
432 for (const unsigned *AI = TRI->getOverlaps(PhysReg); *AI; ++AI) {
433 if (!query(VirtReg, *AI).checkInterference())
435 DEBUG(PhysReg2LiveUnion[*AI].print(dbgs(), TRI));
436 LiveIntervalUnion::SegmentIter IntI =
437 PhysReg2LiveUnion[*AI].find(VirtReg.beginIndex());
441 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
442 BlockInfo &BI = LiveBlocks[i];
443 SpillPlacement::BlockConstraint &BC = SpillConstraints[i];
444 SlotIndex Start, Stop;
445 tie(Start, Stop) = Indexes->getMBBRange(BI.MBB);
447 // Skip interference-free blocks.
448 if (IntI.start() >= Stop)
451 // Handle transparent blocks with interference separately.
452 // Transparent blocks never incur any fixed cost.
453 if (BI.LiveThrough && !BI.Uses) {
454 // Check if interference is live-in - force spill.
455 if (BC.Entry != SpillPlacement::MustSpill) {
456 BC.Entry = SpillPlacement::PrefSpill;
457 IntI.advanceTo(Start);
458 if (IntI.valid() && IntI.start() <= Start)
459 BC.Entry = SpillPlacement::MustSpill;
462 // Check if interference is live-out - force spill.
463 if (BC.Exit != SpillPlacement::MustSpill) {
464 BC.Exit = SpillPlacement::PrefSpill;
465 IntI.advanceTo(Stop);
466 if (IntI.valid() && IntI.start() < Stop)
467 BC.Exit = SpillPlacement::MustSpill;
470 // Nothing more to do for this transparent block.
476 // Now we only have blocks with uses left.
477 // Check if the interference overlaps the uses.
478 assert(BI.Uses && "Non-transparent block without any uses");
480 // Check interference on entry.
481 if (BI.LiveIn && BC.Entry != SpillPlacement::MustSpill) {
482 IntI.advanceTo(Start);
486 // Interference is live-in - force spill.
487 if (IntI.start() <= Start)
488 BC.Entry = SpillPlacement::MustSpill;
489 // Not live in, but before the first use.
490 else if (IntI.start() < BI.FirstUse)
491 BC.Entry = SpillPlacement::PrefSpill;
494 // Does interference overlap the uses in the entry segment
496 if (BI.LiveIn && !BI.OverlapEntry) {
497 IntI.advanceTo(BI.FirstUse);
500 // A live-through interval has no kill.
501 // Check [FirstUse;LastUse) instead.
502 if (IntI.start() < (BI.LiveThrough ? BI.LastUse : BI.Kill))
503 BI.OverlapEntry = true;
506 // Does interference overlap the uses in the exit segment [Def;LastUse)?
507 if (BI.LiveOut && !BI.LiveThrough && !BI.OverlapExit) {
508 IntI.advanceTo(BI.Def);
511 if (IntI.start() < BI.LastUse)
512 BI.OverlapExit = true;
515 // Check interference on exit.
516 if (BI.LiveOut && BC.Exit != SpillPlacement::MustSpill) {
517 // Check interference between LastUse and Stop.
518 if (BC.Exit != SpillPlacement::PrefSpill) {
519 IntI.advanceTo(BI.LastUse);
522 if (IntI.start() < Stop)
523 BC.Exit = SpillPlacement::PrefSpill;
525 // Is the interference live-out?
526 IntI.advanceTo(Stop);
529 if (IntI.start() < Stop)
530 BC.Exit = SpillPlacement::MustSpill;
535 // Accumulate a local cost of this interference pattern.
537 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
538 BlockInfo &BI = LiveBlocks[i];
541 SpillPlacement::BlockConstraint &BC = SpillConstraints[i];
542 unsigned Inserts = 0;
544 // Do we need spill code for the entry segment?
546 Inserts += BI.OverlapEntry || BC.Entry != SpillPlacement::PrefReg;
548 // For the exit segment?
550 Inserts += BI.OverlapExit || BC.Exit != SpillPlacement::PrefReg;
552 // The local cost of spill code in this block is the block frequency times
553 // the number of spill instructions inserted.
555 LocalCost += Inserts * SpillPlacer->getBlockFrequency(BI.MBB);
557 DEBUG(dbgs() << "Local cost of " << PrintReg(PhysReg, TRI) << " = "
558 << LocalCost << '\n');
562 /// calcGlobalSplitCost - Return the global split cost of following the split
563 /// pattern in LiveBundles. This cost should be added to the local cost of the
564 /// interference pattern in SpillConstraints.
566 float RAGreedy::calcGlobalSplitCost(const BitVector &LiveBundles) {
567 float GlobalCost = 0;
568 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
569 SpillPlacement::BlockConstraint &BC = SpillConstraints[i];
570 unsigned Inserts = 0;
571 // Broken entry preference?
572 Inserts += LiveBundles[Bundles->getBundle(BC.Number, 0)] !=
573 (BC.Entry == SpillPlacement::PrefReg);
574 // Broken exit preference?
575 Inserts += LiveBundles[Bundles->getBundle(BC.Number, 1)] !=
576 (BC.Exit == SpillPlacement::PrefReg);
578 GlobalCost += Inserts * SpillPlacer->getBlockFrequency(LiveBlocks[i].MBB);
580 DEBUG(dbgs() << "Global cost = " << GlobalCost << '\n');
584 /// splitAroundRegion - Split VirtReg around the region determined by
585 /// LiveBundles. Make an effort to avoid interference from PhysReg.
587 /// The 'register' interval is going to contain as many uses as possible while
588 /// avoiding interference. The 'stack' interval is the complement constructed by
589 /// SplitEditor. It will contain the rest.
591 void RAGreedy::splitAroundRegion(LiveInterval &VirtReg, unsigned PhysReg,
592 const BitVector &LiveBundles,
593 SmallVectorImpl<LiveInterval*> &NewVRegs) {
595 dbgs() << "Splitting around region for " << PrintReg(PhysReg, TRI)
597 for (int i = LiveBundles.find_first(); i>=0; i = LiveBundles.find_next(i))
598 dbgs() << " EB#" << i;
602 // First compute interference ranges in the live blocks.
603 typedef std::pair<SlotIndex, SlotIndex> IndexPair;
604 SmallVector<IndexPair, 8> InterferenceRanges;
605 InterferenceRanges.resize(LiveBlocks.size());
606 for (const unsigned *AI = TRI->getOverlaps(PhysReg); *AI; ++AI) {
607 if (!query(VirtReg, *AI).checkInterference())
609 LiveIntervalUnion::SegmentIter IntI =
610 PhysReg2LiveUnion[*AI].find(VirtReg.beginIndex());
613 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
614 BlockInfo &BI = LiveBlocks[i];
617 IndexPair &IP = InterferenceRanges[i];
618 SlotIndex Start, Stop;
619 tie(Start, Stop) = Indexes->getMBBRange(BI.MBB);
620 // Skip interference-free blocks.
621 if (IntI.start() >= Stop)
624 // First interference in block.
626 IntI.advanceTo(Start);
629 if (!IP.first.isValid() || IntI.start() < IP.first)
630 IP.first = IntI.start();
633 // Last interference in block.
635 IntI.advanceTo(Stop);
636 if (!IntI.valid() || IntI.start() >= Stop)
638 if (!IP.second.isValid() || IntI.stop() > IP.second)
639 IP.second = IntI.stop();
644 SmallVector<LiveInterval*, 4> SpillRegs;
645 LiveRangeEdit LREdit(VirtReg, NewVRegs, SpillRegs);
646 SplitEditor SE(*SA, *LIS, *VRM, *DomTree, LREdit);
648 // Create the main cross-block interval.
651 // First add all defs that are live out of a block.
652 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
653 BlockInfo &BI = LiveBlocks[i];
654 bool RegIn = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 0)];
655 bool RegOut = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 1)];
657 // Should the register be live out?
658 if (!BI.LiveOut || !RegOut)
661 IndexPair &IP = InterferenceRanges[i];
662 SlotIndex Start, Stop;
663 tie(Start, Stop) = Indexes->getMBBRange(BI.MBB);
665 DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " -> EB#"
666 << Bundles->getBundle(BI.MBB->getNumber(), 1));
668 // Check interference leaving the block.
669 if (!IP.second.isValid() || IP.second < Start) {
670 // Block is interference-free.
671 DEBUG(dbgs() << ", no interference");
673 assert(BI.LiveThrough && "No uses, but not live through block?");
674 // Block is live-through without interference.
675 DEBUG(dbgs() << ", no uses"
676 << (RegIn ? ", live-through.\n" : ", stack in.\n"));
678 SE.enterIntvAtEnd(*BI.MBB);
681 if (!BI.LiveThrough) {
682 DEBUG(dbgs() << ", not live-through.\n");
683 SE.enterIntvBefore(BI.Def);
684 SE.useIntv(BI.Def, Stop);
688 // Block is live-through, but entry bundle is on the stack.
689 // Reload just before the first use.
690 DEBUG(dbgs() << ", not live-in, enter before first use.\n");
691 SE.enterIntvBefore(BI.FirstUse);
692 SE.useIntv(BI.FirstUse, Stop);
695 DEBUG(dbgs() << ", live-through.\n");
699 // Block has interference.
700 DEBUG(dbgs() << ", interference to " << IP.second);
702 // No uses in block, avoid interference by reloading as late as possible.
703 DEBUG(dbgs() << ", no uses.\n");
704 SE.enterIntvAtEnd(*BI.MBB);
707 if (IP.second < BI.LastUse) {
708 // There are interference-free uses at the end of the block.
709 // Find the first use that can get the live-out register.
710 SlotIndex Use = *std::lower_bound(SA->UseSlots.begin(), SA->UseSlots.end(),
712 DEBUG(dbgs() << ", free use at " << Use << ".\n");
713 assert(Use > IP.second && Use <= BI.LastUse);
714 SE.enterIntvBefore(Use);
715 SE.useIntv(Use, Stop);
719 // Interference is after the last use.
720 DEBUG(dbgs() << " after last use.\n");
721 SE.enterIntvAtEnd(*BI.MBB);
724 // Now all defs leading to live bundles are handled, do everything else.
725 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
726 BlockInfo &BI = LiveBlocks[i];
727 bool RegIn = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 0)];
728 bool RegOut = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 1)];
730 // Is the register live-in?
731 if (!BI.LiveIn || !RegIn)
734 // We have an incoming register. Check for interference.
735 IndexPair &IP = InterferenceRanges[i];
736 SlotIndex Start, Stop;
737 tie(Start, Stop) = Indexes->getMBBRange(BI.MBB);
739 DEBUG(dbgs() << "EB#" << Bundles->getBundle(BI.MBB->getNumber(), 0)
740 << " -> BB#" << BI.MBB->getNumber());
742 // Check interference entering the block.
743 if (!IP.first.isValid() || IP.first > Stop) {
744 // Block is interference-free.
745 DEBUG(dbgs() << ", no interference");
747 assert(BI.LiveThrough && "No uses, but not live through block?");
748 // Block is live-through without interference.
750 DEBUG(dbgs() << ", no uses, live-through.\n");
751 SE.useIntv(Start, Stop);
753 DEBUG(dbgs() << ", no uses, stack-out.\n");
754 SE.leaveIntvAtTop(*BI.MBB);
758 if (!BI.LiveThrough) {
759 DEBUG(dbgs() << ", killed in block.\n");
760 SE.useIntv(Start, BI.Kill.getBoundaryIndex());
761 SE.leaveIntvAfter(BI.Kill);
765 // Block is live-through, but exit bundle is on the stack.
766 // Spill immediately after the last use.
767 DEBUG(dbgs() << ", uses, stack-out.\n");
768 SE.useIntv(Start, BI.LastUse.getBoundaryIndex());
769 SE.leaveIntvAfter(BI.LastUse);
772 // Register is live-through.
773 DEBUG(dbgs() << ", uses, live-through.\n");
774 SE.useIntv(Start, Stop);
778 // Block has interference.
779 DEBUG(dbgs() << ", interference from " << IP.first);
781 // No uses in block, avoid interference by spilling as soon as possible.
782 DEBUG(dbgs() << ", no uses.\n");
783 SE.leaveIntvAtTop(*BI.MBB);
786 if (IP.first > BI.FirstUse) {
787 // There are interference-free uses at the beginning of the block.
788 // Find the last use that can get the register.
789 SlotIndex Use = std::lower_bound(SA->UseSlots.begin(), SA->UseSlots.end(),
791 DEBUG(dbgs() << ", free use at " << Use << ".\n");
792 Use = Use.getBoundaryIndex();
793 assert(Use >= BI.FirstUse && Use < IP.first);
794 SE.useIntv(Start, Use);
795 SE.leaveIntvAfter(Use);
799 // Interference is before the first use.
800 DEBUG(dbgs() << " before first use.\n");
801 SE.leaveIntvAtTop(*BI.MBB);
806 // FIXME: Should we be more aggressive about splitting the stack region into
807 // per-block segments? The current approach allows the stack region to
808 // separate into connected components. Some components may be allocatable.
812 MF->verify(this, "After splitting live range around region");
815 unsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
816 SmallVectorImpl<LiveInterval*> &NewVRegs) {
817 calcLiveBlockInfo(VirtReg);
818 BitVector LiveBundles, BestBundles;
820 unsigned BestReg = 0;
822 while (unsigned PhysReg = Order.next()) {
823 float Cost = calcInterferenceInfo(VirtReg, PhysReg);
824 if (BestReg && Cost >= BestCost)
827 SpillPlacer->placeSpills(SpillConstraints, LiveBundles);
828 // No live bundles, defer to splitSingleBlocks().
829 if (!LiveBundles.any())
832 Cost += calcGlobalSplitCost(LiveBundles);
833 if (!BestReg || Cost < BestCost) {
836 BestBundles.swap(LiveBundles);
843 splitAroundRegion(VirtReg, BestReg, BestBundles, NewVRegs);
848 //===----------------------------------------------------------------------===//
849 // Live Range Splitting
850 //===----------------------------------------------------------------------===//
852 /// trySplit - Try to split VirtReg or one of its interferences, making it
854 /// @return Physreg when VirtReg may be assigned and/or new NewVRegs.
855 unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
856 SmallVectorImpl<LiveInterval*>&NewVRegs) {
857 NamedRegionTimer T("Splitter", TimerGroupName, TimePassesIsEnabled);
858 SA->analyze(&VirtReg);
860 // Don't attempt splitting on local intervals for now. TBD.
861 if (LIS->intervalIsInOneMBB(VirtReg))
864 // First try to split around a region spanning multiple blocks.
865 unsigned PhysReg = tryRegionSplit(VirtReg, Order, NewVRegs);
866 if (PhysReg || !NewVRegs.empty())
869 // Then isolate blocks with multiple uses.
870 SplitAnalysis::BlockPtrSet Blocks;
871 if (SA->getMultiUseBlocks(Blocks)) {
872 SmallVector<LiveInterval*, 4> SpillRegs;
873 LiveRangeEdit LREdit(VirtReg, NewVRegs, SpillRegs);
874 SplitEditor(*SA, *LIS, *VRM, *DomTree, LREdit).splitSingleBlocks(Blocks);
877 // Don't assign any physregs.
882 //===----------------------------------------------------------------------===//
884 //===----------------------------------------------------------------------===//
886 /// calcInterferenceWeight - Calculate the combined spill weight of
887 /// interferences when assigning VirtReg to PhysReg.
888 float RAGreedy::calcInterferenceWeight(LiveInterval &VirtReg, unsigned PhysReg){
890 for (const unsigned *AI = TRI->getOverlaps(PhysReg); *AI; ++AI) {
891 LiveIntervalUnion::Query &Q = query(VirtReg, *AI);
892 Q.collectInterferingVRegs();
893 if (Q.seenUnspillableVReg())
895 for (unsigned i = 0, e = Q.interferingVRegs().size(); i != e; ++i)
896 Sum += Q.interferingVRegs()[i]->weight;
901 /// trySpillInterferences - Try to spill interfering registers instead of the
902 /// current one. Only do it if the accumulated spill weight is smaller than the
903 /// current spill weight.
904 unsigned RAGreedy::trySpillInterferences(LiveInterval &VirtReg,
905 AllocationOrder &Order,
906 SmallVectorImpl<LiveInterval*> &NewVRegs) {
907 NamedRegionTimer T("Spill Interference", TimerGroupName, TimePassesIsEnabled);
908 unsigned BestPhys = 0;
909 float BestWeight = 0;
912 while (unsigned PhysReg = Order.next()) {
913 float Weight = calcInterferenceWeight(VirtReg, PhysReg);
914 if (Weight == HUGE_VALF || Weight >= VirtReg.weight)
916 if (!BestPhys || Weight < BestWeight)
917 BestPhys = PhysReg, BestWeight = Weight;
920 // No candidates found.
924 // Collect all interfering registers.
925 SmallVector<LiveInterval*, 8> Spills;
926 for (const unsigned *AI = TRI->getOverlaps(BestPhys); *AI; ++AI) {
927 LiveIntervalUnion::Query &Q = query(VirtReg, *AI);
928 Spills.append(Q.interferingVRegs().begin(), Q.interferingVRegs().end());
929 for (unsigned i = 0, e = Q.interferingVRegs().size(); i != e; ++i) {
930 LiveInterval *VReg = Q.interferingVRegs()[i];
931 PhysReg2LiveUnion[*AI].extract(*VReg);
932 VRM->clearVirt(VReg->reg);
937 DEBUG(dbgs() << "spilling " << Spills.size() << " interferences with weight "
938 << BestWeight << '\n');
939 for (unsigned i = 0, e = Spills.size(); i != e; ++i)
940 spiller().spill(Spills[i], NewVRegs, Spills);
945 //===----------------------------------------------------------------------===//
947 //===----------------------------------------------------------------------===//
949 unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg,
950 SmallVectorImpl<LiveInterval*> &NewVRegs) {
951 // First try assigning a free register.
952 AllocationOrder Order(VirtReg.reg, *VRM, ReservedRegs);
953 while (unsigned PhysReg = Order.next()) {
954 if (!checkPhysRegInterference(VirtReg, PhysReg))
958 // Try to reassign interferences.
959 if (unsigned PhysReg = tryReassign(VirtReg, Order))
962 assert(NewVRegs.empty() && "Cannot append to existing NewVRegs");
964 // Try splitting VirtReg or interferences.
965 unsigned PhysReg = trySplit(VirtReg, Order, NewVRegs);
966 if (PhysReg || !NewVRegs.empty())
969 // Try to spill another interfering reg with less spill weight.
970 PhysReg = trySpillInterferences(VirtReg, Order, NewVRegs);
974 // Finally spill VirtReg itself.
975 NamedRegionTimer T("Spiller", TimerGroupName, TimePassesIsEnabled);
976 SmallVector<LiveInterval*, 1> pendingSpills;
977 spiller().spill(&VirtReg, NewVRegs, pendingSpills);
979 // The live virtual register requesting allocation was spilled, so tell
980 // the caller not to allocate anything during this round.
984 bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
985 DEBUG(dbgs() << "********** GREEDY REGISTER ALLOCATION **********\n"
986 << "********** Function: "
987 << ((Value*)mf.getFunction())->getName() << '\n');
991 MF->verify(this, "Before greedy register allocator");
993 RegAllocBase::init(getAnalysis<VirtRegMap>(), getAnalysis<LiveIntervals>());
994 Indexes = &getAnalysis<SlotIndexes>();
995 DomTree = &getAnalysis<MachineDominatorTree>();
996 ReservedRegs = TRI->getReservedRegs(*MF);
997 SpillerInstance.reset(createInlineSpiller(*this, *MF, *VRM));
998 Loops = &getAnalysis<MachineLoopInfo>();
999 LoopRanges = &getAnalysis<MachineLoopRanges>();
1000 Bundles = &getAnalysis<EdgeBundles>();
1001 SpillPlacer = &getAnalysis<SpillPlacement>();
1003 SA.reset(new SplitAnalysis(*MF, *LIS, *Loops));
1010 NamedRegionTimer T("Rewriter", TimerGroupName, TimePassesIsEnabled);
1011 std::auto_ptr<VirtRegRewriter> rewriter(createVirtRegRewriter());
1012 rewriter->runOnMachineFunction(*MF, *VRM, LIS);
1015 // The pass output is in VirtRegMap. Release all the transient data.