1 //===-- RegAllocGreedy.cpp - greedy register allocator --------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the RAGreedy function pass for register allocation in
13 //===----------------------------------------------------------------------===//
15 #define DEBUG_TYPE "regalloc"
16 #include "AllocationOrder.h"
17 #include "LiveIntervalUnion.h"
18 #include "LiveRangeEdit.h"
19 #include "RegAllocBase.h"
21 #include "SpillPlacement.h"
23 #include "VirtRegMap.h"
24 #include "VirtRegRewriter.h"
25 #include "llvm/Analysis/AliasAnalysis.h"
26 #include "llvm/Function.h"
27 #include "llvm/PassAnalysisSupport.h"
28 #include "llvm/CodeGen/CalcSpillWeights.h"
29 #include "llvm/CodeGen/EdgeBundles.h"
30 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
31 #include "llvm/CodeGen/LiveStackAnalysis.h"
32 #include "llvm/CodeGen/MachineDominators.h"
33 #include "llvm/CodeGen/MachineFunctionPass.h"
34 #include "llvm/CodeGen/MachineLoopInfo.h"
35 #include "llvm/CodeGen/MachineLoopRanges.h"
36 #include "llvm/CodeGen/MachineRegisterInfo.h"
37 #include "llvm/CodeGen/Passes.h"
38 #include "llvm/CodeGen/RegAllocRegistry.h"
39 #include "llvm/CodeGen/RegisterCoalescer.h"
40 #include "llvm/Target/TargetOptions.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Support/raw_ostream.h"
44 #include "llvm/Support/Timer.h"
48 static RegisterRegAlloc greedyRegAlloc("greedy", "greedy register allocator",
49 createGreedyRegisterAllocator);
52 class RAGreedy : public MachineFunctionPass, public RegAllocBase {
55 BitVector ReservedRegs;
60 MachineDominatorTree *DomTree;
61 MachineLoopInfo *Loops;
62 MachineLoopRanges *LoopRanges;
64 SpillPlacement *SpillPlacer;
67 std::auto_ptr<Spiller> SpillerInstance;
68 std::auto_ptr<SplitAnalysis> SA;
72 /// All basic blocks where the current register is live.
73 SmallVector<SpillPlacement::BlockConstraint, 8> SpillConstraints;
75 /// Additional information about basic blocks where the current variable is
76 /// live. Such a block will look like one of these templates:
78 /// 1. | o---x | Internal to block. Variable is only live in this block.
79 /// 2. |---x | Live-in, kill.
80 /// 3. | o---| Def, live-out.
81 /// 4. |---x o---| Live-in, kill, def, live-out.
82 /// 5. |---o---o---| Live-through with uses or defs.
83 /// 6. |-----------| Live-through without uses. Transparent.
86 MachineBasicBlock *MBB;
87 SlotIndex FirstUse; ///< First instr using current reg.
88 SlotIndex LastUse; ///< Last instr using current reg.
89 SlotIndex Kill; ///< Interval end point inside block.
90 SlotIndex Def; ///< Interval start point inside block.
91 /// Last possible point for splitting live ranges.
92 SlotIndex LastSplitPoint;
93 bool Uses; ///< Current reg has uses or defs in block.
94 bool LiveThrough; ///< Live in whole block (Templ 5. or 6. above).
95 bool LiveIn; ///< Current reg is live in.
96 bool LiveOut; ///< Current reg is live out.
98 // Per-interference pattern scratch data.
99 bool OverlapEntry; ///< Interference overlaps entering interval.
100 bool OverlapExit; ///< Interference overlaps exiting interval.
103 /// Basic blocks where var is live. This array is parallel to
104 /// SpillConstraints.
105 SmallVector<BlockInfo, 8> LiveBlocks;
110 /// Return the pass name.
111 virtual const char* getPassName() const {
112 return "Greedy Register Allocator";
115 /// RAGreedy analysis usage.
116 virtual void getAnalysisUsage(AnalysisUsage &AU) const;
118 virtual void releaseMemory();
120 virtual Spiller &spiller() { return *SpillerInstance; }
122 virtual float getPriority(LiveInterval *LI);
124 virtual unsigned selectOrSplit(LiveInterval&,
125 SmallVectorImpl<LiveInterval*>&);
127 /// Perform register allocation.
128 virtual bool runOnMachineFunction(MachineFunction &mf);
133 bool checkUncachedInterference(LiveInterval&, unsigned);
134 LiveInterval *getSingleInterference(LiveInterval&, unsigned);
135 bool reassignVReg(LiveInterval &InterferingVReg, unsigned OldPhysReg);
136 bool reassignInterferences(LiveInterval &VirtReg, unsigned PhysReg);
137 float calcInterferenceWeight(LiveInterval&, unsigned);
138 void calcLiveBlockInfo(LiveInterval&);
139 float calcInterferenceInfo(LiveInterval&, unsigned);
140 float calcGlobalSplitCost(const BitVector&);
141 void splitAroundRegion(LiveInterval&, unsigned, const BitVector&,
142 SmallVectorImpl<LiveInterval*>&);
144 unsigned tryReassign(LiveInterval&, AllocationOrder&);
145 unsigned tryRegionSplit(LiveInterval&, AllocationOrder&,
146 SmallVectorImpl<LiveInterval*>&);
147 unsigned trySplit(LiveInterval&, AllocationOrder&,
148 SmallVectorImpl<LiveInterval*>&);
149 unsigned trySpillInterferences(LiveInterval&, AllocationOrder&,
150 SmallVectorImpl<LiveInterval*>&);
152 } // end anonymous namespace
154 char RAGreedy::ID = 0;
156 FunctionPass* llvm::createGreedyRegisterAllocator() {
157 return new RAGreedy();
160 RAGreedy::RAGreedy(): MachineFunctionPass(ID) {
161 initializeSlotIndexesPass(*PassRegistry::getPassRegistry());
162 initializeLiveIntervalsPass(*PassRegistry::getPassRegistry());
163 initializeSlotIndexesPass(*PassRegistry::getPassRegistry());
164 initializeStrongPHIEliminationPass(*PassRegistry::getPassRegistry());
165 initializeRegisterCoalescerAnalysisGroup(*PassRegistry::getPassRegistry());
166 initializeCalculateSpillWeightsPass(*PassRegistry::getPassRegistry());
167 initializeLiveStacksPass(*PassRegistry::getPassRegistry());
168 initializeMachineDominatorTreePass(*PassRegistry::getPassRegistry());
169 initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry());
170 initializeMachineLoopRangesPass(*PassRegistry::getPassRegistry());
171 initializeVirtRegMapPass(*PassRegistry::getPassRegistry());
172 initializeEdgeBundlesPass(*PassRegistry::getPassRegistry());
173 initializeSpillPlacementPass(*PassRegistry::getPassRegistry());
176 void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
177 AU.setPreservesCFG();
178 AU.addRequired<AliasAnalysis>();
179 AU.addPreserved<AliasAnalysis>();
180 AU.addRequired<LiveIntervals>();
181 AU.addRequired<SlotIndexes>();
182 AU.addPreserved<SlotIndexes>();
184 AU.addRequiredID(StrongPHIEliminationID);
185 AU.addRequiredTransitive<RegisterCoalescer>();
186 AU.addRequired<CalculateSpillWeights>();
187 AU.addRequired<LiveStacks>();
188 AU.addPreserved<LiveStacks>();
189 AU.addRequired<MachineDominatorTree>();
190 AU.addPreserved<MachineDominatorTree>();
191 AU.addRequired<MachineLoopInfo>();
192 AU.addPreserved<MachineLoopInfo>();
193 AU.addRequired<MachineLoopRanges>();
194 AU.addPreserved<MachineLoopRanges>();
195 AU.addRequired<VirtRegMap>();
196 AU.addPreserved<VirtRegMap>();
197 AU.addRequired<EdgeBundles>();
198 AU.addRequired<SpillPlacement>();
199 MachineFunctionPass::getAnalysisUsage(AU);
202 void RAGreedy::releaseMemory() {
203 SpillerInstance.reset(0);
204 RegAllocBase::releaseMemory();
207 float RAGreedy::getPriority(LiveInterval *LI) {
208 float Priority = LI->weight;
210 // Prioritize hinted registers so they are allocated first.
211 std::pair<unsigned, unsigned> Hint;
212 if (Hint.first || Hint.second) {
213 // The hint can be target specific, a virtual register, or a physreg.
216 // Prefer physreg hints above anything else.
217 if (Hint.first == 0 && TargetRegisterInfo::isPhysicalRegister(Hint.second))
224 //===----------------------------------------------------------------------===//
225 // Register Reassignment
226 //===----------------------------------------------------------------------===//
228 // Check interference without using the cache.
229 bool RAGreedy::checkUncachedInterference(LiveInterval &VirtReg,
231 for (const unsigned *AliasI = TRI->getOverlaps(PhysReg); *AliasI; ++AliasI) {
232 LiveIntervalUnion::Query subQ(&VirtReg, &PhysReg2LiveUnion[*AliasI]);
233 if (subQ.checkInterference())
239 /// getSingleInterference - Return the single interfering virtual register
240 /// assigned to PhysReg. Return 0 if more than one virtual register is
242 LiveInterval *RAGreedy::getSingleInterference(LiveInterval &VirtReg,
244 // Check physreg and aliases.
245 LiveInterval *Interference = 0;
246 for (const unsigned *AliasI = TRI->getOverlaps(PhysReg); *AliasI; ++AliasI) {
247 LiveIntervalUnion::Query &Q = query(VirtReg, *AliasI);
248 if (Q.checkInterference()) {
251 Q.collectInterferingVRegs(1);
252 if (!Q.seenAllInterferences())
254 Interference = Q.interferingVRegs().front();
260 // Attempt to reassign this virtual register to a different physical register.
262 // FIXME: we are not yet caching these "second-level" interferences discovered
263 // in the sub-queries. These interferences can change with each call to
264 // selectOrSplit. However, we could implement a "may-interfere" cache that
265 // could be conservatively dirtied when we reassign or split.
267 // FIXME: This may result in a lot of alias queries. We could summarize alias
268 // live intervals in their parent register's live union, but it's messy.
269 bool RAGreedy::reassignVReg(LiveInterval &InterferingVReg,
270 unsigned WantedPhysReg) {
271 assert(TargetRegisterInfo::isVirtualRegister(InterferingVReg.reg) &&
272 "Can only reassign virtual registers");
273 assert(TRI->regsOverlap(WantedPhysReg, VRM->getPhys(InterferingVReg.reg)) &&
274 "inconsistent phys reg assigment");
276 AllocationOrder Order(InterferingVReg.reg, *VRM, ReservedRegs);
277 while (unsigned PhysReg = Order.next()) {
278 // Don't reassign to a WantedPhysReg alias.
279 if (TRI->regsOverlap(PhysReg, WantedPhysReg))
282 if (checkUncachedInterference(InterferingVReg, PhysReg))
285 // Reassign the interfering virtual reg to this physical reg.
286 unsigned OldAssign = VRM->getPhys(InterferingVReg.reg);
287 DEBUG(dbgs() << "reassigning: " << InterferingVReg << " from " <<
288 TRI->getName(OldAssign) << " to " << TRI->getName(PhysReg) << '\n');
289 PhysReg2LiveUnion[OldAssign].extract(InterferingVReg);
290 VRM->clearVirt(InterferingVReg.reg);
291 VRM->assignVirt2Phys(InterferingVReg.reg, PhysReg);
292 PhysReg2LiveUnion[PhysReg].unify(InterferingVReg);
299 /// reassignInterferences - Reassign all interferences to different physical
300 /// registers such that Virtreg can be assigned to PhysReg.
301 /// Currently this only works with a single interference.
302 /// @param VirtReg Currently unassigned virtual register.
303 /// @param PhysReg Physical register to be cleared.
304 /// @return True on success, false if nothing was changed.
305 bool RAGreedy::reassignInterferences(LiveInterval &VirtReg, unsigned PhysReg) {
306 LiveInterval *InterferingVReg = getSingleInterference(VirtReg, PhysReg);
307 if (!InterferingVReg)
309 if (TargetRegisterInfo::isPhysicalRegister(InterferingVReg->reg))
311 return reassignVReg(*InterferingVReg, PhysReg);
314 /// tryReassign - Try to reassign interferences to different physregs.
315 /// @param VirtReg Currently unassigned virtual register.
316 /// @param Order Physregs to try.
317 /// @return Physreg to assign VirtReg, or 0.
318 unsigned RAGreedy::tryReassign(LiveInterval &VirtReg, AllocationOrder &Order) {
319 NamedRegionTimer T("Reassign", TimerGroupName, TimePassesIsEnabled);
321 while (unsigned PhysReg = Order.next())
322 if (reassignInterferences(VirtReg, PhysReg))
328 //===----------------------------------------------------------------------===//
330 //===----------------------------------------------------------------------===//
332 /// calcLiveBlockInfo - Fill the LiveBlocks array with information about blocks
333 /// where VirtReg is live.
334 /// The SpillConstraints array is minimally initialized with MBB->getNumber().
335 void RAGreedy::calcLiveBlockInfo(LiveInterval &VirtReg) {
337 SpillConstraints.clear();
339 assert(!VirtReg.empty() && "Cannot allocate an empty interval");
340 LiveInterval::const_iterator LVI = VirtReg.begin();
341 LiveInterval::const_iterator LVE = VirtReg.end();
343 SmallVectorImpl<SlotIndex>::const_iterator UseI, UseE;
344 UseI = SA->UseSlots.begin();
345 UseE = SA->UseSlots.end();
347 // Loop over basic blocks where VirtReg is live.
348 MachineFunction::iterator MFI = Indexes->getMBBFromIndex(LVI->start);
350 // Block constraints depend on the interference pattern.
351 // Just allocate them here, don't compute anything.
352 SpillPlacement::BlockConstraint BC;
353 BC.Number = MFI->getNumber();
354 SpillConstraints.push_back(BC);
358 SlotIndex Start, Stop;
359 tie(Start, Stop) = Indexes->getMBBRange(BI.MBB);
361 // The last split point is the latest possible insertion point that dominates
362 // all successor blocks. If interference reaches LastSplitPoint, it is not
363 // possible to insert a split or reload that makes VirtReg live in the
365 MachineBasicBlock::iterator LSP = LIS->getLastSplitPoint(VirtReg, BI.MBB);
366 if (LSP == BI.MBB->end())
367 BI.LastSplitPoint = Stop;
369 BI.LastSplitPoint = Indexes->getInstructionIndex(LSP);
371 // LVI is the first live segment overlapping MBB.
372 BI.LiveIn = LVI->start <= Start;
376 // Find the first and last uses in the block.
377 BI.Uses = SA->hasUses(MFI);
378 if (BI.Uses && UseI != UseE) {
380 assert(BI.FirstUse >= Start);
382 while (UseI != UseE && *UseI < Stop);
383 BI.LastUse = UseI[-1];
384 assert(BI.LastUse < Stop);
387 // Look for gaps in the live range.
390 while (LVI->end < Stop) {
391 SlotIndex LastStop = LVI->end;
392 if (++LVI == LVE || LVI->start >= Stop) {
397 if (LastStop < LVI->start) {
404 // Don't set LiveThrough when the block has a gap.
405 BI.LiveThrough = !hasGap && BI.LiveIn && BI.LiveOut;
406 LiveBlocks.push_back(BI);
408 // LVI is now at LVE or LVI->end >= Stop.
412 // Live segment ends exactly at Stop. Move to the next segment.
413 if (LVI->end == Stop && ++LVI == LVE)
416 // Pick the next basic block.
417 if (LVI->start < Stop)
420 MFI = Indexes->getMBBFromIndex(LVI->start);
424 /// calcInterferenceInfo - Compute per-block outgoing and ingoing constraints
425 /// when considering interference from PhysReg. Also compute an optimistic local
426 /// cost of this interference pattern.
428 /// The final cost of a split is the local cost + global cost of preferences
429 /// broken by SpillPlacement.
431 float RAGreedy::calcInterferenceInfo(LiveInterval &VirtReg, unsigned PhysReg) {
432 // Reset interference dependent info.
433 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
434 BlockInfo &BI = LiveBlocks[i];
435 SpillPlacement::BlockConstraint &BC = SpillConstraints[i];
436 BC.Entry = (BI.Uses && BI.LiveIn) ?
437 SpillPlacement::PrefReg : SpillPlacement::DontCare;
438 BC.Exit = (BI.Uses && BI.LiveOut) ?
439 SpillPlacement::PrefReg : SpillPlacement::DontCare;
440 BI.OverlapEntry = BI.OverlapExit = false;
443 // Add interference info from each PhysReg alias.
444 for (const unsigned *AI = TRI->getOverlaps(PhysReg); *AI; ++AI) {
445 if (!query(VirtReg, *AI).checkInterference())
447 LiveIntervalUnion::SegmentIter IntI =
448 PhysReg2LiveUnion[*AI].find(VirtReg.beginIndex());
452 // Determine which blocks have interference live in or after the last split
454 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
455 BlockInfo &BI = LiveBlocks[i];
456 SpillPlacement::BlockConstraint &BC = SpillConstraints[i];
457 SlotIndex Start, Stop;
458 tie(Start, Stop) = Indexes->getMBBRange(BI.MBB);
460 // Skip interference-free blocks.
461 if (IntI.start() >= Stop)
464 // Is the interference live-in?
466 IntI.advanceTo(Start);
469 if (IntI.start() <= Start)
470 BC.Entry = SpillPlacement::MustSpill;
473 // Is the interference overlapping the last split point?
475 if (IntI.stop() < BI.LastSplitPoint)
476 IntI.advanceTo(BI.LastSplitPoint.getPrevSlot());
479 if (IntI.start() < Stop)
480 BC.Exit = SpillPlacement::MustSpill;
484 // Rewind iterator and check other interferences.
485 IntI.find(VirtReg.beginIndex());
486 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
487 BlockInfo &BI = LiveBlocks[i];
488 SpillPlacement::BlockConstraint &BC = SpillConstraints[i];
489 SlotIndex Start, Stop;
490 tie(Start, Stop) = Indexes->getMBBRange(BI.MBB);
492 // Skip interference-free blocks.
493 if (IntI.start() >= Stop)
496 // Handle transparent blocks with interference separately.
497 // Transparent blocks never incur any fixed cost.
498 if (BI.LiveThrough && !BI.Uses) {
499 IntI.advanceTo(Start);
502 if (IntI.start() >= Stop)
505 if (BC.Entry != SpillPlacement::MustSpill)
506 BC.Entry = SpillPlacement::PrefSpill;
507 if (BC.Exit != SpillPlacement::MustSpill)
508 BC.Exit = SpillPlacement::PrefSpill;
512 // Now we only have blocks with uses left.
513 // Check if the interference overlaps the uses.
514 assert(BI.Uses && "Non-transparent block without any uses");
516 // Check interference on entry.
517 if (BI.LiveIn && BC.Entry != SpillPlacement::MustSpill) {
518 IntI.advanceTo(Start);
521 // Not live in, but before the first use.
522 if (IntI.start() < BI.FirstUse)
523 BC.Entry = SpillPlacement::PrefSpill;
526 // Does interference overlap the uses in the entry segment
528 if (BI.LiveIn && !BI.OverlapEntry) {
529 IntI.advanceTo(BI.FirstUse);
532 // A live-through interval has no kill.
533 // Check [FirstUse;LastUse) instead.
534 if (IntI.start() < (BI.LiveThrough ? BI.LastUse : BI.Kill))
535 BI.OverlapEntry = true;
538 // Does interference overlap the uses in the exit segment [Def;LastUse)?
539 if (BI.LiveOut && !BI.LiveThrough && !BI.OverlapExit) {
540 IntI.advanceTo(BI.Def);
543 if (IntI.start() < BI.LastUse)
544 BI.OverlapExit = true;
547 // Check interference on exit.
548 if (BI.LiveOut && BC.Exit != SpillPlacement::MustSpill) {
549 // Check interference between LastUse and Stop.
550 if (BC.Exit != SpillPlacement::PrefSpill) {
551 IntI.advanceTo(BI.LastUse);
554 if (IntI.start() < Stop)
555 BC.Exit = SpillPlacement::PrefSpill;
561 // Accumulate a local cost of this interference pattern.
563 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
564 BlockInfo &BI = LiveBlocks[i];
567 SpillPlacement::BlockConstraint &BC = SpillConstraints[i];
568 unsigned Inserts = 0;
570 // Do we need spill code for the entry segment?
572 Inserts += BI.OverlapEntry || BC.Entry != SpillPlacement::PrefReg;
574 // For the exit segment?
576 Inserts += BI.OverlapExit || BC.Exit != SpillPlacement::PrefReg;
578 // The local cost of spill code in this block is the block frequency times
579 // the number of spill instructions inserted.
581 LocalCost += Inserts * SpillPlacer->getBlockFrequency(BI.MBB);
583 DEBUG(dbgs() << "Local cost of " << PrintReg(PhysReg, TRI) << " = "
584 << LocalCost << '\n');
588 /// calcGlobalSplitCost - Return the global split cost of following the split
589 /// pattern in LiveBundles. This cost should be added to the local cost of the
590 /// interference pattern in SpillConstraints.
592 float RAGreedy::calcGlobalSplitCost(const BitVector &LiveBundles) {
593 float GlobalCost = 0;
594 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
595 SpillPlacement::BlockConstraint &BC = SpillConstraints[i];
596 unsigned Inserts = 0;
597 // Broken entry preference?
598 Inserts += LiveBundles[Bundles->getBundle(BC.Number, 0)] !=
599 (BC.Entry == SpillPlacement::PrefReg);
600 // Broken exit preference?
601 Inserts += LiveBundles[Bundles->getBundle(BC.Number, 1)] !=
602 (BC.Exit == SpillPlacement::PrefReg);
604 GlobalCost += Inserts * SpillPlacer->getBlockFrequency(LiveBlocks[i].MBB);
606 DEBUG(dbgs() << "Global cost = " << GlobalCost << '\n');
610 /// splitAroundRegion - Split VirtReg around the region determined by
611 /// LiveBundles. Make an effort to avoid interference from PhysReg.
613 /// The 'register' interval is going to contain as many uses as possible while
614 /// avoiding interference. The 'stack' interval is the complement constructed by
615 /// SplitEditor. It will contain the rest.
617 void RAGreedy::splitAroundRegion(LiveInterval &VirtReg, unsigned PhysReg,
618 const BitVector &LiveBundles,
619 SmallVectorImpl<LiveInterval*> &NewVRegs) {
621 dbgs() << "Splitting around region for " << PrintReg(PhysReg, TRI)
623 for (int i = LiveBundles.find_first(); i>=0; i = LiveBundles.find_next(i))
624 dbgs() << " EB#" << i;
628 // First compute interference ranges in the live blocks.
629 typedef std::pair<SlotIndex, SlotIndex> IndexPair;
630 SmallVector<IndexPair, 8> InterferenceRanges;
631 InterferenceRanges.resize(LiveBlocks.size());
632 for (const unsigned *AI = TRI->getOverlaps(PhysReg); *AI; ++AI) {
633 if (!query(VirtReg, *AI).checkInterference())
635 LiveIntervalUnion::SegmentIter IntI =
636 PhysReg2LiveUnion[*AI].find(VirtReg.beginIndex());
639 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
640 const BlockInfo &BI = LiveBlocks[i];
641 IndexPair &IP = InterferenceRanges[i];
642 SlotIndex Start, Stop;
643 tie(Start, Stop) = Indexes->getMBBRange(BI.MBB);
644 // Skip interference-free blocks.
645 if (IntI.start() >= Stop)
648 // First interference in block.
650 IntI.advanceTo(Start);
653 if (IntI.start() >= Stop)
655 if (!IP.first.isValid() || IntI.start() < IP.first)
656 IP.first = IntI.start();
659 // Last interference in block.
661 IntI.advanceTo(Stop);
662 if (!IntI.valid() || IntI.start() >= Stop)
664 if (IntI.stop() <= Start)
666 if (!IP.second.isValid() || IntI.stop() > IP.second)
667 IP.second = IntI.stop();
672 SmallVector<LiveInterval*, 4> SpillRegs;
673 LiveRangeEdit LREdit(VirtReg, NewVRegs, SpillRegs);
674 SplitEditor SE(*SA, *LIS, *VRM, *DomTree, LREdit);
676 // Create the main cross-block interval.
679 // First add all defs that are live out of a block.
680 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
681 BlockInfo &BI = LiveBlocks[i];
682 bool RegIn = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 0)];
683 bool RegOut = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 1)];
685 // Should the register be live out?
686 if (!BI.LiveOut || !RegOut)
689 IndexPair &IP = InterferenceRanges[i];
690 SlotIndex Start, Stop;
691 tie(Start, Stop) = Indexes->getMBBRange(BI.MBB);
693 DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " -> EB#"
694 << Bundles->getBundle(BI.MBB->getNumber(), 1)
695 << " intf [" << IP.first << ';' << IP.second << ')');
697 // The interference interval should either be invalid or overlap MBB.
698 assert((!IP.first.isValid() || IP.first < Stop) && "Bad interference");
699 assert((!IP.second.isValid() || IP.second > Start) && "Bad interference");
701 // Check interference leaving the block.
702 if (!IP.second.isValid()) {
703 // Block is interference-free.
704 DEBUG(dbgs() << ", no interference");
706 assert(BI.LiveThrough && "No uses, but not live through block?");
707 // Block is live-through without interference.
708 DEBUG(dbgs() << ", no uses"
709 << (RegIn ? ", live-through.\n" : ", stack in.\n"));
711 SE.enterIntvAtEnd(*BI.MBB);
714 if (!BI.LiveThrough) {
715 DEBUG(dbgs() << ", not live-through.\n");
716 SE.useIntv(SE.enterIntvBefore(BI.Def), Stop);
720 // Block is live-through, but entry bundle is on the stack.
721 // Reload just before the first use.
722 DEBUG(dbgs() << ", not live-in, enter before first use.\n");
723 SE.useIntv(SE.enterIntvBefore(BI.FirstUse), Stop);
726 DEBUG(dbgs() << ", live-through.\n");
730 // Block has interference.
731 DEBUG(dbgs() << ", interference to " << IP.second);
733 if (!BI.LiveThrough && IP.second <= BI.Def) {
734 // The interference doesn't reach the outgoing segment.
735 DEBUG(dbgs() << " doesn't affect def from " << BI.Def << '\n');
736 SE.useIntv(BI.Def, Stop);
742 // No uses in block, avoid interference by reloading as late as possible.
743 DEBUG(dbgs() << ", no uses.\n");
744 SlotIndex SegStart = SE.enterIntvAtEnd(*BI.MBB);
745 assert(SegStart >= IP.second && "Couldn't avoid interference");
749 if (IP.second.getBoundaryIndex() < BI.LastUse &&
750 IP.second.getBoundaryIndex() <= BI.LastSplitPoint) {
751 // There are interference-free uses at the end of the block.
752 // Find the first use that can get the live-out register.
753 SmallVectorImpl<SlotIndex>::const_iterator UI =
754 std::lower_bound(SA->UseSlots.begin(), SA->UseSlots.end(),
755 IP.second.getBoundaryIndex());
756 assert(UI != SA->UseSlots.end() && "Couldn't find last use");
758 DEBUG(dbgs() << ", free use at " << Use << ".\n");
759 assert(Use <= BI.LastUse && "Couldn't find last use");
760 SlotIndex SegStart = SE.enterIntvBefore(Use);
761 assert(SegStart >= IP.second && "Couldn't avoid interference");
762 assert(SegStart < BI.LastSplitPoint && "Impossible split point");
763 SE.useIntv(SegStart, Stop);
767 // Interference is after the last use.
768 DEBUG(dbgs() << " after last use.\n");
769 SlotIndex SegStart = SE.enterIntvAtEnd(*BI.MBB);
770 assert(SegStart >= IP.second && "Couldn't avoid interference");
773 // Now all defs leading to live bundles are handled, do everything else.
774 for (unsigned i = 0, e = LiveBlocks.size(); i != e; ++i) {
775 BlockInfo &BI = LiveBlocks[i];
776 bool RegIn = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 0)];
777 bool RegOut = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 1)];
779 // Is the register live-in?
780 if (!BI.LiveIn || !RegIn)
783 // We have an incoming register. Check for interference.
784 IndexPair &IP = InterferenceRanges[i];
785 SlotIndex Start, Stop;
786 tie(Start, Stop) = Indexes->getMBBRange(BI.MBB);
788 DEBUG(dbgs() << "EB#" << Bundles->getBundle(BI.MBB->getNumber(), 0)
789 << " -> BB#" << BI.MBB->getNumber());
791 // Check interference entering the block.
792 if (!IP.first.isValid()) {
793 // Block is interference-free.
794 DEBUG(dbgs() << ", no interference");
796 assert(BI.LiveThrough && "No uses, but not live through block?");
797 // Block is live-through without interference.
799 DEBUG(dbgs() << ", no uses, live-through.\n");
800 SE.useIntv(Start, Stop);
802 DEBUG(dbgs() << ", no uses, stack-out.\n");
803 SE.leaveIntvAtTop(*BI.MBB);
807 if (!BI.LiveThrough) {
808 DEBUG(dbgs() << ", killed in block.\n");
809 SE.useIntv(Start, SE.leaveIntvAfter(BI.Kill));
813 // Block is live-through, but exit bundle is on the stack.
814 // Spill immediately after the last use.
815 if (BI.LastUse < BI.LastSplitPoint) {
816 DEBUG(dbgs() << ", uses, stack-out.\n");
817 SE.useIntv(Start, SE.leaveIntvAfter(BI.LastUse));
820 // The last use is after the last split point, it is probably an
822 DEBUG(dbgs() << ", uses at " << BI.LastUse << " after split point "
823 << BI.LastSplitPoint << ", stack-out.\n");
825 // Find the last real instruction before the split point.
826 MachineBasicBlock::iterator SplitI =
827 LIS->getInstructionFromIndex(BI.LastSplitPoint);
828 MachineBasicBlock::iterator I = SplitI, B = BI.MBB->begin();
829 while (I != B && (--I)->isDebugValue())
832 SegEnd = SE.leaveIntvAtTop(*BI.MBB);
834 SegEnd = SE.leaveIntvAfter(LIS->getInstructionIndex(I));
835 SE.useIntv(Start, SegEnd);
837 // Run a double interval from the split to the last use.
838 // This makes it possible to spill the complement without affecting the
840 SE.overlapIntv(SegEnd, BI.LastUse);
843 // Register is live-through.
844 DEBUG(dbgs() << ", uses, live-through.\n");
845 SE.useIntv(Start, Stop);
849 // Block has interference.
850 DEBUG(dbgs() << ", interference from " << IP.first);
852 if (!BI.LiveThrough && IP.first >= BI.Kill) {
853 // The interference doesn't reach the outgoing segment.
854 DEBUG(dbgs() << " doesn't affect kill at " << BI.Kill << '\n');
855 SE.useIntv(Start, BI.Kill);
860 // No uses in block, avoid interference by spilling as soon as possible.
861 DEBUG(dbgs() << ", no uses.\n");
862 SlotIndex SegEnd = SE.leaveIntvAtTop(*BI.MBB);
863 assert(SegEnd <= IP.first && "Couldn't avoid interference");
866 if (IP.first.getBaseIndex() > BI.FirstUse) {
867 // There are interference-free uses at the beginning of the block.
868 // Find the last use that can get the register.
869 SmallVectorImpl<SlotIndex>::const_iterator UI =
870 std::lower_bound(SA->UseSlots.begin(), SA->UseSlots.end(),
871 IP.first.getBaseIndex());
872 assert(UI != SA->UseSlots.begin() && "Couldn't find first use");
873 SlotIndex Use = (--UI)->getBoundaryIndex();
874 DEBUG(dbgs() << ", free use at " << *UI << ".\n");
875 SlotIndex SegEnd = SE.leaveIntvAfter(Use);
876 assert(SegEnd <= IP.first && "Couldn't avoid interference");
877 SE.useIntv(Start, SegEnd);
881 // Interference is before the first use.
882 DEBUG(dbgs() << " before first use.\n");
883 SlotIndex SegEnd = SE.leaveIntvAtTop(*BI.MBB);
884 assert(SegEnd <= IP.first && "Couldn't avoid interference");
889 // FIXME: Should we be more aggressive about splitting the stack region into
890 // per-block segments? The current approach allows the stack region to
891 // separate into connected components. Some components may be allocatable.
895 MF->verify(this, "After splitting live range around region");
898 // Make sure that at least one of the new intervals can allocate to PhysReg.
899 // That was the whole point of splitting the live range.
901 for (LiveRangeEdit::iterator I = LREdit.begin(), E = LREdit.end(); I != E;
903 if (!checkUncachedInterference(**I, PhysReg)) {
907 assert(found && "No allocatable intervals after pointless splitting");
912 unsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
913 SmallVectorImpl<LiveInterval*> &NewVRegs) {
914 calcLiveBlockInfo(VirtReg);
915 BitVector LiveBundles, BestBundles;
917 unsigned BestReg = 0;
919 while (unsigned PhysReg = Order.next()) {
920 float Cost = calcInterferenceInfo(VirtReg, PhysReg);
921 if (BestReg && Cost >= BestCost)
924 SpillPlacer->placeSpills(SpillConstraints, LiveBundles);
925 // No live bundles, defer to splitSingleBlocks().
926 if (!LiveBundles.any())
929 Cost += calcGlobalSplitCost(LiveBundles);
930 if (!BestReg || Cost < BestCost) {
933 BestBundles.swap(LiveBundles);
940 splitAroundRegion(VirtReg, BestReg, BestBundles, NewVRegs);
945 //===----------------------------------------------------------------------===//
946 // Live Range Splitting
947 //===----------------------------------------------------------------------===//
949 /// trySplit - Try to split VirtReg or one of its interferences, making it
951 /// @return Physreg when VirtReg may be assigned and/or new NewVRegs.
952 unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
953 SmallVectorImpl<LiveInterval*>&NewVRegs) {
954 NamedRegionTimer T("Splitter", TimerGroupName, TimePassesIsEnabled);
955 SA->analyze(&VirtReg);
957 // Don't attempt splitting on local intervals for now. TBD.
958 if (LIS->intervalIsInOneMBB(VirtReg))
961 // First try to split around a region spanning multiple blocks.
962 unsigned PhysReg = tryRegionSplit(VirtReg, Order, NewVRegs);
963 if (PhysReg || !NewVRegs.empty())
966 // Then isolate blocks with multiple uses.
967 SplitAnalysis::BlockPtrSet Blocks;
968 if (SA->getMultiUseBlocks(Blocks)) {
969 SmallVector<LiveInterval*, 4> SpillRegs;
970 LiveRangeEdit LREdit(VirtReg, NewVRegs, SpillRegs);
971 SplitEditor(*SA, *LIS, *VRM, *DomTree, LREdit).splitSingleBlocks(Blocks);
973 MF->verify(this, "After splitting live range around basic blocks");
976 // Don't assign any physregs.
981 //===----------------------------------------------------------------------===//
983 //===----------------------------------------------------------------------===//
985 /// calcInterferenceWeight - Calculate the combined spill weight of
986 /// interferences when assigning VirtReg to PhysReg.
987 float RAGreedy::calcInterferenceWeight(LiveInterval &VirtReg, unsigned PhysReg){
989 for (const unsigned *AI = TRI->getOverlaps(PhysReg); *AI; ++AI) {
990 LiveIntervalUnion::Query &Q = query(VirtReg, *AI);
991 Q.collectInterferingVRegs();
992 if (Q.seenUnspillableVReg())
994 for (unsigned i = 0, e = Q.interferingVRegs().size(); i != e; ++i)
995 Sum += Q.interferingVRegs()[i]->weight;
1000 /// trySpillInterferences - Try to spill interfering registers instead of the
1001 /// current one. Only do it if the accumulated spill weight is smaller than the
1002 /// current spill weight.
1003 unsigned RAGreedy::trySpillInterferences(LiveInterval &VirtReg,
1004 AllocationOrder &Order,
1005 SmallVectorImpl<LiveInterval*> &NewVRegs) {
1006 NamedRegionTimer T("Spill Interference", TimerGroupName, TimePassesIsEnabled);
1007 unsigned BestPhys = 0;
1008 float BestWeight = 0;
1011 while (unsigned PhysReg = Order.next()) {
1012 float Weight = calcInterferenceWeight(VirtReg, PhysReg);
1013 if (Weight == HUGE_VALF || Weight >= VirtReg.weight)
1015 if (!BestPhys || Weight < BestWeight)
1016 BestPhys = PhysReg, BestWeight = Weight;
1019 // No candidates found.
1023 // Collect all interfering registers.
1024 SmallVector<LiveInterval*, 8> Spills;
1025 for (const unsigned *AI = TRI->getOverlaps(BestPhys); *AI; ++AI) {
1026 LiveIntervalUnion::Query &Q = query(VirtReg, *AI);
1027 Spills.append(Q.interferingVRegs().begin(), Q.interferingVRegs().end());
1028 for (unsigned i = 0, e = Q.interferingVRegs().size(); i != e; ++i) {
1029 LiveInterval *VReg = Q.interferingVRegs()[i];
1030 PhysReg2LiveUnion[*AI].extract(*VReg);
1031 VRM->clearVirt(VReg->reg);
1036 DEBUG(dbgs() << "spilling " << Spills.size() << " interferences with weight "
1037 << BestWeight << '\n');
1038 for (unsigned i = 0, e = Spills.size(); i != e; ++i)
1039 spiller().spill(Spills[i], NewVRegs, Spills);
1044 //===----------------------------------------------------------------------===//
1046 //===----------------------------------------------------------------------===//
1048 unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg,
1049 SmallVectorImpl<LiveInterval*> &NewVRegs) {
1050 // First try assigning a free register.
1051 AllocationOrder Order(VirtReg.reg, *VRM, ReservedRegs);
1052 while (unsigned PhysReg = Order.next()) {
1053 if (!checkPhysRegInterference(VirtReg, PhysReg))
1057 // Try to reassign interferences.
1058 if (unsigned PhysReg = tryReassign(VirtReg, Order))
1061 assert(NewVRegs.empty() && "Cannot append to existing NewVRegs");
1063 // Try splitting VirtReg or interferences.
1064 unsigned PhysReg = trySplit(VirtReg, Order, NewVRegs);
1065 if (PhysReg || !NewVRegs.empty())
1068 // Try to spill another interfering reg with less spill weight.
1069 PhysReg = trySpillInterferences(VirtReg, Order, NewVRegs);
1073 // Finally spill VirtReg itself.
1074 NamedRegionTimer T("Spiller", TimerGroupName, TimePassesIsEnabled);
1075 SmallVector<LiveInterval*, 1> pendingSpills;
1076 spiller().spill(&VirtReg, NewVRegs, pendingSpills);
1078 // The live virtual register requesting allocation was spilled, so tell
1079 // the caller not to allocate anything during this round.
1083 bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
1084 DEBUG(dbgs() << "********** GREEDY REGISTER ALLOCATION **********\n"
1085 << "********** Function: "
1086 << ((Value*)mf.getFunction())->getName() << '\n');
1090 MF->verify(this, "Before greedy register allocator");
1092 RegAllocBase::init(getAnalysis<VirtRegMap>(), getAnalysis<LiveIntervals>());
1093 Indexes = &getAnalysis<SlotIndexes>();
1094 DomTree = &getAnalysis<MachineDominatorTree>();
1095 ReservedRegs = TRI->getReservedRegs(*MF);
1096 SpillerInstance.reset(createInlineSpiller(*this, *MF, *VRM));
1097 Loops = &getAnalysis<MachineLoopInfo>();
1098 LoopRanges = &getAnalysis<MachineLoopRanges>();
1099 Bundles = &getAnalysis<EdgeBundles>();
1100 SpillPlacer = &getAnalysis<SpillPlacement>();
1102 SA.reset(new SplitAnalysis(*MF, *LIS, *Loops));
1106 LIS->addKillFlags();
1110 NamedRegionTimer T("Rewriter", TimerGroupName, TimePassesIsEnabled);
1111 std::auto_ptr<VirtRegRewriter> rewriter(createVirtRegRewriter());
1112 rewriter->runOnMachineFunction(*MF, *VRM, LIS);
1115 // The pass output is in VirtRegMap. Release all the transient data.