X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FRegAllocPBQP.cpp;h=946d70e32fb863901fb778077cdcba7bdc14b071;hb=05ec712e7f75635abbdd84dced69f4a45fe0f541;hp=61450a7cca7c3332198a88b2c28c839f79fe36d5;hpb=51b16f473759c1546acbf308a5d3f3e7bf3ea23c;p=oota-llvm.git diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp index 61450a7cca7..946d70e32fb 100644 --- a/lib/CodeGen/RegAllocPBQP.cpp +++ b/lib/CodeGen/RegAllocPBQP.cpp @@ -31,713 +31,565 @@ #define DEBUG_TYPE "regalloc" -#include "PBQP.h" +#include "RenderMachineFunction.h" +#include "Spiller.h" #include "VirtRegMap.h" -#include "VirtRegRewriter.h" +#include "RegisterCoalescer.h" +#include "llvm/Module.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/CodeGen/CalcSpillWeights.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" +#include "llvm/CodeGen/LiveRangeEdit.h" #include "llvm/CodeGen/LiveStackAnalysis.h" +#include "llvm/CodeGen/RegAllocPBQP.h" +#include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/PBQP/HeuristicSolver.h" +#include "llvm/CodeGen/PBQP/Graph.h" +#include "llvm/CodeGen/PBQP/Heuristics/Briggs.h" #include "llvm/CodeGen/RegAllocRegistry.h" -#include "llvm/CodeGen/RegisterCoalescer.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" #include -#include #include #include +#include #include using namespace llvm; static RegisterRegAlloc registerPBQPRepAlloc("pbqp", "PBQP register allocator", - createPBQPRegisterAllocator); + createDefaultPBQPRegisterAllocator); -namespace { - - //! - //! PBQP based allocators solve the register allocation problem by mapping - //! register allocation problems to Partitioned Boolean Quadratic - //! Programming problems. - class VISIBILITY_HIDDEN PBQPRegAlloc : public MachineFunctionPass { - public: +static cl::opt +pbqpCoalescing("pbqp-coalescing", + cl::desc("Attempt coalescing during PBQP register allocation."), + cl::init(false), cl::Hidden); - static char ID; +#ifndef NDEBUG +static cl::opt +pbqpDumpGraphs("pbqp-dump-graphs", + cl::desc("Dump graphs for each function/round in the compilation unit."), + cl::init(false), cl::Hidden); +#endif - //! Construct a PBQP register allocator. - PBQPRegAlloc() : MachineFunctionPass((intptr_t)&ID) {} +namespace { - //! Return the pass name. - virtual const char* getPassName() const throw() { - return "PBQP Register Allocator"; - } +/// +/// PBQP based allocators solve the register allocation problem by mapping +/// register allocation problems to Partitioned Boolean Quadratic +/// Programming problems. +class RegAllocPBQP : public MachineFunctionPass { +public: + + static char ID; + + /// Construct a PBQP register allocator. + RegAllocPBQP(std::auto_ptr b, char *cPassID=0) + : MachineFunctionPass(ID), builder(b), customPassID(cPassID) { + initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); + initializeLiveIntervalsPass(*PassRegistry::getPassRegistry()); + initializeCalculateSpillWeightsPass(*PassRegistry::getPassRegistry()); + initializeLiveStacksPass(*PassRegistry::getPassRegistry()); + initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry()); + initializeVirtRegMapPass(*PassRegistry::getPassRegistry()); + initializeRenderMachineFunctionPass(*PassRegistry::getPassRegistry()); + } - //! PBQP analysis usage. - virtual void getAnalysisUsage(AnalysisUsage &au) const { - au.addRequired(); - au.addRequiredTransitive(); - au.addRequired(); - au.addPreserved(); - au.addRequired(); - au.addPreserved(); - au.addRequired(); - MachineFunctionPass::getAnalysisUsage(au); - } + /// Return the pass name. + virtual const char* getPassName() const { + return "PBQP Register Allocator"; + } - //! Perform register allocation - virtual bool runOnMachineFunction(MachineFunction &MF); - - private: - typedef std::map LI2NodeMap; - typedef std::vector Node2LIMap; - typedef std::vector AllowedSet; - typedef std::vector AllowedSetMap; - typedef std::set RegSet; - typedef std::pair RegPair; - typedef std::map CoalesceMap; - - typedef std::set LiveIntervalSet; - - MachineFunction *mf; - const TargetMachine *tm; - const TargetRegisterInfo *tri; - const TargetInstrInfo *tii; - const MachineLoopInfo *loopInfo; - MachineRegisterInfo *mri; - - LiveIntervals *lis; - LiveStacks *lss; - VirtRegMap *vrm; - - LI2NodeMap li2Node; - Node2LIMap node2LI; - AllowedSetMap allowedSets; - LiveIntervalSet vregIntervalsToAlloc, - emptyVRegIntervals; - - - //! Builds a PBQP cost vector. - template - PBQPVector* buildCostVector(unsigned vReg, - const RegContainer &allowed, - const CoalesceMap &cealesces, - PBQPNum spillCost) const; - - //! \brief Builds a PBQP interference matrix. - //! - //! @return Either a pointer to a non-zero PBQP matrix representing the - //! allocation option costs, or a null pointer for a zero matrix. - //! - //! Expects allowed sets for two interfering LiveIntervals. These allowed - //! sets should contain only allocable registers from the LiveInterval's - //! register class, with any interfering pre-colored registers removed. - template - PBQPMatrix* buildInterferenceMatrix(const RegContainer &allowed1, - const RegContainer &allowed2) const; - - //! - //! Expects allowed sets for two potentially coalescable LiveIntervals, - //! and an estimated benefit due to coalescing. The allowed sets should - //! contain only allocable registers from the LiveInterval's register - //! classes, with any interfering pre-colored registers removed. - template - PBQPMatrix* buildCoalescingMatrix(const RegContainer &allowed1, - const RegContainer &allowed2, - PBQPNum cBenefit) const; - - //! \brief Finds coalescing opportunities and returns them as a map. - //! - //! Any entries in the map are guaranteed coalescable, even if their - //! corresponding live intervals overlap. - CoalesceMap findCoalesces(); - - //! \brief Finds the initial set of vreg intervals to allocate. - void findVRegIntervalsToAlloc(); - - //! \brief Constructs a PBQP problem representation of the register - //! allocation problem for this function. - //! - //! @return a PBQP solver object for the register allocation problem. - pbqp* constructPBQPProblem(); - - //! \brief Adds a stack interval if the given live interval has been - //! spilled. Used to support stack slot coloring. - void addStackInterval(const LiveInterval *spilled,MachineRegisterInfo* mri); - - //! \brief Given a solved PBQP problem maps this solution back to a register - //! assignment. - bool mapPBQPToRegAlloc(pbqp *problem); - - //! \brief Postprocessing before final spilling. Sets basic block "live in" - //! variables. - void finalizeAlloc() const; - - }; - - char PBQPRegAlloc::ID = 0; -} + /// PBQP analysis usage. + virtual void getAnalysisUsage(AnalysisUsage &au) const; + /// Perform register allocation + virtual bool runOnMachineFunction(MachineFunction &MF); -template -PBQPVector* PBQPRegAlloc::buildCostVector(unsigned vReg, - const RegContainer &allowed, - const CoalesceMap &coalesces, - PBQPNum spillCost) const { +private: - typedef typename RegContainer::const_iterator AllowedItr; + typedef std::map LI2NodeMap; + typedef std::vector Node2LIMap; + typedef std::vector AllowedSet; + typedef std::vector AllowedSetMap; + typedef std::pair RegPair; + typedef std::map CoalesceMap; + typedef std::vector NodeVector; + typedef std::set RegSet; - // Allocate vector. Additional element (0th) used for spill option - PBQPVector *v = new PBQPVector(allowed.size() + 1); - (*v)[0] = spillCost; + std::auto_ptr builder; - // Iterate over the allowed registers inserting coalesce benefits if there - // are any. - unsigned ai = 0; - for (AllowedItr itr = allowed.begin(), end = allowed.end(); - itr != end; ++itr, ++ai) { + char *customPassID; - unsigned pReg = *itr; + MachineFunction *mf; + const TargetMachine *tm; + const TargetRegisterInfo *tri; + const TargetInstrInfo *tii; + const MachineLoopInfo *loopInfo; + MachineRegisterInfo *mri; + RenderMachineFunction *rmf; - CoalesceMap::const_iterator cmItr = - coalesces.find(RegPair(vReg, pReg)); + std::auto_ptr spiller; + LiveIntervals *lis; + LiveStacks *lss; + VirtRegMap *vrm; - // No coalesce - on to the next preg. - if (cmItr == coalesces.end()) - continue; + RegSet vregsToAlloc, emptyIntervalVRegs; - // We have a coalesce - insert the benefit. - (*v)[ai + 1] = -cmItr->second; - } + /// \brief Finds the initial set of vreg intervals to allocate. + void findVRegIntervalsToAlloc(); - return v; -} + /// \brief Given a solved PBQP problem maps this solution back to a register + /// assignment. + bool mapPBQPToRegAlloc(const PBQPRAProblem &problem, + const PBQP::Solution &solution); -template -PBQPMatrix* PBQPRegAlloc::buildInterferenceMatrix( - const RegContainer &allowed1, const RegContainer &allowed2) const { + /// \brief Postprocessing before final spilling. Sets basic block "live in" + /// variables. + void finalizeAlloc() const; - typedef typename RegContainer::const_iterator RegContainerIterator; +}; - // Construct a PBQP matrix representing the cost of allocation options. The - // rows and columns correspond to the allocation options for the two live - // intervals. Elements will be infinite where corresponding registers alias, - // since we cannot allocate aliasing registers to interfering live intervals. - // All other elements (non-aliasing combinations) will have zero cost. Note - // that the spill option (element 0,0) has zero cost, since we can allocate - // both intervals to memory safely (the cost for each individual allocation - // to memory is accounted for by the cost vectors for each live interval). - PBQPMatrix *m = new PBQPMatrix(allowed1.size() + 1, allowed2.size() + 1); +char RegAllocPBQP::ID = 0; - // Assume this is a zero matrix until proven otherwise. Zero matrices occur - // between interfering live ranges with non-overlapping register sets (e.g. - // non-overlapping reg classes, or disjoint sets of allowed regs within the - // same class). The term "overlapping" is used advisedly: sets which do not - // intersect, but contain registers which alias, will have non-zero matrices. - // We optimize zero matrices away to improve solver speed. - bool isZeroMatrix = true; +} // End anonymous namespace. +unsigned PBQPRAProblem::getVRegForNode(PBQP::Graph::ConstNodeItr node) const { + Node2VReg::const_iterator vregItr = node2VReg.find(node); + assert(vregItr != node2VReg.end() && "No vreg for node."); + return vregItr->second; +} - // Row index. Starts at 1, since the 0th row is for the spill option, which - // is always zero. - unsigned ri = 1; +PBQP::Graph::NodeItr PBQPRAProblem::getNodeForVReg(unsigned vreg) const { + VReg2Node::const_iterator nodeItr = vreg2Node.find(vreg); + assert(nodeItr != vreg2Node.end() && "No node for vreg."); + return nodeItr->second; - // Iterate over allowed sets, insert infinities where required. - for (RegContainerIterator a1Itr = allowed1.begin(), a1End = allowed1.end(); - a1Itr != a1End; ++a1Itr) { +} - // Column index, starts at 1 as for row index. - unsigned ci = 1; - unsigned reg1 = *a1Itr; +const PBQPRAProblem::AllowedSet& + PBQPRAProblem::getAllowedSet(unsigned vreg) const { + AllowedSetMap::const_iterator allowedSetItr = allowedSets.find(vreg); + assert(allowedSetItr != allowedSets.end() && "No pregs for vreg."); + const AllowedSet &allowedSet = allowedSetItr->second; + return allowedSet; +} - for (RegContainerIterator a2Itr = allowed2.begin(), a2End = allowed2.end(); - a2Itr != a2End; ++a2Itr) { +unsigned PBQPRAProblem::getPRegForOption(unsigned vreg, unsigned option) const { + assert(isPRegOption(vreg, option) && "Not a preg option."); - unsigned reg2 = *a2Itr; + const AllowedSet& allowedSet = getAllowedSet(vreg); + assert(option <= allowedSet.size() && "Option outside allowed set."); + return allowedSet[option - 1]; +} - // If the row/column regs are identical or alias insert an infinity. - if ((reg1 == reg2) || tri->areAliases(reg1, reg2)) { - (*m)[ri][ci] = std::numeric_limits::infinity(); - isZeroMatrix = false; - } +std::auto_ptr PBQPBuilder::build(MachineFunction *mf, + const LiveIntervals *lis, + const MachineLoopInfo *loopInfo, + const RegSet &vregs) { - ++ci; - } + typedef std::vector LIVector; + ArrayRef regMaskSlots = lis->getRegMaskSlots(); + MachineRegisterInfo *mri = &mf->getRegInfo(); + const TargetRegisterInfo *tri = mf->getTarget().getRegisterInfo(); - ++ri; - } + std::auto_ptr p(new PBQPRAProblem()); + PBQP::Graph &g = p->getGraph(); + RegSet pregs; - // If this turns out to be a zero matrix... - if (isZeroMatrix) { - // free it and return null. - delete m; - return 0; + // Collect the set of preg intervals, record that they're used in the MF. + for (LiveIntervals::const_iterator itr = lis->begin(), end = lis->end(); + itr != end; ++itr) { + if (TargetRegisterInfo::isPhysicalRegister(itr->first)) { + pregs.insert(itr->first); + mri->setPhysRegUsed(itr->first); + } } - // ...otherwise return the cost matrix. - return m; -} - -template -PBQPMatrix* PBQPRegAlloc::buildCoalescingMatrix( - const RegContainer &allowed1, const RegContainer &allowed2, - PBQPNum cBenefit) const { - - typedef typename RegContainer::const_iterator RegContainerIterator; - - // Construct a PBQP Matrix representing the benefits of coalescing. As with - // interference matrices the rows and columns represent allowed registers - // for the LiveIntervals which are (potentially) to be coalesced. The amount - // -cBenefit will be placed in any element representing the same register - // for both intervals. - PBQPMatrix *m = new PBQPMatrix(allowed1.size() + 1, allowed2.size() + 1); - - // Reset costs to zero. - m->reset(0); - - // Assume the matrix is zero till proven otherwise. Zero matrices will be - // optimized away as in the interference case. - bool isZeroMatrix = true; - - // Row index. Starts at 1, since the 0th row is for the spill option, which - // is always zero. - unsigned ri = 1; - - // Iterate over the allowed sets, insert coalescing benefits where - // appropriate. - for (RegContainerIterator a1Itr = allowed1.begin(), a1End = allowed1.end(); - a1Itr != a1End; ++a1Itr) { - - // Column index, starts at 1 as for row index. - unsigned ci = 1; - unsigned reg1 = *a1Itr; - - for (RegContainerIterator a2Itr = allowed2.begin(), a2End = allowed2.end(); - a2Itr != a2End; ++a2Itr) { - - // If the row and column represent the same register insert a beneficial - // cost to preference this allocation - it would allow us to eliminate a - // move instruction. - if (reg1 == *a2Itr) { - (*m)[ri][ci] = -cBenefit; - isZeroMatrix = false; + BitVector reservedRegs = tri->getReservedRegs(*mf); + + // Iterate over vregs. + for (RegSet::const_iterator vregItr = vregs.begin(), vregEnd = vregs.end(); + vregItr != vregEnd; ++vregItr) { + unsigned vreg = *vregItr; + const TargetRegisterClass *trc = mri->getRegClass(vreg); + const LiveInterval *vregLI = &lis->getInterval(vreg); + + // Compute an initial allowed set for the current vreg. + typedef std::vector VRAllowed; + VRAllowed vrAllowed; + ArrayRef rawOrder = trc->getRawAllocationOrder(*mf); + for (unsigned i = 0; i != rawOrder.size(); ++i) { + unsigned preg = rawOrder[i]; + if (!reservedRegs.test(preg)) { + vrAllowed.push_back(preg); } - - ++ci; } - ++ri; - } - - // If this turns out to be a zero matrix... - if (isZeroMatrix) { - // ...free it and return null. - delete m; - return 0; - } - - return m; -} - -PBQPRegAlloc::CoalesceMap PBQPRegAlloc::findCoalesces() { - - typedef MachineFunction::const_iterator MFIterator; - typedef MachineBasicBlock::const_iterator MBBIterator; - typedef LiveInterval::const_vni_iterator VNIIterator; - - CoalesceMap coalescesFound; - - // To find coalesces we need to iterate over the function looking for - // copy instructions. - for (MFIterator bbItr = mf->begin(), bbEnd = mf->end(); - bbItr != bbEnd; ++bbItr) { - - const MachineBasicBlock *mbb = &*bbItr; - - for (MBBIterator iItr = mbb->begin(), iEnd = mbb->end(); - iItr != iEnd; ++iItr) { - - const MachineInstr *instr = &*iItr; - unsigned srcReg, dstReg, srcSubReg, dstSubReg; - - // If this isn't a copy then continue to the next instruction. - if (!tii->isMoveInstr(*instr, srcReg, dstReg, srcSubReg, dstSubReg)) - continue; - - // If the registers are already the same our job is nice and easy. - if (dstReg == srcReg) - continue; - - bool srcRegIsPhysical = TargetRegisterInfo::isPhysicalRegister(srcReg), - dstRegIsPhysical = TargetRegisterInfo::isPhysicalRegister(dstReg); - - // If both registers are physical then we can't coalesce. - if (srcRegIsPhysical && dstRegIsPhysical) - continue; - - // If it's a copy that includes a virtual register but the source and - // destination classes differ then we can't coalesce, so continue with - // the next instruction. - const TargetRegisterClass *srcRegClass = srcRegIsPhysical ? - tri->getPhysicalRegisterRegClass(srcReg) : mri->getRegClass(srcReg); + RegSet overlappingPRegs; - const TargetRegisterClass *dstRegClass = dstRegIsPhysical ? - tri->getPhysicalRegisterRegClass(dstReg) : mri->getRegClass(dstReg); + // Record physical registers whose ranges overlap. + for (RegSet::const_iterator pregItr = pregs.begin(), + pregEnd = pregs.end(); + pregItr != pregEnd; ++pregItr) { + unsigned preg = *pregItr; + const LiveInterval *pregLI = &lis->getInterval(preg); - if (srcRegClass != dstRegClass) + if (pregLI->empty()) { continue; - - // We also need any physical regs to be allocable, coalescing with - // a non-allocable register is invalid. - if (srcRegIsPhysical) { - if (std::find(srcRegClass->allocation_order_begin(*mf), - srcRegClass->allocation_order_end(*mf), srcReg) == - srcRegClass->allocation_order_end(*mf)) - continue; - } - - if (dstRegIsPhysical) { - if (std::find(dstRegClass->allocation_order_begin(*mf), - dstRegClass->allocation_order_end(*mf), dstReg) == - dstRegClass->allocation_order_end(*mf)) - continue; } - // If we've made it here we have a copy with compatible register classes. - // We can probably coalesce, but we need to consider overlap. - const LiveInterval *srcLI = &lis->getInterval(srcReg), - *dstLI = &lis->getInterval(dstReg); - - if (srcLI->overlaps(*dstLI)) { - // Even in the case of an overlap we might still be able to coalesce, - // but we need to make sure that no definition of either range occurs - // while the other range is live. - - // Otherwise start by assuming we're ok. - bool badDef = false; - - // Test all defs of the source range. - for (VNIIterator - vniItr = srcLI->vni_begin(), vniEnd = srcLI->vni_end(); - vniItr != vniEnd; ++vniItr) { + if (vregLI->overlaps(*pregLI)) + overlappingPRegs.insert(preg); + } - // If we find a def that kills the coalescing opportunity then - // record it and break from the loop. - if (dstLI->liveAt((*vniItr)->def)) { - badDef = true; + // Record any overlaps with regmask operands. + BitVector regMaskOverlaps(tri->getNumRegs()); + for (ArrayRef::iterator rmItr = regMaskSlots.begin(), + rmEnd = regMaskSlots.end(); + rmItr != rmEnd; ++rmItr) { + SlotIndex rmIdx = *rmItr; + if (vregLI->liveAt(rmIdx)) { + MachineInstr *rmMI = lis->getInstructionFromIndex(rmIdx); + const uint32_t* regMask = 0; + for (MachineInstr::mop_iterator mopItr = rmMI->operands_begin(), + mopEnd = rmMI->operands_end(); + mopItr != mopEnd; ++mopItr) { + if (mopItr->isRegMask()) { + regMask = mopItr->getRegMask(); break; } } + assert(regMask != 0 && "Couldn't find register mask."); + regMaskOverlaps.setBitsNotInMask(regMask); + } + } - // If we have a bad def give up, continue to the next instruction. - if (badDef) - continue; + for (unsigned preg = 0; preg < tri->getNumRegs(); ++preg) { + if (regMaskOverlaps.test(preg)) + overlappingPRegs.insert(preg); + } - // Otherwise test definitions of the destination range. - for (VNIIterator - vniItr = dstLI->vni_begin(), vniEnd = dstLI->vni_end(); - vniItr != vniEnd; ++vniItr) { + for (RegSet::const_iterator pregItr = overlappingPRegs.begin(), + pregEnd = overlappingPRegs.end(); + pregItr != pregEnd; ++pregItr) { + unsigned preg = *pregItr; - // We want to make sure we skip the copy instruction itself. - if ((*vniItr)->copy == instr) - continue; + // Remove the register from the allowed set. + VRAllowed::iterator eraseItr = + std::find(vrAllowed.begin(), vrAllowed.end(), preg); - if (srcLI->liveAt((*vniItr)->def)) { - badDef = true; - break; - } - } - - // As before a bad def we give up and continue to the next instr. - if (badDef) - continue; + if (eraseItr != vrAllowed.end()) { + vrAllowed.erase(eraseItr); } - // If we make it to here then either the ranges didn't overlap, or they - // did, but none of their definitions would prevent us from coalescing. - // We're good to go with the coalesce. - - float cBenefit = powf(10.0f, loopInfo->getLoopDepth(mbb)) / 5.0; - - coalescesFound[RegPair(srcReg, dstReg)] = cBenefit; - coalescesFound[RegPair(dstReg, srcReg)] = cBenefit; + // Also remove any aliases. + for (MCRegAliasIterator AI(preg, tri, false); AI.isValid(); ++AI) { + VRAllowed::iterator eraseItr = + std::find(vrAllowed.begin(), vrAllowed.end(), *AI); + if (eraseItr != vrAllowed.end()) { + vrAllowed.erase(eraseItr); + } + } } - } + // Construct the node. + PBQP::Graph::NodeItr node = + g.addNode(PBQP::Vector(vrAllowed.size() + 1, 0)); - return coalescesFound; -} + // Record the mapping and allowed set in the problem. + p->recordVReg(vreg, node, vrAllowed.begin(), vrAllowed.end()); -void PBQPRegAlloc::findVRegIntervalsToAlloc() { + PBQP::PBQPNum spillCost = (vregLI->weight != 0.0) ? + vregLI->weight : std::numeric_limits::min(); - // Iterate over all live ranges. - for (LiveIntervals::iterator itr = lis->begin(), end = lis->end(); - itr != end; ++itr) { - - // Ignore physical ones. - if (TargetRegisterInfo::isPhysicalRegister(itr->first)) - continue; - - LiveInterval *li = itr->second; + addSpillCosts(g.getNodeCosts(node), spillCost); + } - // If this live interval is non-empty we will use pbqp to allocate it. - // Empty intervals we allocate in a simple post-processing stage in - // finalizeAlloc. - if (!li->empty()) { - vregIntervalsToAlloc.insert(li); - } - else { - emptyVRegIntervals.insert(li); + for (RegSet::const_iterator vr1Itr = vregs.begin(), vrEnd = vregs.end(); + vr1Itr != vrEnd; ++vr1Itr) { + unsigned vr1 = *vr1Itr; + const LiveInterval &l1 = lis->getInterval(vr1); + const PBQPRAProblem::AllowedSet &vr1Allowed = p->getAllowedSet(vr1); + + for (RegSet::const_iterator vr2Itr = llvm::next(vr1Itr); + vr2Itr != vrEnd; ++vr2Itr) { + unsigned vr2 = *vr2Itr; + const LiveInterval &l2 = lis->getInterval(vr2); + const PBQPRAProblem::AllowedSet &vr2Allowed = p->getAllowedSet(vr2); + + assert(!l2.empty() && "Empty interval in vreg set?"); + if (l1.overlaps(l2)) { + PBQP::Graph::EdgeItr edge = + g.addEdge(p->getNodeForVReg(vr1), p->getNodeForVReg(vr2), + PBQP::Matrix(vr1Allowed.size()+1, vr2Allowed.size()+1, 0)); + + addInterferenceCosts(g.getEdgeCosts(edge), vr1Allowed, vr2Allowed, tri); + } } } -} -pbqp* PBQPRegAlloc::constructPBQPProblem() { + return p; +} - typedef std::vector LIVector; - typedef std::vector RegVector; +void PBQPBuilder::addSpillCosts(PBQP::Vector &costVec, + PBQP::PBQPNum spillCost) { + costVec[0] = spillCost; +} - // This will store the physical intervals for easy reference. - LIVector physIntervals; +void PBQPBuilder::addInterferenceCosts( + PBQP::Matrix &costMat, + const PBQPRAProblem::AllowedSet &vr1Allowed, + const PBQPRAProblem::AllowedSet &vr2Allowed, + const TargetRegisterInfo *tri) { + assert(costMat.getRows() == vr1Allowed.size() + 1 && "Matrix height mismatch."); + assert(costMat.getCols() == vr2Allowed.size() + 1 && "Matrix width mismatch."); - // Start by clearing the old node <-> live interval mappings & allowed sets - li2Node.clear(); - node2LI.clear(); - allowedSets.clear(); + for (unsigned i = 0; i != vr1Allowed.size(); ++i) { + unsigned preg1 = vr1Allowed[i]; - // Populate physIntervals, update preg use: - for (LiveIntervals::iterator itr = lis->begin(), end = lis->end(); - itr != end; ++itr) { + for (unsigned j = 0; j != vr2Allowed.size(); ++j) { + unsigned preg2 = vr2Allowed[j]; - if (TargetRegisterInfo::isPhysicalRegister(itr->first)) { - physIntervals.push_back(itr->second); - mri->setPhysRegUsed(itr->second->reg); + if (tri->regsOverlap(preg1, preg2)) { + costMat[i + 1][j + 1] = std::numeric_limits::infinity(); + } } } +} - // Iterate over vreg intervals, construct live interval <-> node number - // mappings. - for (LiveIntervalSet::const_iterator - itr = vregIntervalsToAlloc.begin(), end = vregIntervalsToAlloc.end(); - itr != end; ++itr) { - const LiveInterval *li = *itr; - - li2Node[li] = node2LI.size(); - node2LI.push_back(li); - } - - // Get the set of potential coalesces. - CoalesceMap coalesces(findCoalesces()); - - // Construct a PBQP solver for this problem - pbqp *solver = alloc_pbqp(vregIntervalsToAlloc.size()); - - // Resize allowedSets container appropriately. - allowedSets.resize(vregIntervalsToAlloc.size()); - - // Iterate over virtual register intervals to compute allowed sets... - for (unsigned node = 0; node < node2LI.size(); ++node) { - - // Grab pointers to the interval and its register class. - const LiveInterval *li = node2LI[node]; - const TargetRegisterClass *liRC = mri->getRegClass(li->reg); +std::auto_ptr PBQPBuilderWithCoalescing::build( + MachineFunction *mf, + const LiveIntervals *lis, + const MachineLoopInfo *loopInfo, + const RegSet &vregs) { - // Start by assuming all allocable registers in the class are allowed... - RegVector liAllowed(liRC->allocation_order_begin(*mf), - liRC->allocation_order_end(*mf)); + std::auto_ptr p = PBQPBuilder::build(mf, lis, loopInfo, vregs); + PBQP::Graph &g = p->getGraph(); - // Eliminate the physical registers which overlap with this range, along - // with all their aliases. - for (LIVector::iterator pItr = physIntervals.begin(), - pEnd = physIntervals.end(); pItr != pEnd; ++pItr) { + const TargetMachine &tm = mf->getTarget(); + CoalescerPair cp(*tm.getRegisterInfo()); - if (!li->overlaps(**pItr)) - continue; + // Scan the machine function and add a coalescing cost whenever CoalescerPair + // gives the Ok. + for (MachineFunction::const_iterator mbbItr = mf->begin(), + mbbEnd = mf->end(); + mbbItr != mbbEnd; ++mbbItr) { + const MachineBasicBlock *mbb = &*mbbItr; - unsigned pReg = (*pItr)->reg; + for (MachineBasicBlock::const_iterator miItr = mbb->begin(), + miEnd = mbb->end(); + miItr != miEnd; ++miItr) { + const MachineInstr *mi = &*miItr; - // If we get here then the live intervals overlap, but we're still ok - // if they're coalescable. - if (coalesces.find(RegPair(li->reg, pReg)) != coalesces.end()) - continue; + if (!cp.setRegisters(mi)) { + continue; // Not coalescable. + } - // If we get here then we have a genuine exclusion. + if (cp.getSrcReg() == cp.getDstReg()) { + continue; // Already coalesced. + } - // Remove the overlapping reg... - RegVector::iterator eraseItr = - std::find(liAllowed.begin(), liAllowed.end(), pReg); + unsigned dst = cp.getDstReg(), + src = cp.getSrcReg(); - if (eraseItr != liAllowed.end()) - liAllowed.erase(eraseItr); + const float copyFactor = 0.5; // Cost of copy relative to load. Current + // value plucked randomly out of the air. - const unsigned *aliasItr = tri->getAliasSet(pReg); + PBQP::PBQPNum cBenefit = + copyFactor * LiveIntervals::getSpillWeight(false, true, + loopInfo->getLoopDepth(mbb)); - if (aliasItr != 0) { - // ...and its aliases. - for (; *aliasItr != 0; ++aliasItr) { - RegVector::iterator eraseItr = - std::find(liAllowed.begin(), liAllowed.end(), *aliasItr); + if (cp.isPhys()) { + if (!lis->isAllocatable(dst)) { + continue; + } - if (eraseItr != liAllowed.end()) { - liAllowed.erase(eraseItr); + const PBQPRAProblem::AllowedSet &allowed = p->getAllowedSet(src); + unsigned pregOpt = 0; + while (pregOpt < allowed.size() && allowed[pregOpt] != dst) { + ++pregOpt; + } + if (pregOpt < allowed.size()) { + ++pregOpt; // +1 to account for spill option. + PBQP::Graph::NodeItr node = p->getNodeForVReg(src); + addPhysRegCoalesce(g.getNodeCosts(node), pregOpt, cBenefit); + } + } else { + const PBQPRAProblem::AllowedSet *allowed1 = &p->getAllowedSet(dst); + const PBQPRAProblem::AllowedSet *allowed2 = &p->getAllowedSet(src); + PBQP::Graph::NodeItr node1 = p->getNodeForVReg(dst); + PBQP::Graph::NodeItr node2 = p->getNodeForVReg(src); + PBQP::Graph::EdgeItr edge = g.findEdge(node1, node2); + if (edge == g.edgesEnd()) { + edge = g.addEdge(node1, node2, PBQP::Matrix(allowed1->size() + 1, + allowed2->size() + 1, + 0)); + } else { + if (g.getEdgeNode1(edge) == node2) { + std::swap(node1, node2); + std::swap(allowed1, allowed2); } } + + addVirtRegCoalesce(g.getEdgeCosts(edge), *allowed1, *allowed2, + cBenefit); } } - - // Copy the allowed set into a member vector for use when constructing cost - // vectors & matrices, and mapping PBQP solutions back to assignments. - allowedSets[node] = AllowedSet(liAllowed.begin(), liAllowed.end()); - - // Set the spill cost to the interval weight, or epsilon if the - // interval weight is zero - PBQPNum spillCost = (li->weight != 0.0) ? - li->weight : std::numeric_limits::min(); - - // Build a cost vector for this interval. - add_pbqp_nodecosts(solver, node, - buildCostVector(li->reg, allowedSets[node], coalesces, - spillCost)); - } + return p; +} - // Now add the cost matrices... - for (unsigned node1 = 0; node1 < node2LI.size(); ++node1) { - const LiveInterval *li = node2LI[node1]; - - // Test for live range overlaps and insert interference matrices. - for (unsigned node2 = node1 + 1; node2 < node2LI.size(); ++node2) { - const LiveInterval *li2 = node2LI[node2]; +void PBQPBuilderWithCoalescing::addPhysRegCoalesce(PBQP::Vector &costVec, + unsigned pregOption, + PBQP::PBQPNum benefit) { + costVec[pregOption] += -benefit; +} - CoalesceMap::const_iterator cmItr = - coalesces.find(RegPair(li->reg, li2->reg)); +void PBQPBuilderWithCoalescing::addVirtRegCoalesce( + PBQP::Matrix &costMat, + const PBQPRAProblem::AllowedSet &vr1Allowed, + const PBQPRAProblem::AllowedSet &vr2Allowed, + PBQP::PBQPNum benefit) { - PBQPMatrix *m = 0; + assert(costMat.getRows() == vr1Allowed.size() + 1 && "Size mismatch."); + assert(costMat.getCols() == vr2Allowed.size() + 1 && "Size mismatch."); - if (cmItr != coalesces.end()) { - m = buildCoalescingMatrix(allowedSets[node1], allowedSets[node2], - cmItr->second); - } - else if (li->overlaps(*li2)) { - m = buildInterferenceMatrix(allowedSets[node1], allowedSets[node2]); - } + for (unsigned i = 0; i != vr1Allowed.size(); ++i) { + unsigned preg1 = vr1Allowed[i]; + for (unsigned j = 0; j != vr2Allowed.size(); ++j) { + unsigned preg2 = vr2Allowed[j]; - if (m != 0) { - add_pbqp_edgecosts(solver, node1, node2, m); - delete m; + if (preg1 == preg2) { + costMat[i + 1][j + 1] += -benefit; } } } +} + - // We're done, PBQP problem constructed - return it. - return solver; +void RegAllocPBQP::getAnalysisUsage(AnalysisUsage &au) const { + au.setPreservesCFG(); + au.addRequired(); + au.addPreserved(); + au.addRequired(); + au.addPreserved(); + au.addRequired(); + //au.addRequiredID(SplitCriticalEdgesID); + if (customPassID) + au.addRequiredID(*customPassID); + au.addRequired(); + au.addRequired(); + au.addPreserved(); + au.addRequired(); + au.addPreserved(); + au.addRequired(); + au.addPreserved(); + au.addRequired(); + au.addRequired(); + MachineFunctionPass::getAnalysisUsage(au); } -void PBQPRegAlloc::addStackInterval(const LiveInterval *spilled, - MachineRegisterInfo* mri) { - int stackSlot = vrm->getStackSlot(spilled->reg); +void RegAllocPBQP::findVRegIntervalsToAlloc() { - if (stackSlot == VirtRegMap::NO_STACK_SLOT) - return; + // Iterate over all live ranges. + for (LiveIntervals::iterator itr = lis->begin(), end = lis->end(); + itr != end; ++itr) { - const TargetRegisterClass *RC = mri->getRegClass(spilled->reg); - LiveInterval &stackInterval = lss->getOrCreateInterval(stackSlot, RC); + // Ignore physical ones. + if (TargetRegisterInfo::isPhysicalRegister(itr->first)) + continue; - VNInfo *vni; - if (stackInterval.getNumValNums() != 0) - vni = stackInterval.getValNumInfo(0); - else - vni = stackInterval.getNextValue(-0U, 0, lss->getVNInfoAllocator()); + LiveInterval *li = itr->second; - LiveInterval &rhsInterval = lis->getInterval(spilled->reg); - stackInterval.MergeRangesInAsValue(rhsInterval, vni); + // If this live interval is non-empty we will use pbqp to allocate it. + // Empty intervals we allocate in a simple post-processing stage in + // finalizeAlloc. + if (!li->empty()) { + vregsToAlloc.insert(li->reg); + } else { + emptyIntervalVRegs.insert(li->reg); + } + } } -bool PBQPRegAlloc::mapPBQPToRegAlloc(pbqp *problem) { - +bool RegAllocPBQP::mapPBQPToRegAlloc(const PBQPRAProblem &problem, + const PBQP::Solution &solution) { // Set to true if we have any spills bool anotherRoundNeeded = false; // Clear the existing allocation. vrm->clearAllVirt(); - // Iterate over the nodes mapping the PBQP solution to a register assignment. - for (unsigned node = 0; node < node2LI.size(); ++node) { - unsigned virtReg = node2LI[node]->reg, - allocSelection = get_pbqp_solution(problem, node); - - // If the PBQP solution is non-zero it's a physical register... - if (allocSelection != 0) { - // Get the physical reg, subtracting 1 to account for the spill option. - unsigned physReg = allowedSets[node][allocSelection - 1]; - - DOUT << "VREG " << virtReg << " -> " << tri->getName(physReg) << "\n"; - - assert(physReg != 0); - - // Add to the virt reg map and update the used phys regs. - vrm->assignVirt2Phys(virtReg, physReg); - } - // ...Otherwise it's a spill. - else { - - // Make sure we ignore this virtual reg on the next round - // of allocation - vregIntervalsToAlloc.erase(&lis->getInterval(virtReg)); - - // Insert spill ranges for this live range - const LiveInterval *spillInterval = node2LI[node]; - double oldSpillWeight = spillInterval->weight; - SmallVector spillIs; - std::vector newSpills = - lis->addIntervalsForSpills(*spillInterval, spillIs, loopInfo, *vrm); - addStackInterval(spillInterval, mri); - - DOUT << "VREG " << virtReg << " -> SPILLED (Cost: " - << oldSpillWeight << ", New vregs: "; + const PBQP::Graph &g = problem.getGraph(); + // Iterate over the nodes mapping the PBQP solution to a register + // assignment. + for (PBQP::Graph::ConstNodeItr node = g.nodesBegin(), + nodeEnd = g.nodesEnd(); + node != nodeEnd; ++node) { + unsigned vreg = problem.getVRegForNode(node); + unsigned alloc = solution.getSelection(node); + + if (problem.isPRegOption(vreg, alloc)) { + unsigned preg = problem.getPRegForOption(vreg, alloc); + DEBUG(dbgs() << "VREG " << PrintReg(vreg, tri) << " -> " + << tri->getName(preg) << "\n"); + assert(preg != 0 && "Invalid preg selected."); + vrm->assignVirt2Phys(vreg, preg); + } else if (problem.isSpillOption(vreg, alloc)) { + vregsToAlloc.erase(vreg); + SmallVector newSpills; + LiveRangeEdit LRE(&lis->getInterval(vreg), newSpills, *mf, *lis, vrm); + spiller->spill(LRE); + + DEBUG(dbgs() << "VREG " << PrintReg(vreg, tri) << " -> SPILLED (Cost: " + << LRE.getParent().weight << ", New vregs: "); // Copy any newly inserted live intervals into the list of regs to // allocate. - for (std::vector::const_iterator - itr = newSpills.begin(), end = newSpills.end(); + for (LiveRangeEdit::iterator itr = LRE.begin(), end = LRE.end(); itr != end; ++itr) { - assert(!(*itr)->empty() && "Empty spill range."); - - DOUT << (*itr)->reg << " "; - - vregIntervalsToAlloc.insert(*itr); + DEBUG(dbgs() << PrintReg((*itr)->reg, tri) << " "); + vregsToAlloc.insert((*itr)->reg); } - DOUT << ")\n"; + DEBUG(dbgs() << ")\n"); // We need another round if spill intervals were added. - anotherRoundNeeded |= !newSpills.empty(); + anotherRoundNeeded |= !LRE.empty(); + } else { + llvm_unreachable("Unknown allocation option."); } } return !anotherRoundNeeded; } -void PBQPRegAlloc::finalizeAlloc() const { + +void RegAllocPBQP::finalizeAlloc() const { typedef LiveIntervals::iterator LIIterator; typedef LiveInterval::Ranges::const_iterator LRIterator; // First allocate registers for the empty intervals. - for (LiveIntervalSet::const_iterator - itr = emptyVRegIntervals.begin(), end = emptyVRegIntervals.end(); + for (RegSet::const_iterator + itr = emptyIntervalVRegs.begin(), end = emptyIntervalVRegs.end(); itr != end; ++itr) { - LiveInterval *li = *itr; + LiveInterval *li = &lis->getInterval(*itr); - unsigned physReg = li->preference; + unsigned physReg = vrm->getRegAllocPref(li->reg); if (physReg == 0) { const TargetRegisterClass *liRC = mri->getRegClass(li->reg); - physReg = *liRC->allocation_order_begin(*mf); + physReg = liRC->getRawAllocationOrder(*mf).front(); } vrm->assignVirt2Phys(li->reg, physReg); @@ -756,17 +608,15 @@ void PBQPRegAlloc::finalizeAlloc() const { // Get the physical register for this interval if (TargetRegisterInfo::isPhysicalRegister(li->reg)) { reg = li->reg; - } - else if (vrm->isAssignedReg(li->reg)) { + } else if (vrm->isAssignedReg(li->reg)) { reg = vrm->getPhys(li->reg); - } - else { + } else { // Ranges which are assigned a stack slot only are ignored. continue; } - // Ignore unallocated vregs: if (reg == 0) { + // Filter out zero regs - they're for intervals that were spilled. continue; } @@ -777,7 +627,7 @@ void PBQPRegAlloc::finalizeAlloc() const { // Find the set of basic blocks which this range is live into... if (lis->findLiveInMBBs(lrItr->start, lrItr->end, liveInMBBs)) { // And add the physreg for this interval to their live-in sets. - for (unsigned i = 0; i < liveInMBBs.size(); ++i) { + for (unsigned i = 0; i != liveInMBBs.size(); ++i) { if (liveInMBBs[i] != entryMBB) { if (!liveInMBBs[i]->isLiveIn(reg)) { liveInMBBs[i]->addLiveIn(reg); @@ -791,7 +641,7 @@ void PBQPRegAlloc::finalizeAlloc() const { } -bool PBQPRegAlloc::runOnMachineFunction(MachineFunction &MF) { +bool RegAllocPBQP::runOnMachineFunction(MachineFunction &MF) { mf = &MF; tm = &mf->getTarget(); @@ -802,10 +652,14 @@ bool PBQPRegAlloc::runOnMachineFunction(MachineFunction &MF) { lis = &getAnalysis(); lss = &getAnalysis(); loopInfo = &getAnalysis(); + rmf = &getAnalysis(); vrm = &getAnalysis(); + spiller.reset(createInlineSpiller(*this, MF, *vrm)); - DOUT << "PBQP Register Allocating for " << mf->getFunction()->getName() << "\n"; + mri->freezeReservedRegs(MF); + + DEBUG(dbgs() << "PBQP Register Allocating for " << mf->getFunction()->getName() << "\n"); // Allocator main loop: // @@ -819,26 +673,42 @@ bool PBQPRegAlloc::runOnMachineFunction(MachineFunction &MF) { // Find the vreg intervals in need of allocation. findVRegIntervalsToAlloc(); - // If there aren't any then we're done here. - if (vregIntervalsToAlloc.empty() && emptyVRegIntervals.empty()) - return true; + const Function* func = mf->getFunction(); + std::string fqn = + func->getParent()->getModuleIdentifier() + "." + + func->getName().str(); + (void)fqn; // If there are non-empty intervals allocate them using pbqp. - if (!vregIntervalsToAlloc.empty()) { + if (!vregsToAlloc.empty()) { bool pbqpAllocComplete = false; unsigned round = 0; while (!pbqpAllocComplete) { - DOUT << " PBQP Regalloc round " << round << ":\n"; - - pbqp *problem = constructPBQPProblem(); - - solve_pbqp(problem); + DEBUG(dbgs() << " PBQP Regalloc round " << round << ":\n"); + + std::auto_ptr problem = + builder->build(mf, lis, loopInfo, vregsToAlloc); + +#ifndef NDEBUG + if (pbqpDumpGraphs) { + std::ostringstream rs; + rs << round; + std::string graphFileName(fqn + "." + rs.str() + ".pbqpgraph"); + std::string tmp; + raw_fd_ostream os(graphFileName.c_str(), tmp); + DEBUG(dbgs() << "Dumping graph for round " << round << " to \"" + << graphFileName << "\"\n"); + problem->getGraph().dump(os); + } +#endif - pbqpAllocComplete = mapPBQPToRegAlloc(problem); + PBQP::Solution solution = + PBQP::HeuristicSolver::solve( + problem->getGraph()); - free_pbqp(problem); + pbqpAllocComplete = mapPBQPToRegAlloc(*problem, solution); ++round; } @@ -847,25 +717,29 @@ bool PBQPRegAlloc::runOnMachineFunction(MachineFunction &MF) { // Finalise allocation, allocate empty ranges. finalizeAlloc(); - vregIntervalsToAlloc.clear(); - emptyVRegIntervals.clear(); - li2Node.clear(); - node2LI.clear(); - allowedSets.clear(); + rmf->renderMachineFunction("After PBQP register allocation.", vrm); - DOUT << "Post alloc VirtRegMap:\n" << *vrm << "\n"; + vregsToAlloc.clear(); + emptyIntervalVRegs.clear(); - // Run rewriter - std::auto_ptr rewriter(createVirtRegRewriter()); - - rewriter->runOnMachineFunction(*mf, *vrm, lis); + DEBUG(dbgs() << "Post alloc VirtRegMap:\n" << *vrm << "\n"); return true; } -FunctionPass* llvm::createPBQPRegisterAllocator() { - return new PBQPRegAlloc(); +FunctionPass* llvm::createPBQPRegisterAllocator( + std::auto_ptr builder, + char *customPassID) { + return new RegAllocPBQP(builder, customPassID); } +FunctionPass* llvm::createDefaultPBQPRegisterAllocator() { + if (pbqpCoalescing) { + return createPBQPRegisterAllocator( + std::auto_ptr(new PBQPBuilderWithCoalescing())); + } // else + return createPBQPRegisterAllocator( + std::auto_ptr(new PBQPBuilder())); +} #undef DEBUG_TYPE