X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FTailDuplication.cpp;h=4377236323a709fad4190910c504fa031b2615d3;hb=fe0bf8fbf9d401a67a1842da6cefbb58aa8975a3;hp=ac4e2c452523a2f031f01de93c36f8a7f576265a;hpb=d2a7bedbc9d3db35ff424a6a2d257c72341af224;p=oota-llvm.git diff --git a/lib/CodeGen/TailDuplication.cpp b/lib/CodeGen/TailDuplication.cpp index ac4e2c45252..4377236323a 100644 --- a/lib/CodeGen/TailDuplication.cpp +++ b/lib/CodeGen/TailDuplication.cpp @@ -12,25 +12,30 @@ // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "tailduplication" -#include "llvm/Function.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/MachineSSAUpdater.h" -#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/CodeGen/RegisterScavenging.h" +#include "llvm/IR/Function.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SetVector.h" -#include "llvm/ADT/Statistic.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; +#define DEBUG_TYPE "tailduplication" + STATISTIC(NumTails , "Number of tails duplicated"); STATISTIC(NumTailDups , "Number of tail duplicated blocks"); STATISTIC(NumInstrDups , "Additional instructions due to tail duplication"); @@ -57,8 +62,11 @@ namespace { /// TailDuplicatePass - Perform tail duplication. class TailDuplicatePass : public MachineFunctionPass { const TargetInstrInfo *TII; + const TargetRegisterInfo *TRI; + const MachineBranchProbabilityInfo *MBPI; MachineModuleInfo *MMI; MachineRegisterInfo *MRI; + std::unique_ptr RS; bool PreRegAlloc; // SSAUpdateVRs - A list of virtual registers for which to update SSA form. @@ -73,8 +81,9 @@ namespace { explicit TailDuplicatePass() : MachineFunctionPass(ID), PreRegAlloc(false) {} - virtual bool runOnMachineFunction(MachineFunction &MF); - virtual const char *getPassName() const { return "Tail Duplication"; } + bool runOnMachineFunction(MachineFunction &MF) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override; private: void AddSSAUpdateEntry(unsigned OrigReg, unsigned NewReg, @@ -82,7 +91,7 @@ namespace { void ProcessPHI(MachineInstr *MI, MachineBasicBlock *TailBB, MachineBasicBlock *PredBB, DenseMap &LocalVRMap, - SmallVector, 4> &Copies, + SmallVectorImpl > &Copies, const DenseSet &UsedByPhi, bool Remove); void DuplicateInstruction(MachineInstr *MI, @@ -92,7 +101,7 @@ namespace { DenseMap &LocalVRMap, const DenseSet &UsedByPhi); void UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, - SmallVector &TDBBs, + SmallVectorImpl &TDBBs, SmallSetVector &Succs); bool TailDuplicateBlocks(MachineFunction &MF); bool shouldTailDuplicate(const MachineFunction &MF, @@ -100,14 +109,14 @@ namespace { bool isSimpleBB(MachineBasicBlock *TailBB); bool canCompletelyDuplicateBB(MachineBasicBlock &BB); bool duplicateSimpleBB(MachineBasicBlock *TailBB, - SmallVector &TDBBs, + SmallVectorImpl &TDBBs, const DenseSet &RegsUsedByPhi, - SmallVector &Copies); + SmallVectorImpl &Copies); bool TailDuplicate(MachineBasicBlock *TailBB, bool IsSimple, MachineFunction &MF, - SmallVector &TDBBs, - SmallVector &Copies); + SmallVectorImpl &TDBBs, + SmallVectorImpl &Copies); bool TailDuplicateAndUpdate(MachineBasicBlock *MBB, bool IsSimple, MachineFunction &MF); @@ -118,15 +127,25 @@ namespace { char TailDuplicatePass::ID = 0; } -FunctionPass *llvm::createTailDuplicatePass() { - return new TailDuplicatePass(); -} +char &llvm::TailDuplicateID = TailDuplicatePass::ID; + +INITIALIZE_PASS(TailDuplicatePass, "tailduplication", "Tail Duplication", + false, false) bool TailDuplicatePass::runOnMachineFunction(MachineFunction &MF) { - TII = MF.getTarget().getInstrInfo(); + if (skipOptnoneFunction(*MF.getFunction())) + return false; + + TII = MF.getSubtarget().getInstrInfo(); + TRI = MF.getSubtarget().getRegisterInfo(); MRI = &MF.getRegInfo(); MMI = getAnalysisIfAvailable(); + MBPI = &getAnalysis(); + PreRegAlloc = MRI->isSSA(); + RS.reset(); + if (MRI->tracksLiveness() && TRI->trackLivenessAfterRegAlloc(MF)) + RS.reset(new RegScavenger()); bool MadeChange = false; while (TailDuplicateBlocks(MF)) @@ -135,6 +154,11 @@ bool TailDuplicatePass::runOnMachineFunction(MachineFunction &MF) { return MadeChange; } +void TailDuplicatePass::getAnalysisUsage(AnalysisUsage &AU) const { + AU.addRequired(); + MachineFunctionPass::getAnalysisUsage(AU); +} + static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) { for (MachineFunction::iterator I = ++MF.begin(), E = MF.end(); I != E; ++I) { MachineBasicBlock *MBB = I; @@ -159,7 +183,7 @@ static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) { dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI; dbgs() << " missing input from predecessor BB#" << PredBB->getNumber() << '\n'; - llvm_unreachable(0); + llvm_unreachable(nullptr); } } @@ -170,12 +194,12 @@ static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) { << ": " << *MI; dbgs() << " extra input from predecessor BB#" << PHIBB->getNumber() << '\n'; - llvm_unreachable(0); + llvm_unreachable(nullptr); } if (PHIBB->getNumber() < 0) { dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI; dbgs() << " non-existing BB#" << PHIBB->getNumber() << '\n'; - llvm_unreachable(0); + llvm_unreachable(nullptr); } } ++MI; @@ -225,7 +249,7 @@ TailDuplicatePass::TailDuplicateAndUpdate(MachineBasicBlock *MBB, // If the original definition is still around, add it as an available // value. MachineInstr *DefMI = MRI->getVRegDef(VReg); - MachineBasicBlock *DefBB = 0; + MachineBasicBlock *DefBB = nullptr; if (DefMI) { DefBB = DefMI->getParent(); SSAUpdate.AddAvailableValue(DefBB, VReg); @@ -243,8 +267,8 @@ TailDuplicatePass::TailDuplicateAndUpdate(MachineBasicBlock *MBB, // Rewrite uses that are outside of the original def's block. MachineRegisterInfo::use_iterator UI = MRI->use_begin(VReg); while (UI != MRI->use_end()) { - MachineOperand &UseMO = UI.getOperand(); - MachineInstr *UseMI = &*UI; + MachineOperand &UseMO = *UI; + MachineInstr *UseMI = UseMO.getParent(); ++UI; if (UseMI->isDebugValue()) { // SSAUpdate can replace the use with an undef. That creates @@ -272,8 +296,8 @@ TailDuplicatePass::TailDuplicateAndUpdate(MachineBasicBlock *MBB, continue; unsigned Dst = Copy->getOperand(0).getReg(); unsigned Src = Copy->getOperand(1).getReg(); - MachineRegisterInfo::use_iterator UI = MRI->use_begin(Src); - if (++UI == MRI->use_end()) { + if (MRI->hasOneNonDBGUse(Src) && + MRI->constrainRegClass(Src, MRI->getRegClass(Dst))) { // Copy is the only use. Do trivial copy propagation here. MRI->replaceRegWith(Dst, Src); Copy->eraseFromParent(); @@ -319,12 +343,10 @@ bool TailDuplicatePass::TailDuplicateBlocks(MachineFunction &MF) { static bool isDefLiveOut(unsigned Reg, MachineBasicBlock *BB, const MachineRegisterInfo *MRI) { - for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(Reg), - UE = MRI->use_end(); UI != UE; ++UI) { - MachineInstr *UseMI = &*UI; - if (UseMI->isDebugValue()) + for (MachineInstr &UseMI : MRI->use_instructions(Reg)) { + if (UseMI.isDebugValue()) continue; - if (UseMI->getParent() != BB) + if (UseMI.getParent() != BB) return true; } return false; @@ -343,9 +365,7 @@ static unsigned getPHISrcRegOpIdx(MachineInstr *MI, MachineBasicBlock *SrcBB) { // block (which is why we need to copy the information). static void getRegsUsedByPHIs(const MachineBasicBlock &BB, DenseSet *UsedByPhi) { - for(MachineBasicBlock::const_iterator I = BB.begin(), E = BB.end(); - I != E; ++I) { - const MachineInstr &MI = *I; + for (const auto &MI : BB) { if (!MI.isPHI()) break; for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) { @@ -373,13 +393,11 @@ void TailDuplicatePass::AddSSAUpdateEntry(unsigned OrigReg, unsigned NewReg, /// ProcessPHI - Process PHI node in TailBB by turning it into a copy in PredBB. /// Remember the source register that's contributed by PredBB and update SSA /// update map. -void TailDuplicatePass::ProcessPHI(MachineInstr *MI, - MachineBasicBlock *TailBB, - MachineBasicBlock *PredBB, - DenseMap &LocalVRMap, - SmallVector, 4> &Copies, - const DenseSet &RegsUsedByPhi, - bool Remove) { +void TailDuplicatePass::ProcessPHI( + MachineInstr *MI, MachineBasicBlock *TailBB, MachineBasicBlock *PredBB, + DenseMap &LocalVRMap, + SmallVectorImpl > &Copies, + const DenseSet &RegsUsedByPhi, bool Remove) { unsigned DefReg = MI->getOperand(0).getReg(); unsigned SrcOpIdx = getPHISrcRegOpIdx(MI, PredBB); assert(SrcOpIdx && "Unable to find matching PHI source?"); @@ -429,11 +447,13 @@ void TailDuplicatePass::DuplicateInstruction(MachineInstr *MI, AddSSAUpdateEntry(Reg, NewReg, PredBB); } else { DenseMap::iterator VI = LocalVRMap.find(Reg); - if (VI != LocalVRMap.end()) + if (VI != LocalVRMap.end()) { MO.setReg(VI->second); + MRI->constrainRegClass(VI->second, MRI->getRegClass(Reg)); + } } } - PredBB->insert(PredBB->end(), NewMI); + PredBB->insert(PredBB->instr_end(), NewMI); } /// UpdateSuccessorsPHIs - After FromBB is tail duplicated into its predecessor @@ -441,7 +461,7 @@ void TailDuplicatePass::DuplicateInstruction(MachineInstr *MI, /// instructions in them accordingly. void TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, - SmallVector &TDBBs, + SmallVectorImpl &TDBBs, SmallSetVector &Succs) { for (SmallSetVector::iterator SI = Succs.begin(), SE = Succs.end(); SI != SE; ++SI) { @@ -450,6 +470,7 @@ TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, II != EE; ++II) { if (!II->isPHI()) break; + MachineInstrBuilder MIB(*FromBB->getParent(), II); unsigned Idx = 0; for (unsigned i = 1, e = II->getNumOperands(); i != e; i += 2) { MachineOperand &MO = II->getOperand(i+1); @@ -497,8 +518,7 @@ TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, II->getOperand(Idx+1).setMBB(SrcBB); Idx = 0; } else { - II->addOperand(MachineOperand::CreateReg(SrcReg, false)); - II->addOperand(MachineOperand::CreateMBB(SrcBB)); + MIB.addReg(SrcReg).addMBB(SrcBB); } } } else { @@ -510,8 +530,7 @@ TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, II->getOperand(Idx+1).setMBB(SrcBB); Idx = 0; } else { - II->addOperand(MachineOperand::CreateReg(Reg, false)); - II->addOperand(MachineOperand::CreateMBB(SrcBB)); + MIB.addReg(Reg).addMBB(SrcBB); } } } @@ -541,7 +560,8 @@ TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, // compensate for the duplication. unsigned MaxDuplicateCount; if (TailDuplicateSize.getNumOccurrences() == 0 && - MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize)) + MF.getFunction()->getAttributes(). + hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize)) MaxDuplicateCount = 1; else MaxDuplicateCount = TailDuplicateSize; @@ -629,8 +649,6 @@ bothUsedInPHI(const MachineBasicBlock &A, bool TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB) { - SmallPtrSet Succs(BB.succ_begin(), BB.succ_end()); - for (MachineBasicBlock::pred_iterator PI = BB.pred_begin(), PE = BB.pred_end(); PI != PE; ++PI) { MachineBasicBlock *PredBB = *PI; @@ -638,7 +656,7 @@ TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB) { if (PredBB->succ_size() > 1) return false; - MachineBasicBlock *PredTBB = NULL, *PredFBB = NULL; + MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr; SmallVector PredCond; if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)) return false; @@ -651,9 +669,9 @@ TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB) { bool TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, - SmallVector &TDBBs, - const DenseSet &UsedByPhi, - SmallVector &Copies) { + SmallVectorImpl &TDBBs, + const DenseSet &UsedByPhi, + SmallVectorImpl &Copies) { SmallPtrSet Succs(TailBB->succ_begin(), TailBB->succ_end()); SmallVector Preds(TailBB->pred_begin(), @@ -669,7 +687,7 @@ TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, if (bothUsedInPHI(*PredBB, Succs)) continue; - MachineBasicBlock *PredTBB = NULL, *PredFBB = NULL; + MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr; SmallVector PredCond; if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)) continue; @@ -679,7 +697,7 @@ TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, << "From simple Succ: " << *TailBB); MachineBasicBlock *NewTarget = *TailBB->succ_begin(); - MachineBasicBlock *NextBB = llvm::next(MachineFunction::iterator(PredBB)); + MachineBasicBlock *NextBB = std::next(MachineFunction::iterator(PredBB)); // Make PredFBB explicit. if (PredCond.empty()) @@ -700,25 +718,26 @@ TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, // Make the branch unconditional if possible if (PredTBB == PredFBB) { PredCond.clear(); - PredFBB = NULL; + PredFBB = nullptr; } // Avoid adding fall through branches. if (PredFBB == NextBB) - PredFBB = NULL; - if (PredTBB == NextBB && PredFBB == NULL) - PredTBB = NULL; + PredFBB = nullptr; + if (PredTBB == NextBB && PredFBB == nullptr) + PredTBB = nullptr; TII->RemoveBranch(*PredBB); if (PredTBB) TII->InsertBranch(*PredBB, PredTBB, PredFBB, PredCond, DebugLoc()); + uint32_t Weight = MBPI->getEdgeWeight(PredBB, TailBB); PredBB->removeSuccessor(TailBB); unsigned NumSuccessors = PredBB->succ_size(); assert(NumSuccessors <= 1); if (NumSuccessors == 0 || *PredBB->succ_begin() != NewTarget) - PredBB->addSuccessor(NewTarget); + PredBB->addSuccessor(NewTarget, Weight); TDBBs.push_back(PredBB); } @@ -731,8 +750,8 @@ bool TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, bool IsSimple, MachineFunction &MF, - SmallVector &TDBBs, - SmallVector &Copies) { + SmallVectorImpl &TDBBs, + SmallVectorImpl &Copies) { DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n'); DenseSet UsedByPhi; @@ -775,11 +794,28 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, // Remove PredBB's unconditional branch. TII->RemoveBranch(*PredBB); + if (RS && !TailBB->livein_empty()) { + // Update PredBB livein. + RS->enterBasicBlock(PredBB); + if (!PredBB->empty()) + RS->forward(std::prev(PredBB->end())); + for (MachineBasicBlock::livein_iterator I = TailBB->livein_begin(), + E = TailBB->livein_end(); I != E; ++I) { + if (!RS->isRegUsed(*I, false)) + // If a register is previously livein to the tail but it's not live + // at the end of predecessor BB, then it should be added to its + // livein list. + PredBB->addLiveIn(*I); + } + } + // Clone the contents of TailBB into PredBB. DenseMap LocalVRMap; SmallVector, 4> CopyInfos; - MachineBasicBlock::iterator I = TailBB->begin(); - while (I != TailBB->end()) { + // Use instr_iterator here to properly handle bundles, e.g. + // ARM Thumb2 IT block. + MachineBasicBlock::instr_iterator I = TailBB->instr_begin(); + while (I != TailBB->instr_end()) { MachineInstr *MI = &*I; ++I; if (MI->isPHI()) { @@ -810,7 +846,7 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, "TailDuplicate called on block with multiple successors!"); for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(), E = TailBB->succ_end(); I != E; ++I) - PredBB->addSuccessor(*I); + PredBB->addSuccessor(*I, MBPI->getEdgeWeight(TailBB, I)); Changed = true; ++NumTailDups; @@ -819,8 +855,8 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, // If TailBB was duplicated into all its predecessors except for the prior // block, which falls through unconditionally, move the contents of this // block into the prior block. - MachineBasicBlock *PrevBB = prior(MachineFunction::iterator(TailBB)); - MachineBasicBlock *PriorTBB = 0, *PriorFBB = 0; + MachineBasicBlock *PrevBB = std::prev(MachineFunction::iterator(TailBB)); + MachineBasicBlock *PriorTBB = nullptr, *PriorFBB = nullptr; SmallVector PriorCond; // This has to check PrevBB->succ_size() because EH edges are ignored by // AnalyzeBranch. @@ -849,6 +885,7 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, // Replace def of virtual registers with new registers, and update // uses with PHI source register or the new registers. MachineInstr *MI = &*I++; + assert(!MI->isBundle() && "Not expecting bundles before regalloc!"); DuplicateInstruction(MI, TailBB, PrevBB, MF, LocalVRMap, UsedByPhi); MI->eraseFromParent(); }