};
class MachineBasicBlock : public ilist_node<MachineBasicBlock> {
+public:
+ /// Pair of physical register and lane mask.
+ /// This is not simply a std::pair typedef because the members should be named
+ /// clearly as they both have an integer type.
+ struct RegisterMaskPair {
+ public:
+ MCPhysReg PhysReg;
+ unsigned LaneMask;
+
+ RegisterMaskPair(MCPhysReg PhysReg, unsigned LaneMask)
+ : PhysReg(PhysReg), LaneMask(LaneMask) {}
+ };
+
+protected:
typedef ilist<MachineInstr> Instructions;
Instructions Insts;
const BasicBlock *BB;
typedef std::vector<uint32_t>::const_iterator const_weight_iterator;
/// Keep track of the physical registers that are livein of the basicblock.
- typedef std::vector<MCPhysReg> LiveInVector;
+ typedef std::vector<RegisterMaskPair> LiveInVector;
LiveInVector LiveIns;
/// Alignment of the basic block. Zero if the basic block does not need to be
/// Adds the specified register as a live in. Note that it is an error to add
/// the same register to the same set more than once unless the intention is
/// to call sortUniqueLiveIns after all registers are added.
- void addLiveIn(MCPhysReg PhysReg) { LiveIns.push_back(PhysReg); }
+ void addLiveIn(MCPhysReg PhysReg, unsigned LaneMask = ~0u) {
+ LiveIns.push_back(RegisterMaskPair(PhysReg, LaneMask));
+ }
+ void addLiveIn(const RegisterMaskPair &RegMaskPair) {
+ LiveIns.push_back(RegMaskPair);
+ }
/// Sorts and uniques the LiveIns vector. It can be significantly faster to do
/// this than repeatedly calling isLiveIn before calling addLiveIn for every
/// LiveIn insertion.
- void sortUniqueLiveIns() {
- std::sort(LiveIns.begin(), LiveIns.end());
- LiveIns.erase(std::unique(LiveIns.begin(), LiveIns.end()), LiveIns.end());
- }
+ void sortUniqueLiveIns();
/// Add PhysReg as live in to this block, and ensure that there is a copy of
/// PhysReg to a virtual register of class RC. Return the virtual register
unsigned addLiveIn(MCPhysReg PhysReg, const TargetRegisterClass *RC);
/// Remove the specified register from the live in set.
- void removeLiveIn(MCPhysReg Reg);
+ void removeLiveIn(MCPhysReg Reg, unsigned LaneMask = ~0u);
/// Return true if the specified register is in the live in set.
- bool isLiveIn(MCPhysReg Reg) const;
+ bool isLiveIn(MCPhysReg Reg, unsigned LaneMask = ~0u) const;
// Iteration support for live in sets. These sets are kept in sorted
// order by their register number.
}
/// Tell the scavenger a register is used.
- void setRegUsed(unsigned Reg);
+ void setRegUsed(unsigned Reg, unsigned LaneMask = ~0u);
private:
/// Returns true if a register is reserved. It is never "unused".
bool isReserved(unsigned Reg) const { return MRI->isReserved(Reg); }
// Examine the live-in regs of all successors.
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
- for (unsigned LI : (*SI)->liveins()) {
- for (MCRegAliasIterator AI(LI, TRI, true); AI.isValid(); ++AI) {
+ for (const auto &LI : (*SI)->liveins()) {
+ for (MCRegAliasIterator AI(LI.PhysReg, TRI, true); AI.isValid(); ++AI) {
unsigned Reg = *AI;
State->UnionGroups(Reg, 0);
KillIndices[Reg] = BB->size();
// Examine the live-in regs of all successors.
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
- for (unsigned LI : (*SI)->liveins()) {
- for (MCRegAliasIterator AI(LI, TRI, true); AI.isValid(); ++AI) {
+ for (const auto &LI : (*SI)->liveins()) {
+ for (MCRegAliasIterator AI(LI.PhysReg, TRI, true); AI.isValid(); ++AI) {
unsigned Reg = *AI;
Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1);
KillIndices[Reg] = BBSize;
// block.
for (MachineBasicBlock::succ_iterator S = MBB.succ_begin(),
E = MBB.succ_end(); S != E; S++)
- for (unsigned LI : (*S)->liveins())
- LivePhysRegs.set(LI);
+ for (const auto &LI : (*S)->liveins())
+ LivePhysRegs.set(LI.PhysReg);
// Now scan the instructions and delete dead ones, tracking physreg
// liveness as we go.
// This is the entry block.
if (MBB->pred_empty()) {
- for (unsigned LI : MBB->liveins()) {
- for (int rx : regIndices(LI)) {
+ for (const auto &LI : MBB->liveins()) {
+ for (int rx : regIndices(LI.PhysReg)) {
// Treat function live-ins as if they were defined just before the first
// instruction. Usually, function arguments are set up immediately
// before the call.
// Create phi-defs at Begin for all live-in registers.
SlotIndex Begin = Indexes->getMBBStartIdx(MBB);
DEBUG(dbgs() << Begin << "\tBB#" << MBB->getNumber());
- for (unsigned LI : MBB->liveins()) {
- for (MCRegUnitIterator Units(LI, TRI); Units.isValid(); ++Units) {
+ for (const auto &LI : MBB->liveins()) {
+ for (MCRegUnitIterator Units(LI.PhysReg, TRI); Units.isValid(); ++Units) {
unsigned Unit = *Units;
LiveRange *LR = RegUnitRanges[Unit];
if (!LR) {
/// Add live-in registers of basic block \p MBB to \p LiveRegs.
static void addLiveIns(LivePhysRegs &LiveRegs, const MachineBasicBlock &MBB) {
- for (unsigned LI : MBB.liveins())
- LiveRegs.addReg(LI);
+ for (const auto &LI : MBB.liveins())
+ LiveRegs.addReg(LI.PhysReg);
}
/// Add pristine registers to the given \p LiveRegs. This function removes
void LiveVariables::runOnBlock(MachineBasicBlock *MBB, const unsigned NumRegs) {
// Mark live-in registers as live-in.
SmallVector<unsigned, 4> Defs;
- for (unsigned LI : MBB->liveins()) {
- assert(TargetRegisterInfo::isPhysicalRegister(LI) &&
+ for (const auto &LI : MBB->liveins()) {
+ assert(TargetRegisterInfo::isPhysicalRegister(LI.PhysReg) &&
"Cannot have a live-in virtual register!");
- HandlePhysRegDef(LI, nullptr, Defs);
+ HandlePhysRegDef(LI.PhysReg, nullptr, Defs);
}
// Loop over all of the instructions, processing them.
MachineBasicBlock *SuccMBB = *SI;
if (SuccMBB->isEHPad())
continue;
- for (unsigned LI : SuccMBB->liveins()) {
- if (!TRI->isInAllocatableClass(LI))
+ for (const auto &LI : SuccMBB->liveins()) {
+ if (!TRI->isInAllocatableClass(LI.PhysReg))
// Ignore other live-ins, e.g. those that are live into landing pads.
- LiveOuts.insert(LI);
+ LiveOuts.insert(LI.PhysReg);
}
}
#include "llvm/IR/Module.h"
#include "llvm/IR/ModuleSlotTracker.h"
#include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Format.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/YAMLTraits.h"
if (!MBB.livein_empty()) {
OS.indent(2) << "liveins: ";
bool First = true;
- for (unsigned LI : MBB.liveins()) {
+ for (const auto &LI : MBB.liveins()) {
if (!First)
OS << ", ";
First = false;
- printReg(LI, OS, TRI);
+ printReg(LI.PhysReg, OS, TRI);
+ if (LI.LaneMask != ~0u)
+ OS << format(":%08X", LI.LaneMask);
}
OS << "\n";
HasLineAttributes = true;
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/Format.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetInstrInfo.h"
#include "llvm/Target/TargetMachine.h"
if (!livein_empty()) {
if (Indexes) OS << '\t';
OS << " Live Ins:";
- for (unsigned LI : make_range(livein_begin(), livein_end())) {
- OS << ' ' << PrintReg(LI, TRI);
+ for (const auto &LI : make_range(livein_begin(), livein_end())) {
+ OS << ' ' << PrintReg(LI.PhysReg, TRI);
+ if (LI.LaneMask != ~0u)
+ OS << format(":%08X", LI.LaneMask);
}
OS << '\n';
}
OS << "BB#" << getNumber();
}
-void MachineBasicBlock::removeLiveIn(MCPhysReg Reg) {
- LiveInVector::iterator I = std::find(LiveIns.begin(), LiveIns.end(), Reg);
- if (I != LiveIns.end())
+void MachineBasicBlock::removeLiveIn(MCPhysReg Reg, unsigned LaneMask) {
+ LiveInVector::iterator I = std::find_if(
+ LiveIns.begin(), LiveIns.end(),
+ [Reg] (const RegisterMaskPair &LI) { return LI.PhysReg == Reg; });
+ if (I == LiveIns.end())
+ return;
+
+ I->LaneMask &= ~LaneMask;
+ if (I->LaneMask == 0)
LiveIns.erase(I);
}
-bool MachineBasicBlock::isLiveIn(MCPhysReg Reg) const {
- livein_iterator I = std::find(livein_begin(), livein_end(), Reg);
- return I != livein_end();
+bool MachineBasicBlock::isLiveIn(MCPhysReg Reg, unsigned LaneMask) const {
+ livein_iterator I = std::find_if(
+ LiveIns.begin(), LiveIns.end(),
+ [Reg] (const RegisterMaskPair &LI) { return LI.PhysReg == Reg; });
+ return I != livein_end() && (I->LaneMask & LaneMask) != 0;
+}
+
+void MachineBasicBlock::sortUniqueLiveIns() {
+ std::sort(LiveIns.begin(), LiveIns.end(),
+ [](const RegisterMaskPair &LI0, const RegisterMaskPair &LI1) {
+ return LI0.PhysReg < LI1.PhysReg;
+ });
+ // Liveins are sorted by physreg now we can merge their lanemasks.
+ LiveInVector::const_iterator I = LiveIns.begin();
+ LiveInVector::const_iterator J;
+ LiveInVector::iterator Out = LiveIns.begin();
+ for (; I != LiveIns.end(); ++Out, I = J) {
+ unsigned PhysReg = I->PhysReg;
+ unsigned LaneMask = I->LaneMask;
+ for (J = std::next(I); J != LiveIns.end() && J->PhysReg == PhysReg; ++J)
+ LaneMask |= J->LaneMask;
+ Out->PhysReg = PhysReg;
+ Out->LaneMask = LaneMask;
+ }
+ LiveIns.erase(Out, LiveIns.end());
}
unsigned
i->getOperand(ni+1).setMBB(NMBB);
// Inherit live-ins from the successor
- for (unsigned LI : Succ->liveins())
+ for (const auto &LI : Succ->liveins())
NMBB->addLiveIn(LI);
// Update LiveVariables.
// Conservatively treat live-in's as an external def.
// FIXME: That means a reload that're reused in successor block(s) will not
// be LICM'ed.
- for (unsigned LI : BB->liveins()) {
- for (MCRegAliasIterator AI(LI, TRI, true); AI.isValid(); ++AI)
+ for (const auto &LI : BB->liveins()) {
+ for (MCRegAliasIterator AI(LI.PhysReg, TRI, true); AI.isValid(); ++AI)
PhysRegDefs.set(*AI);
}
if (MRI->isSSA()) {
// If this block has allocatable physical registers live-in, check that
// it is an entry block or landing pad.
- for (unsigned LI : MBB->liveins()) {
- if (isAllocatable(LI) && !MBB->isEHPad() &&
+ for (const auto &LI : MBB->liveins()) {
+ if (isAllocatable(LI.PhysReg) && !MBB->isEHPad() &&
MBB != MBB->getParent()->begin()) {
report("MBB has allocable live-in, but isn't entry or landing-pad.", MBB);
}
}
regsLive.clear();
- for (unsigned LI : MBB->liveins()) {
- if (!TargetRegisterInfo::isPhysicalRegister(LI)) {
+ for (const auto &LI : MBB->liveins()) {
+ if (!TargetRegisterInfo::isPhysicalRegister(LI.PhysReg)) {
report("MBB live-in list contains non-physical register", MBB);
continue;
}
- for (MCSubRegIterator SubRegs(LI, TRI, /*IncludeSelf=*/true);
+ for (MCSubRegIterator SubRegs(LI.PhysReg, TRI, /*IncludeSelf=*/true);
SubRegs.isValid(); ++SubRegs)
regsLive.insert(*SubRegs);
}
MachineBasicBlock::iterator MII = MBB->begin();
// Add live-in registers as live.
- for (unsigned LI : MBB->liveins())
- if (MRI->isAllocatable(LI))
- definePhysReg(MII, LI, regReserved);
+ for (const auto &LI : MBB->liveins())
+ if (MRI->isAllocatable(LI.PhysReg))
+ definePhysReg(MII, LI.PhysReg, regReserved);
SmallVector<unsigned, 8> VirtDead;
SmallVector<MachineInstr*, 32> Coalesced;
#define DEBUG_TYPE "reg-scavenging"
/// setUsed - Set the register units of this register as used.
-void RegScavenger::setRegUsed(unsigned Reg) {
- for (MCRegUnitIterator RUI(Reg, TRI); RUI.isValid(); ++RUI)
- RegUnitsAvailable.reset(*RUI);
+void RegScavenger::setRegUsed(unsigned Reg, unsigned LaneMask) {
+ for (MCRegUnitMaskIterator RUI(Reg, TRI); RUI.isValid(); ++RUI) {
+ unsigned UnitMask = (*RUI).second;
+ if (UnitMask == 0 || (LaneMask & UnitMask) != 0)
+ RegUnitsAvailable.reset((*RUI).first);
+ }
}
void RegScavenger::initRegState() {
return;
// Live-in registers are in use.
- for (unsigned LI : MBB->liveins())
- setRegUsed(LI);
+ for (const auto &LI : MBB->liveins())
+ setRegUsed(LI.PhysReg, LI.LaneMask);
// Pristine CSRs are also unavailable.
const MachineFunction &MF = *MBB->getParent();
assert(Uses.empty() && "Uses in set before adding deps?");
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
- for (unsigned LI : (*SI)->liveins()) {
- if (!Uses.contains(LI))
- Uses.insert(PhysRegSUOper(&ExitSU, -1, LI));
+ for (const auto &LI : (*SI)->liveins()) {
+ if (!Uses.contains(LI.PhysReg))
+ Uses.insert(PhysRegSUOper(&ExitSU, -1, LI.PhysReg));
}
}
}
// Examine the live-in regs of all successors.
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI) {
- for (unsigned LI : (*SI)->liveins()) {
+ for (const auto &LI : (*SI)->liveins()) {
// Repeat, for reg and all subregs.
- for (MCSubRegIterator SubRegs(LI, TRI, /*IncludeSelf=*/true);
+ for (MCSubRegIterator SubRegs(LI.PhysReg, TRI, /*IncludeSelf=*/true);
SubRegs.isValid(); ++SubRegs)
LiveRegs.set(*SubRegs);
}
RS->enterBasicBlock(PredBB);
if (!PredBB->empty())
RS->forward(std::prev(PredBB->end()));
- for (unsigned LI : TailBB->liveins()) {
- if (!RS->isRegUsed(LI, false))
+ for (const auto &LI : TailBB->liveins()) {
+ if (!RS->isRegUsed(LI.PhysReg, false))
// If a register is previously livein to the tail but it's not live
// at the end of predecessor BB, then it should be added to its
// livein list.
if (LaneMask == 0)
continue;
MachineBasicBlock *MBB = MBBI->second;
- for (MCSubRegIndexIterator SR(PhysReg, TRI); SR.isValid(); ++SR) {
- unsigned SubReg = SR.getSubReg();
- unsigned SubRegIndex = SR.getSubRegIndex();
- unsigned SubRegLaneMask = TRI->getSubRegIndexLaneMask(SubRegIndex);
- if ((SubRegLaneMask & LaneMask) != 0)
- MBB->addLiveIn(SubReg);
- }
+ MBB->addLiveIn(PhysReg, LaneMask);
}
}
for (int Idx = 0; Idx < NbAddedBlocks; ++Idx)
BeforePrologueRegion.insert(AddedBlocks[Idx]);
- for (unsigned LI : PrologueMBB.liveins()) {
+ for (const auto &LI : PrologueMBB.liveins()) {
for (MachineBasicBlock *PredBB : BeforePrologueRegion)
PredBB->addLiveIn(LI);
}
// The live-in to LayoutSucc is now all values live-in to
// JumpAroundTarget.
//
- std::vector<unsigned> OrigLiveIn(LayoutSucc->livein_begin(),
- LayoutSucc->livein_end());
- std::vector<unsigned> NewLiveIn(JumpAroundTarget->livein_begin(),
- JumpAroundTarget->livein_end());
- for (unsigned i = 0; i < OrigLiveIn.size(); ++i) {
- LayoutSucc->removeLiveIn(OrigLiveIn[i]);
- }
- for (unsigned i = 0; i < NewLiveIn.size(); ++i) {
- LayoutSucc->addLiveIn(NewLiveIn[i]);
- }
+ std::vector<MachineBasicBlock::RegisterMaskPair> OrigLiveIn(
+ LayoutSucc->livein_begin(), LayoutSucc->livein_end());
+ std::vector<MachineBasicBlock::RegisterMaskPair> NewLiveIn(
+ JumpAroundTarget->livein_begin(),
+ JumpAroundTarget->livein_end());
+ for (const auto &OrigLI : OrigLiveIn)
+ LayoutSucc->removeLiveIn(OrigLI.PhysReg);
+ for (const auto &NewLI : NewLiveIn)
+ LayoutSucc->addLiveIn(NewLI);
}
}
}
for (MachineBasicBlock::const_succ_iterator SI = MBB.succ_begin(),
SE = MBB.succ_end(); SI != SE; ++SI)
if (*SI != &SuccBB)
- for (unsigned LI : (*SI)->liveins())
- Uses.set(LI);
+ for (const auto &LI : (*SI)->liveins())
+ Uses.set(LI.PhysReg);
}
bool RegDefsUses::update(const MachineInstr &MI, unsigned Begin, unsigned End) {
unsigned LiveLow = 0;
unsigned LiveHigh = 0;
for (auto SI = MBB.succ_begin(), SE = MBB.succ_end(); SI != SE; ++SI) {
- for (auto LI = (*SI)->livein_begin(), LE = (*SI)->livein_end();
- LI != LE; ++LI) {
- unsigned Reg = *LI;
+ for (const auto &LI : (*SI)->liveins()) {
+ unsigned Reg = LI.PhysReg;
assert(Reg < SystemZ::NUM_TARGET_REGS && "Invalid register number");
LiveLow |= LowGPRs[Reg];
LiveHigh |= HighGPRs[Reg];
// Return a bitmask of FP registers in block's live-in list.
static unsigned calcLiveInMask(MachineBasicBlock *MBB) {
unsigned Mask = 0;
- for (unsigned LI : MBB->liveins()) {
- if (LI < X86::FP0 || LI > X86::FP6)
+ for (const auto &LI : MBB->liveins()) {
+ if (LI.PhysReg < X86::FP0 || LI.PhysReg > X86::FP6)
continue;
- Mask |= 1 << (LI - X86::FP0);
+ Mask |= 1 << (LI.PhysReg - X86::FP0);
}
return Mask;
}
// The MOV R10, RAX needs to be in a different block, since the RET we emit in
// allocMBB needs to be last (terminating) instruction.
- for (unsigned LI : PrologueMBB.liveins()) {
+ for (const auto &LI : PrologueMBB.liveins()) {
allocMBB->addLiveIn(LI);
checkMBB->addLiveIn(LI);
}
MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock();
MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock();
- for (unsigned LI : PrologueMBB.liveins()) {
+ for (const auto &LI : PrologueMBB.liveins()) {
stackCheckMBB->addLiveIn(LI);
incStackMBB->addLiveIn(LI);
}