#define DEBUG_TYPE "stackcoloring"
#include "llvm/CodeGen/Passes.h"
-#include "MachineTraceMetrics.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SparseSet.h"
#include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/Dominators.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/LiveInterval.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
-#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/CodeGen/SlotIndexes.h"
-#include "llvm/DebugInfo.h"
-#include "llvm/Function.h"
-#include "llvm/Instructions.h"
+#include "llvm/CodeGen/StackProtector.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
#include "llvm/MC/MCInstrItineraries.h"
-#include "llvm/Module.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
/// code. If this flag is enabled, we try to save the user.
static cl::opt<bool>
ProtectFromEscapedAllocas("protect-from-escaped-allocas",
- cl::init(false), cl::Hidden,
- cl::desc("Do not optimize lifetime zones that are broken"));
+ cl::init(false), cl::Hidden,
+ cl::desc("Do not optimize lifetime zones that "
+ "are broken"));
STATISTIC(NumMarkerSeen, "Number of lifetime markers found.");
STATISTIC(StackSpaceSaved, "Number of bytes saved due to merging slots.");
STATISTIC(StackSlotMerged, "Number of stack slot merged.");
-STATISTIC(EscapedAllocas,
- "Number of allocas that escaped the lifetime region");
+STATISTIC(EscapedAllocas, "Number of allocas that escaped the lifetime region");
//===----------------------------------------------------------------------===//
// StackColoring Pass
};
/// Maps active slots (per bit) for each basic block.
- DenseMap<MachineBasicBlock*, BlockLifetimeInfo> BlockLiveness;
+ typedef DenseMap<const MachineBasicBlock*, BlockLifetimeInfo> LivenessMap;
+ LivenessMap BlockLiveness;
/// Maps serial numbers to basic blocks.
- DenseMap<MachineBasicBlock*, int> BasicBlocks;
+ DenseMap<const MachineBasicBlock*, int> BasicBlocks;
/// Maps basic blocks to a serial number.
- SmallVector<MachineBasicBlock*, 8> BasicBlockNumbering;
+ SmallVector<const MachineBasicBlock*, 8> BasicBlockNumbering;
/// Maps liveness intervals for each slot.
SmallVector<LiveInterval*, 16> Intervals;
VNInfo::Allocator VNInfoAllocator;
/// SlotIndex analysis object.
SlotIndexes *Indexes;
+ /// The stack protector object.
+ StackProtector *SP;
/// The list of lifetime markers found. These markers are to be removed
/// once the coloring is done.
SmallVector<MachineInstr*, 8> Markers;
- /// SlotSizeSorter - A Sort utility for arranging stack slots according
- /// to their size.
- struct SlotSizeSorter {
- MachineFrameInfo *MFI;
- SlotSizeSorter(MachineFrameInfo *mfi) : MFI(mfi) { }
- bool operator()(int LHS, int RHS) {
- // We use -1 to denote a uninteresting slot. Place these slots at the end.
- if (LHS == -1) return false;
- if (RHS == -1) return true;
- // Sort according to size.
- return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS);
- }
-};
-
public:
static char ID;
StackColoring() : MachineFunctionPass(ID) {
private:
/// Debug.
- void dump();
+ void dump() const;
/// Removes all of the lifetime marker instructions from the function.
/// \returns true if any markers were removed.
/// slots to use the joint slots.
void remapInstructions(DenseMap<int, int> &SlotRemap);
- /// The input program may contain intructions which are not inside lifetime
+ /// The input program may contain instructions which are not inside lifetime
/// markers. This can happen due to a bug in the compiler or due to a bug in
/// user code (for example, returning a reference to a local variable).
/// This procedure checks all of the instructions in the function and
"stack-coloring", "Merge disjoint stack slots", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
+INITIALIZE_PASS_DEPENDENCY(StackProtector)
INITIALIZE_PASS_END(StackColoring,
"stack-coloring", "Merge disjoint stack slots", false, false)
AU.addRequired<MachineDominatorTree>();
AU.addPreserved<MachineDominatorTree>();
AU.addRequired<SlotIndexes>();
+ AU.addRequired<StackProtector>();
MachineFunctionPass::getAnalysisUsage(AU);
}
-void StackColoring::dump() {
+void StackColoring::dump() const {
for (df_iterator<MachineFunction*> FI = df_begin(MF), FE = df_end(MF);
FI != FE; ++FI) {
- unsigned Num = BasicBlocks[*FI];
- DEBUG(dbgs()<<"Inspecting block #"<<Num<<" ["<<FI->getName()<<"]\n");
- Num = 0;
+ DEBUG(dbgs()<<"Inspecting block #"<<BasicBlocks.lookup(*FI)<<
+ " ["<<FI->getName()<<"]\n");
+
+ LivenessMap::const_iterator BI = BlockLiveness.find(*FI);
+ assert(BI != BlockLiveness.end() && "Block not found");
+ const BlockLifetimeInfo &BlockInfo = BI->second;
+
DEBUG(dbgs()<<"BEGIN : {");
- for (unsigned i=0; i < BlockLiveness[*FI].Begin.size(); ++i)
- DEBUG(dbgs()<<BlockLiveness[*FI].Begin.test(i)<<" ");
+ for (unsigned i=0; i < BlockInfo.Begin.size(); ++i)
+ DEBUG(dbgs()<<BlockInfo.Begin.test(i)<<" ");
DEBUG(dbgs()<<"}\n");
DEBUG(dbgs()<<"END : {");
- for (unsigned i=0; i < BlockLiveness[*FI].End.size(); ++i)
- DEBUG(dbgs()<<BlockLiveness[*FI].End.test(i)<<" ");
+ for (unsigned i=0; i < BlockInfo.End.size(); ++i)
+ DEBUG(dbgs()<<BlockInfo.End.test(i)<<" ");
DEBUG(dbgs()<<"}\n");
DEBUG(dbgs()<<"LIVE_IN: {");
- for (unsigned i=0; i < BlockLiveness[*FI].LiveIn.size(); ++i)
- DEBUG(dbgs()<<BlockLiveness[*FI].LiveIn.test(i)<<" ");
+ for (unsigned i=0; i < BlockInfo.LiveIn.size(); ++i)
+ DEBUG(dbgs()<<BlockInfo.LiveIn.test(i)<<" ");
DEBUG(dbgs()<<"}\n");
DEBUG(dbgs()<<"LIVEOUT: {");
- for (unsigned i=0; i < BlockLiveness[*FI].LiveOut.size(); ++i)
- DEBUG(dbgs()<<BlockLiveness[*FI].LiveOut.test(i)<<" ");
+ for (unsigned i=0; i < BlockInfo.LiveOut.size(); ++i)
+ DEBUG(dbgs()<<BlockInfo.LiveOut.test(i)<<" ");
DEBUG(dbgs()<<"}\n");
}
}
BasicBlocks[*FI] = BasicBlockNumbering.size();
BasicBlockNumbering.push_back(*FI);
- BlockLiveness[*FI].Begin.resize(NumSlot);
- BlockLiveness[*FI].End.resize(NumSlot);
+ // Keep a reference to avoid repeated lookups.
+ BlockLifetimeInfo &BlockInfo = BlockLiveness[*FI];
+
+ BlockInfo.Begin.resize(NumSlot);
+ BlockInfo.End.resize(NumSlot);
for (MachineBasicBlock::iterator BI = (*FI)->begin(), BE = (*FI)->end();
BI != BE; ++BI) {
Markers.push_back(BI);
bool IsStart = BI->getOpcode() == TargetOpcode::LIFETIME_START;
- MachineOperand &MI = BI->getOperand(0);
+ const MachineOperand &MI = BI->getOperand(0);
unsigned Slot = MI.getIndex();
MarkersFound++;
}
if (IsStart) {
- BlockLiveness[*FI].Begin.set(Slot);
+ BlockInfo.Begin.set(Slot);
} else {
- if (BlockLiveness[*FI].Begin.test(Slot)) {
+ if (BlockInfo.Begin.test(Slot)) {
// Allocas that start and end within a single block are handled
// specially when computing the LiveIntervals to avoid pessimizing
// the liveness propagation.
- BlockLiveness[*FI].Begin.reset(Slot);
+ BlockInfo.Begin.reset(Slot);
} else {
- BlockLiveness[*FI].End.set(Slot);
+ BlockInfo.End.set(Slot);
}
}
}
// formulation, and END is equivalent to GEN. The result of this computation
// is a map from blocks to bitvectors where the bitvectors represent which
// allocas are live in/out of that block.
- SmallPtrSet<MachineBasicBlock*, 8> BBSet(BasicBlockNumbering.begin(),
- BasicBlockNumbering.end());
+ SmallPtrSet<const MachineBasicBlock*, 8> BBSet(BasicBlockNumbering.begin(),
+ BasicBlockNumbering.end());
unsigned NumSSMIters = 0;
bool changed = true;
while (changed) {
changed = false;
++NumSSMIters;
- SmallPtrSet<MachineBasicBlock*, 8> NextBBSet;
+ SmallPtrSet<const MachineBasicBlock*, 8> NextBBSet;
- for (SmallVector<MachineBasicBlock*, 8>::iterator
- PI = BasicBlockNumbering.begin(), PE = BasicBlockNumbering.end();
- PI != PE; ++PI) {
+ for (SmallVectorImpl<const MachineBasicBlock *>::iterator
+ PI = BasicBlockNumbering.begin(), PE = BasicBlockNumbering.end();
+ PI != PE; ++PI) {
- MachineBasicBlock *BB = *PI;
+ const MachineBasicBlock *BB = *PI;
if (!BBSet.count(BB)) continue;
+ // Use an iterator to avoid repeated lookups.
+ LivenessMap::iterator BI = BlockLiveness.find(BB);
+ assert(BI != BlockLiveness.end() && "Block not found");
+ BlockLifetimeInfo &BlockInfo = BI->second;
+
BitVector LocalLiveIn;
BitVector LocalLiveOut;
// Forward propagation from begins to ends.
- for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(),
- PE = BB->pred_end(); PI != PE; ++PI)
- LocalLiveIn |= BlockLiveness[*PI].LiveOut;
- LocalLiveIn |= BlockLiveness[BB].End;
- LocalLiveIn.reset(BlockLiveness[BB].Begin);
+ for (MachineBasicBlock::const_pred_iterator PI = BB->pred_begin(),
+ PE = BB->pred_end(); PI != PE; ++PI) {
+ LivenessMap::const_iterator I = BlockLiveness.find(*PI);
+ assert(I != BlockLiveness.end() && "Predecessor not found");
+ LocalLiveIn |= I->second.LiveOut;
+ }
+ LocalLiveIn |= BlockInfo.End;
+ LocalLiveIn.reset(BlockInfo.Begin);
// Reverse propagation from ends to begins.
- for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
- SE = BB->succ_end(); SI != SE; ++SI)
- LocalLiveOut |= BlockLiveness[*SI].LiveIn;
- LocalLiveOut |= BlockLiveness[BB].Begin;
- LocalLiveOut.reset(BlockLiveness[BB].End);
+ for (MachineBasicBlock::const_succ_iterator SI = BB->succ_begin(),
+ SE = BB->succ_end(); SI != SE; ++SI) {
+ LivenessMap::const_iterator I = BlockLiveness.find(*SI);
+ assert(I != BlockLiveness.end() && "Successor not found");
+ LocalLiveOut |= I->second.LiveIn;
+ }
+ LocalLiveOut |= BlockInfo.Begin;
+ LocalLiveOut.reset(BlockInfo.End);
LocalLiveIn |= LocalLiveOut;
LocalLiveOut |= LocalLiveIn;
// After adopting the live bits, we need to turn-off the bits which
// are de-activated in this block.
- LocalLiveOut.reset(BlockLiveness[BB].End);
- LocalLiveIn.reset(BlockLiveness[BB].Begin);
+ LocalLiveOut.reset(BlockInfo.End);
+ LocalLiveIn.reset(BlockInfo.Begin);
// If we have both BEGIN and END markers in the same basic block then
// we know that the BEGIN marker comes after the END, because we already
// Want to enable the LIVE_IN and LIVE_OUT of slots that have both
// BEGIN and END because it means that the value lives before and after
// this basic block.
- BitVector LocalEndBegin = BlockLiveness[BB].End;
- LocalEndBegin &= BlockLiveness[BB].Begin;
+ BitVector LocalEndBegin = BlockInfo.End;
+ LocalEndBegin &= BlockInfo.Begin;
LocalLiveIn |= LocalEndBegin;
LocalLiveOut |= LocalEndBegin;
- if (LocalLiveIn.test(BlockLiveness[BB].LiveIn)) {
+ if (LocalLiveIn.test(BlockInfo.LiveIn)) {
changed = true;
- BlockLiveness[BB].LiveIn |= LocalLiveIn;
+ BlockInfo.LiveIn |= LocalLiveIn;
- for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(),
+ for (MachineBasicBlock::const_pred_iterator PI = BB->pred_begin(),
PE = BB->pred_end(); PI != PE; ++PI)
NextBBSet.insert(*PI);
}
- if (LocalLiveOut.test(BlockLiveness[BB].LiveOut)) {
+ if (LocalLiveOut.test(BlockInfo.LiveOut)) {
changed = true;
- BlockLiveness[BB].LiveOut |= LocalLiveOut;
+ BlockInfo.LiveOut |= LocalLiveOut;
- for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
+ for (MachineBasicBlock::const_succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
NextBBSet.insert(*SI);
}
Finishes.resize(NumSlots);
// Create the interval for the basic blocks with lifetime markers in them.
- for (SmallVector<MachineInstr*, 8>::iterator it = Markers.begin(),
+ for (SmallVectorImpl<MachineInstr*>::const_iterator it = Markers.begin(),
e = Markers.end(); it != e; ++it) {
- MachineInstr *MI = *it;
+ const MachineInstr *MI = *it;
if (MI->getParent() != MBB)
continue;
"Invalid Lifetime marker");
bool IsStart = MI->getOpcode() == TargetOpcode::LIFETIME_START;
- MachineOperand &Mo = MI->getOperand(0);
+ const MachineOperand &Mo = MI->getOperand(0);
int Slot = Mo.getIndex();
assert(Slot >= 0 && "Invalid slot");
}
// Create the interval of the blocks that we previously found to be 'alive'.
- BitVector Alive = BlockLiveness[MBB].LiveIn;
- Alive |= BlockLiveness[MBB].LiveOut;
-
- if (Alive.any()) {
- for (int pos = Alive.find_first(); pos != -1;
- pos = Alive.find_next(pos)) {
- if (!Starts[pos].isValid())
- Starts[pos] = Indexes->getMBBStartIdx(MBB);
- if (!Finishes[pos].isValid())
- Finishes[pos] = Indexes->getMBBEndIdx(MBB);
- }
+ BlockLifetimeInfo &MBBLiveness = BlockLiveness[MBB];
+ for (int pos = MBBLiveness.LiveIn.find_first(); pos != -1;
+ pos = MBBLiveness.LiveIn.find_next(pos)) {
+ Starts[pos] = Indexes->getMBBStartIdx(MBB);
+ }
+ for (int pos = MBBLiveness.LiveOut.find_first(); pos != -1;
+ pos = MBBLiveness.LiveOut.find_next(pos)) {
+ Finishes[pos] = Indexes->getMBBEndIdx(MBB);
}
for (unsigned i = 0; i < NumSlots; ++i) {
SlotIndex F = Finishes[i];
if (S < F) {
// We have a single consecutive region.
- Intervals[i]->addRange(LiveRange(S, F, ValNum));
+ Intervals[i]->addSegment(LiveInterval::Segment(S, F, ValNum));
} else {
- // We have two non consecutive regions. This happens when
+ // We have two non-consecutive regions. This happens when
// LIFETIME_START appears after the LIFETIME_END marker.
SlotIndex NewStart = Indexes->getMBBStartIdx(MBB);
SlotIndex NewFin = Indexes->getMBBEndIdx(MBB);
- Intervals[i]->addRange(LiveRange(NewStart, F, ValNum));
- Intervals[i]->addRange(LiveRange(S, NewFin, ValNum));
+ Intervals[i]->addSegment(LiveInterval::Segment(NewStart, F, ValNum));
+ Intervals[i]->addSegment(LiveInterval::Segment(S, NewFin, ValNum));
}
}
}
// Keep a list of *allocas* which need to be remapped.
DenseMap<const AllocaInst*, const AllocaInst*> Allocas;
- for (DenseMap<int, int>::iterator it = SlotRemap.begin(),
+ for (DenseMap<int, int>::const_iterator it = SlotRemap.begin(),
e = SlotRemap.end(); it != e; ++it) {
const AllocaInst *From = MFI->getObjectAllocation(it->first);
const AllocaInst *To = MFI->getObjectAllocation(it->second);
assert(To && From && "Invalid allocation object");
Allocas[From] = To;
+
+ // AA might be used later for instruction scheduling, and we need it to be
+ // able to deduce the correct aliasing releationships between pointers
+ // derived from the alloca being remapped and the target of that remapping.
+ // The only safe way, without directly informing AA about the remapping
+ // somehow, is to directly update the IR to reflect the change being made
+ // here.
+ Instruction *Inst = const_cast<AllocaInst *>(To);
+ if (From->getType() != To->getType()) {
+ BitCastInst *Cast = new BitCastInst(Inst, From->getType());
+ Cast->insertAfter(Inst);
+ Inst = Cast;
+ }
+
+ // Allow the stack protector to adjust its value map to account for the
+ // upcoming replacement.
+ SP->adjustForColoring(From, To);
+
+ // Note that this will not replace uses in MMOs (which we'll update below),
+ // or anywhere else (which is why we won't delete the original
+ // instruction).
+ const_cast<AllocaInst *>(From)->replaceAllUsesWith(Inst);
}
// Remap all instructions to the new stack slots.
if (!V)
continue;
- // Climb up and find the original alloca.
- V = GetUnderlyingObject(V);
- // If we did not find one, or if the one that we found is not in our
- // map, then move on.
- if (!V || !isa<AllocaInst>(V)) {
- // Clear mem operand since we don't know for sure that it doesn't
- // alias a merged alloca.
- MMO->setValue(0);
+ // FIXME: In order to enable the use of TBAA when using AA in CodeGen,
+ // we'll also need to update the TBAA nodes in MMOs with values
+ // derived from the merged allocas. When doing this, we'll need to use
+ // the same variant of GetUnderlyingObjects that is used by the
+ // instruction scheduler (that can look through ptrtoint/inttoptr
+ // pairs).
+
+ // We've replaced IR-level uses of the remapped allocas, so we only
+ // need to replace direct uses here.
+ if (!isa<AllocaInst>(V))
continue;
- }
+
const AllocaInst *AI= cast<AllocaInst>(V);
if (!Allocas.count(AI))
continue;
SlotIndex Index = Indexes->getInstructionIndex(I);
LiveInterval *Interval = Intervals[FromSlot];
assert(Interval->find(Index) != Interval->end() &&
- "Found instruction usage outside of live range.");
+ "Found instruction usage outside of live range.");
}
#endif
}
void StackColoring::removeInvalidSlotRanges() {
- MachineFunction::iterator BB, BBE;
- MachineBasicBlock::iterator I, IE;
+ MachineFunction::const_iterator BB, BBE;
+ MachineBasicBlock::const_iterator I, IE;
for (BB = MF->begin(), BBE = MF->end(); BB != BBE; ++BB)
for (I = BB->begin(), IE = BB->end(); I != IE; ++I) {
// Check all of the machine operands.
for (unsigned i = 0 ; i < I->getNumOperands(); ++i) {
- MachineOperand &MO = I->getOperand(i);
+ const MachineOperand &MO = I->getOperand(i);
if (!MO.isFI())
continue;
MF = &Func;
MFI = MF->getFrameInfo();
Indexes = &getAnalysis<SlotIndexes>();
+ SP = &getAnalysis<StackProtector>();
BlockLiveness.clear();
BasicBlocks.clear();
BasicBlockNumbering.clear();
// Sort the slots according to their size. Place unused slots at the end.
// Use stable sort to guarantee deterministic code generation.
std::stable_sort(SortedSlots.begin(), SortedSlots.end(),
- SlotSizeSorter(MFI));
-
- bool Chanded = true;
- while (Chanded) {
- Chanded = false;
+ [this](int LHS, int RHS) {
+ // We use -1 to denote a uninteresting slot. Place these slots at the end.
+ if (LHS == -1) return false;
+ if (RHS == -1) return true;
+ // Sort according to size.
+ return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS);
+ });
+
+ bool Changed = true;
+ while (Changed) {
+ Changed = false;
for (unsigned I = 0; I < NumSlots; ++I) {
if (SortedSlots[I] == -1)
continue;
// Merge disjoint slots.
if (!First->overlaps(*Second)) {
- Chanded = true;
- First->MergeRangesInAsValue(*Second, First->getValNumInfo(0));
+ Changed = true;
+ First->MergeSegmentsInAsValue(*Second, First->getValNumInfo(0));
SlotRemap[SecondSlot] = FirstSlot;
SortedSlots[J] = -1;
DEBUG(dbgs()<<"Merging #"<<FirstSlot<<" and slots #"<<
projects / oota-llvm.git / blobdiff