#include "llvm/Type.h"
#include "llvm/DerivedTypes.h"
#include "llvm/Analysis/Dominators.h"
-#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Assembly/AsmAnnotationWriter.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
return new IVUsers();
}
-/// containsAddRecFromDifferentLoop - Determine whether expression S involves a
-/// subexpression that is an AddRec from a loop other than L. An outer loop
-/// of L is OK, but not an inner loop nor a disjoint loop.
-static bool containsAddRecFromDifferentLoop(const SCEV *S, Loop *L) {
- // This is very common, put it first.
- if (isa<SCEVConstant>(S))
- return false;
- if (const SCEVCommutativeExpr *AE = dyn_cast<SCEVCommutativeExpr>(S)) {
- for (unsigned int i=0; i< AE->getNumOperands(); i++)
- if (containsAddRecFromDifferentLoop(AE->getOperand(i), L))
- return true;
- return false;
- }
- if (const SCEVAddRecExpr *AE = dyn_cast<SCEVAddRecExpr>(S)) {
- if (const Loop *newLoop = AE->getLoop()) {
- if (newLoop == L)
- return false;
- // if newLoop is an outer loop of L, this is OK.
- if (!LoopInfo::isNotAlreadyContainedIn(L, newLoop))
- return false;
+/// CollectSubexprs - Split S into subexpressions which can be pulled out into
+/// separate registers.
+static void CollectSubexprs(const SCEV *S,
+ SmallVectorImpl<const SCEV *> &Ops,
+ ScalarEvolution &SE) {
+ if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
+ // Break out add operands.
+ for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
+ I != E; ++I)
+ CollectSubexprs(*I, Ops, SE);
+ return;
+ } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
+ // Split a non-zero base out of an addrec.
+ if (!AR->getStart()->isZero()) {
+ CollectSubexprs(AR->getStart(), Ops, SE);
+ CollectSubexprs(SE.getAddRecExpr(SE.getIntegerSCEV(0, AR->getType()),
+ AR->getStepRecurrence(SE),
+ AR->getLoop()), Ops, SE);
+ return;
}
- return true;
}
- if (const SCEVUDivExpr *DE = dyn_cast<SCEVUDivExpr>(S))
- return containsAddRecFromDifferentLoop(DE->getLHS(), L) ||
- containsAddRecFromDifferentLoop(DE->getRHS(), L);
-#if 0
- // SCEVSDivExpr has been backed out temporarily, but will be back; we'll
- // need this when it is.
- if (const SCEVSDivExpr *DE = dyn_cast<SCEVSDivExpr>(S))
- return containsAddRecFromDifferentLoop(DE->getLHS(), L) ||
- containsAddRecFromDifferentLoop(DE->getRHS(), L);
-#endif
- if (const SCEVCastExpr *CE = dyn_cast<SCEVCastExpr>(S))
- return containsAddRecFromDifferentLoop(CE->getOperand(), L);
- return false;
+
+ // Otherwise use the value itself.
+ Ops.push_back(S);
}
/// getSCEVStartAndStride - Compute the start and stride of this expression,
if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(SH)) {
for (unsigned i = 0, e = AE->getNumOperands(); i != e; ++i)
if (const SCEVAddRecExpr *AddRec =
- dyn_cast<SCEVAddRecExpr>(AE->getOperand(i))) {
- if (AddRec->getLoop() == L)
- TheAddRec = SE->getAddExpr(AddRec, TheAddRec);
- else
- return false; // Nested IV of some sort?
- } else {
+ dyn_cast<SCEVAddRecExpr>(AE->getOperand(i)))
+ TheAddRec = SE->getAddExpr(AddRec, TheAddRec);
+ else
Start = SE->getAddExpr(Start, AE->getOperand(i));
- }
} else if (isa<SCEVAddRecExpr>(SH)) {
TheAddRec = SH;
} else {
return false; // not analyzable.
}
- const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(TheAddRec);
- if (!AddRec || AddRec->getLoop() != L) return false;
+ // Break down TheAddRec into its component parts.
+ SmallVector<const SCEV *, 4> Subexprs;
+ CollectSubexprs(TheAddRec, Subexprs, *SE);
+
+ // Look for an addrec on the current loop among the parts.
+ const SCEV *AddRecStride = 0;
+ for (SmallVectorImpl<const SCEV *>::iterator I = Subexprs.begin(),
+ E = Subexprs.end(); I != E; ++I) {
+ const SCEV *S = *I;
+ if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S))
+ if (AR->getLoop() == L) {
+ *I = AR->getStart();
+ AddRecStride = AR->getStepRecurrence(*SE);
+ break;
+ }
+ }
+ if (!AddRecStride)
+ return false;
+
+ // Add up everything else into a start value (which may not be
+ // loop-invariant).
+ const SCEV *AddRecStart = SE->getAddExpr(Subexprs);
// Use getSCEVAtScope to attempt to simplify other loops out of
// the picture.
- const SCEV *AddRecStart = AddRec->getStart();
AddRecStart = SE->getSCEVAtScope(AddRecStart, UseLoop);
- const SCEV *AddRecStride = AddRec->getStepRecurrence(*SE);
-
- // FIXME: If Start contains an SCEVAddRecExpr from a different loop, other
- // than an outer loop of the current loop, reject it. LSR has no concept of
- // operating on more than one loop at a time so don't confuse it with such
- // expressions.
- if (containsAddRecFromDifferentLoop(AddRecStart, L))
- return false;
Start = SE->getAddExpr(Start, AddRecStart);
- // If stride is an instruction, make sure it dominates the loop preheader.
+ // If stride is an instruction, make sure it properly dominates the header.
// Otherwise we could end up with a use before def situation.
if (!isa<SCEVConstant>(AddRecStride)) {
- BasicBlock *Preheader = L->getLoopPreheader();
- if (!AddRecStride->dominates(Preheader, DT))
+ BasicBlock *Header = L->getHeader();
+ if (!AddRecStride->properlyDominates(Header, DT))
return false;
- DOUT << "[" << L->getHeader()->getName()
- << "] Variable stride: " << *AddRec << "\n";
+ DEBUG(dbgs() << "[";
+ WriteAsOperand(dbgs(), L->getHeader(), /*PrintType=*/false);
+ dbgs() << "] Variable stride: " << *AddRecStride << "\n");
}
Stride = AddRecStride;
Loop *L, LoopInfo *LI, DominatorTree *DT,
Pass *P) {
// If the user is in the loop, use the preinc value.
- if (L->contains(User->getParent())) return false;
+ if (L->contains(User)) return false;
BasicBlock *LatchBlock = L->getLoopLatch();
+ if (!LatchBlock)
+ return false;
// Ok, the user is outside of the loop. If it is dominated by the latch
// block, use the post-inc value.
if (!getSCEVStartAndStride(ISE, L, UseLoop, Start, Stride, SE, DT))
return false; // Non-reducible symbolic expression, bail out.
+ // Keep things simple. Don't touch loop-variant strides.
+ if (!Stride->isLoopInvariant(L) && L->contains(I))
+ return false;
+
SmallPtrSet<Instruction *, 4> UniqueUsers;
for (Value::use_iterator UI = I->use_begin(), E = I->use_end();
UI != E; ++UI) {
if (LI->getLoopFor(User->getParent()) != L) {
if (isa<PHINode>(User) || Processed.count(User) ||
!AddUsersIfInteresting(User)) {
- DOUT << "FOUND USER in other loop: " << *User
- << " OF SCEV: " << *ISE << "\n";
+ DEBUG(dbgs() << "FOUND USER in other loop: " << *User << '\n'
+ << " OF SCEV: " << *ISE << '\n');
AddUserToIVUsers = true;
}
} else if (Processed.count(User) ||
!AddUsersIfInteresting(User)) {
- DOUT << "FOUND USER: " << *User
- << " OF SCEV: " << *ISE << "\n";
+ DEBUG(dbgs() << "FOUND USER: " << *User << '\n'
+ << " OF SCEV: " << *ISE << '\n');
AddUserToIVUsers = true;
}
if (AddUserToIVUsers) {
- IVUsersOfOneStride *StrideUses = IVUsesByStride[Stride];
- if (!StrideUses) { // First occurrence of this stride?
- StrideOrder.push_back(Stride);
- StrideUses = new IVUsersOfOneStride(Stride);
- IVUses.push_back(StrideUses);
- IVUsesByStride[Stride] = StrideUses;
- }
-
// Okay, we found a user that we cannot reduce. Analyze the instruction
// and decide what to do with it. If we are a use inside of the loop, use
// the value before incrementation, otherwise use it after incrementation.
// The value used will be incremented by the stride more than we are
// expecting, so subtract this off.
const SCEV *NewStart = SE->getMinusSCEV(Start, Stride);
- StrideUses->addUser(NewStart, User, I);
- StrideUses->Users.back().setIsUseOfPostIncrementedValue(true);
- DOUT << " USING POSTINC SCEV, START=" << *NewStart<< "\n";
+ IVUses.push_back(new IVStrideUse(this, Stride, NewStart, User, I));
+ IVUses.back().setIsUseOfPostIncrementedValue(true);
+ DEBUG(dbgs() << " USING POSTINC SCEV, START=" << *NewStart<< "\n");
} else {
- StrideUses->addUser(Start, User, I);
+ IVUses.push_back(new IVStrideUse(this, Stride, Start, User, I));
}
}
}
return true;
}
+IVStrideUse &IVUsers::AddUser(const SCEV *Stride, const SCEV *Offset,
+ Instruction *User, Value *Operand) {
+ IVUses.push_back(new IVStrideUse(this, Stride, Offset, User, Operand));
+ return IVUses.back();
+}
+
IVUsers::IVUsers()
: LoopPass(&ID) {
}
/// value of the OperandValToReplace of the given IVStrideUse.
const SCEV *IVUsers::getReplacementExpr(const IVStrideUse &U) const {
// Start with zero.
- const SCEV *RetVal = SE->getIntegerSCEV(0, U.getParent()->Stride->getType());
+ const SCEV *RetVal = SE->getIntegerSCEV(0, U.getStride()->getType());
// Create the basic add recurrence.
- RetVal = SE->getAddRecExpr(RetVal, U.getParent()->Stride, L);
+ RetVal = SE->getAddRecExpr(RetVal, U.getStride(), L);
// Add the offset in a separate step, because it may be loop-variant.
RetVal = SE->getAddExpr(RetVal, U.getOffset());
// For uses of post-incremented values, add an extra stride to compute
// the actual replacement value.
if (U.isUseOfPostIncrementedValue())
- RetVal = SE->getAddExpr(RetVal, U.getParent()->Stride);
- // Evaluate the expression out of the loop, if possible.
- if (!L->contains(U.getUser()->getParent())) {
- const SCEV *ExitVal = SE->getSCEVAtScope(RetVal, L->getParentLoop());
- if (ExitVal->isLoopInvariant(L))
- RetVal = ExitVal;
- }
+ RetVal = SE->getAddExpr(RetVal, U.getStride());
+ return RetVal;
+}
+
+/// getCanonicalExpr - Return a SCEV expression which computes the
+/// value of the SCEV of the given IVStrideUse, ignoring the
+/// isUseOfPostIncrementedValue flag.
+const SCEV *IVUsers::getCanonicalExpr(const IVStrideUse &U) const {
+ // Start with zero.
+ const SCEV *RetVal = SE->getIntegerSCEV(0, U.getStride()->getType());
+ // Create the basic add recurrence.
+ RetVal = SE->getAddRecExpr(RetVal, U.getStride(), L);
+ // Add the offset in a separate step, because it may be loop-variant.
+ RetVal = SE->getAddExpr(RetVal, U.getOffset());
return RetVal;
}
}
OS << ":\n";
- for (unsigned Stride = 0, e = StrideOrder.size(); Stride != e; ++Stride) {
- std::map<const SCEV *, IVUsersOfOneStride*>::const_iterator SI =
- IVUsesByStride.find(StrideOrder[Stride]);
- assert(SI != IVUsesByStride.end() && "Stride doesn't exist!");
- OS << " Stride " << *SI->first->getType() << " " << *SI->first << ":\n";
-
- for (ilist<IVStrideUse>::const_iterator UI = SI->second->Users.begin(),
- E = SI->second->Users.end(); UI != E; ++UI) {
- OS << " ";
- WriteAsOperand(OS, UI->getOperandValToReplace(), false);
- OS << " = ";
- OS << *getReplacementExpr(*UI);
- if (UI->isUseOfPostIncrementedValue())
- OS << " (post-inc)";
- OS << " in ";
- UI->getUser()->print(OS);
- }
+ // Use a defualt AssemblyAnnotationWriter to suppress the default info
+ // comments, which aren't relevant here.
+ AssemblyAnnotationWriter Annotator;
+ for (ilist<IVStrideUse>::const_iterator UI = IVUses.begin(),
+ E = IVUses.end(); UI != E; ++UI) {
+ OS << " ";
+ WriteAsOperand(OS, UI->getOperandValToReplace(), false);
+ OS << " = "
+ << *getReplacementExpr(*UI);
+ if (UI->isUseOfPostIncrementedValue())
+ OS << " (post-inc)";
+ OS << " in ";
+ UI->getUser()->print(OS, &Annotator);
+ OS << '\n';
}
}
-void IVUsers::print(std::ostream &o, const Module *M) const {
- raw_os_ostream OS(o);
- print(OS, M);
-}
-
void IVUsers::dump() const {
- print(errs());
+ print(dbgs());
}
void IVUsers::releaseMemory() {
- IVUsesByStride.clear();
- StrideOrder.clear();
Processed.clear();
+ IVUses.clear();
}
void IVStrideUse::deleted() {
// Remove this user from the list.
- Parent->Users.erase(this);
+ Parent->IVUses.erase(this);
// this now dangles!
}
projects / oota-llvm.git / blobdiff