#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/ConstantFolding.h"
-#include "llvm/Analysis/Dominators.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/Assembly/Writer.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Dominators.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Instructions.h"
INITIALIZE_PASS_BEGIN(ScalarEvolution, "scalar-evolution",
"Scalar Evolution Analysis", false, true)
INITIALIZE_PASS_DEPENDENCY(LoopInfo)
-INITIALIZE_PASS_DEPENDENCY(DominatorTree)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfo)
INITIALIZE_PASS_END(ScalarEvolution, "scalar-evolution",
"Scalar Evolution Analysis", false, true)
#endif
void SCEV::print(raw_ostream &OS) const {
- switch (getSCEVType()) {
+ switch (static_cast<SCEVTypes>(getSCEVType())) {
case scConstant:
- WriteAsOperand(OS, cast<SCEVConstant>(this)->getValue(), false);
+ cast<SCEVConstant>(this)->getValue()->printAsOperand(OS, false);
return;
case scTruncate: {
const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this);
if (AR->getNoWrapFlags(FlagNW) &&
!AR->getNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW)))
OS << "nw><";
- WriteAsOperand(OS, AR->getLoop()->getHeader(), /*PrintType=*/false);
+ AR->getLoop()->getHeader()->printAsOperand(OS, /*PrintType=*/false);
OS << ">";
return;
}
Constant *FieldNo;
if (U->isOffsetOf(CTy, FieldNo)) {
OS << "offsetof(" << *CTy << ", ";
- WriteAsOperand(OS, FieldNo, false);
+ FieldNo->printAsOperand(OS, false);
OS << ")";
return;
}
// Otherwise just print it normally.
- WriteAsOperand(OS, U->getValue(), false);
+ U->getValue()->printAsOperand(OS, false);
return;
}
case scCouldNotCompute:
OS << "***COULDNOTCOMPUTE***";
return;
- default: break;
}
llvm_unreachable("Unknown SCEV kind!");
}
Type *SCEV::getType() const {
- switch (getSCEVType()) {
+ switch (static_cast<SCEVTypes>(getSCEVType())) {
case scConstant:
return cast<SCEVConstant>(this)->getType();
case scTruncate:
return cast<SCEVUnknown>(this)->getType();
case scCouldNotCompute:
llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
- default:
- llvm_unreachable("Unknown SCEV kind!");
}
+ llvm_unreachable("Unknown SCEV kind!");
}
bool SCEV::isZero() const {
return S;
}
-const SCEV *ScalarEvolution::getConstant(const APInt& Val) {
+const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
return getConstant(ConstantInt::get(getContext(), Val));
}
// Aside from the getSCEVType() ordering, the particular ordering
// isn't very important except that it's beneficial to be consistent,
// so that (a + b) and (b + a) don't end up as different expressions.
- switch (LType) {
+ switch (static_cast<SCEVTypes>(LType)) {
case scUnknown: {
const SCEVUnknown *LU = cast<SCEVUnknown>(LHS);
const SCEVUnknown *RU = cast<SCEVUnknown>(RHS);
return compare(LC->getOperand(), RC->getOperand());
}
- default:
- llvm_unreachable("Unknown SCEV kind!");
+ case scCouldNotCompute:
+ llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
}
+ llvm_unreachable("Unknown SCEV kind!");
}
};
}
return S;
}
+static const APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) {
+ APInt A = C1->getValue()->getValue().abs();
+ APInt B = C2->getValue()->getValue().abs();
+ uint32_t ABW = A.getBitWidth();
+ uint32_t BBW = B.getBitWidth();
+
+ if (ABW > BBW)
+ B = B.zext(ABW);
+ else if (ABW < BBW)
+ A = A.zext(BBW);
+
+ return APIntOps::GreatestCommonDivisor(A, B);
+}
+
+/// getUDivExactExpr - Get a canonical unsigned division expression, or
+/// something simpler if possible. There is no representation for an exact udiv
+/// in SCEV IR, but we can attempt to remove factors from the LHS and RHS.
+/// We can't do this when it's not exact because the udiv may be clearing bits.
+const SCEV *ScalarEvolution::getUDivExactExpr(const SCEV *LHS,
+ const SCEV *RHS) {
+ // TODO: we could try to find factors in all sorts of things, but for now we
+ // just deal with u/exact (multiply, constant). See SCEVDivision towards the
+ // end of this file for inspiration.
+
+ const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS);
+ if (!Mul)
+ return getUDivExpr(LHS, RHS);
+
+ if (const SCEVConstant *RHSCst = dyn_cast<SCEVConstant>(RHS)) {
+ // If the mulexpr multiplies by a constant, then that constant must be the
+ // first element of the mulexpr.
+ if (const SCEVConstant *LHSCst =
+ dyn_cast<SCEVConstant>(Mul->getOperand(0))) {
+ if (LHSCst == RHSCst) {
+ SmallVector<const SCEV *, 2> Operands;
+ Operands.append(Mul->op_begin() + 1, Mul->op_end());
+ return getMulExpr(Operands);
+ }
+
+ // We can't just assume that LHSCst divides RHSCst cleanly, it could be
+ // that there's a factor provided by one of the other terms. We need to
+ // check.
+ APInt Factor = gcd(LHSCst, RHSCst);
+ if (!Factor.isIntN(1)) {
+ LHSCst = cast<SCEVConstant>(
+ getConstant(LHSCst->getValue()->getValue().udiv(Factor)));
+ RHSCst = cast<SCEVConstant>(
+ getConstant(RHSCst->getValue()->getValue().udiv(Factor)));
+ SmallVector<const SCEV *, 2> Operands;
+ Operands.push_back(LHSCst);
+ Operands.append(Mul->op_begin() + 1, Mul->op_end());
+ LHS = getMulExpr(Operands);
+ RHS = RHSCst;
+ Mul = dyn_cast<SCEVMulExpr>(LHS);
+ if (!Mul)
+ return getUDivExactExpr(LHS, RHS);
+ }
+ }
+ }
+
+ for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
+ if (Mul->getOperand(i) == RHS) {
+ SmallVector<const SCEV *, 2> Operands;
+ Operands.append(Mul->op_begin(), Mul->op_begin() + i);
+ Operands.append(Mul->op_begin() + i + 1, Mul->op_end());
+ return getMulExpr(Operands);
+ }
+ }
+
+ return getUDivExpr(LHS, RHS);
+}
/// getAddRecExpr - Get an add recurrence expression for the specified loop.
/// Simplify the expression as much as possible.
// If we have DataLayout, we can bypass creating a target-independent
// constant expression and then folding it back into a ConstantInt.
// This is just a compile-time optimization.
- if (TD)
- return getConstant(IntTy, TD->getTypeAllocSize(AllocTy));
+ if (DL)
+ return getConstant(IntTy, DL->getTypeAllocSize(AllocTy));
Constant *C = ConstantExpr::getSizeOf(AllocTy);
if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
- if (Constant *Folded = ConstantFoldConstantExpression(CE, TD, TLI))
+ if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI))
C = Folded;
Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy));
assert(Ty == IntTy && "Effective SCEV type doesn't match");
// If we have DataLayout, we can bypass creating a target-independent
// constant expression and then folding it back into a ConstantInt.
// This is just a compile-time optimization.
- if (TD) {
+ if (DL) {
return getConstant(IntTy,
- TD->getStructLayout(STy)->getElementOffset(FieldNo));
+ DL->getStructLayout(STy)->getElementOffset(FieldNo));
}
Constant *C = ConstantExpr::getOffsetOf(STy, FieldNo);
if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
- if (Constant *Folded = ConstantFoldConstantExpression(CE, TD, TLI))
+ if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI))
C = Folded;
Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(STy));
assert(isSCEVable(Ty) && "Type is not SCEVable!");
// If we have a DataLayout, use it!
- if (TD)
- return TD->getTypeSizeInBits(Ty);
+ if (DL)
+ return DL->getTypeSizeInBits(Ty);
// Integer types have fixed sizes.
if (Ty->isIntegerTy())
// The only other support type is pointer.
assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
- if (TD)
- return TD->getIntPtrType(Ty);
+ if (DL)
+ return DL->getIntPtrType(Ty);
// Without DataLayout, conservatively assume pointers are 64-bit.
return Type::getInt64Ty(getContext());
bool FindOne;
FindInvalidSCEVUnknown() { FindOne = false; }
bool follow(const SCEV *S) {
- switch (S->getSCEVType()) {
+ switch (static_cast<SCEVTypes>(S->getSCEVType())) {
case scConstant:
return false;
case scUnknown:
// PHI's incoming blocks are in a different loop, in which case doing so
// risks breaking LCSSA form. Instcombine would normally zap these, but
// it doesn't have DominatorTree information, so it may miss cases.
- if (Value *V = SimplifyInstruction(PN, TD, TLI, DT))
+ if (Value *V = SimplifyInstruction(PN, DL, TLI, DT))
if (LI->replacementPreservesLCSSAForm(PN, V))
return getSCEV(V);
if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
// For a SCEVUnknown, ask ValueTracking.
APInt Zeros(BitWidth, 0), Ones(BitWidth, 0);
- ComputeMaskedBits(U->getValue(), Zeros, Ones, TD);
+ ComputeMaskedBits(U->getValue(), Zeros, Ones, DL);
if (Ones == ~Zeros + 1)
return setUnsignedRange(U, ConservativeResult);
return setUnsignedRange(U,
if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
// For a SCEVUnknown, ask ValueTracking.
- if (!U->getValue()->getType()->isIntegerTy() && !TD)
+ if (!U->getValue()->getType()->isIntegerTy() && !DL)
return setSignedRange(U, ConservativeResult);
- unsigned NS = ComputeNumSignBits(U->getValue(), TD);
+ unsigned NS = ComputeNumSignBits(U->getValue(), DL);
if (NS <= 1)
return setSignedRange(U, ConservativeResult);
return setSignedRange(U, ConservativeResult.intersectWith(
// Use ComputeMaskedBits to compute what ShrinkDemandedConstant
// knew about to reconstruct a low-bits mask value.
unsigned LZ = A.countLeadingZeros();
+ unsigned TZ = A.countTrailingZeros();
unsigned BitWidth = A.getBitWidth();
APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
- ComputeMaskedBits(U->getOperand(0), KnownZero, KnownOne, TD);
-
- APInt EffectiveMask = APInt::getLowBitsSet(BitWidth, BitWidth - LZ);
-
- if (LZ != 0 && !((~A & ~KnownZero) & EffectiveMask))
- return
- getZeroExtendExpr(getTruncateExpr(getSCEV(U->getOperand(0)),
- IntegerType::get(getContext(), BitWidth - LZ)),
- U->getType());
+ ComputeMaskedBits(U->getOperand(0), KnownZero, KnownOne, DL);
+
+ APInt EffectiveMask =
+ APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ);
+ if ((LZ != 0 || TZ != 0) && !((~A & ~KnownZero) & EffectiveMask)) {
+ const SCEV *MulCount = getConstant(
+ ConstantInt::get(getContext(), APInt::getOneBitSet(BitWidth, TZ)));
+ return getMulExpr(
+ getZeroExtendExpr(
+ getTruncateExpr(
+ getUDivExactExpr(getSCEV(U->getOperand(0)), MulCount),
+ IntegerType::get(getContext(), BitWidth - LZ - TZ)),
+ U->getType()),
+ MulCount);
+ }
}
break;
// Examine all exits and pick the most conservative values.
const SCEV *MaxBECount = getCouldNotCompute();
bool CouldComputeBECount = true;
+ BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
+ const SCEV *LatchMaxCount = 0;
SmallVector<std::pair<BasicBlock *, const SCEV *>, 4> ExitCounts;
for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
ExitLimit EL = ComputeExitLimit(L, ExitingBlocks[i]);
// We cannot take the "min" MaxBECount, because non-unit stride loops may
// skip some loop tests. Taking the max over the exits is sufficiently
// conservative. TODO: We could do better taking into consideration
- // that (1) the loop has unit stride (2) the last loop test is
- // less-than/greater-than (3) any loop test is less-than/greater-than AND
- // falls-through some constant times less then the other tests.
- MaxBECount = getUMaxFromMismatchedTypes(MaxBECount, EL.Max);
+ // non-latch exits that dominate the latch.
+ if (EL.MustExit && ExitingBlocks[i] == Latch)
+ LatchMaxCount = EL.Max;
+ else
+ MaxBECount = getUMaxFromMismatchedTypes(MaxBECount, EL.Max);
}
}
-
+ // Be more precise in the easy case of a loop latch that must exit.
+ if (LatchMaxCount) {
+ MaxBECount = getUMinFromMismatchedTypes(MaxBECount, LatchMaxCount);
+ }
return BackedgeTakenInfo(ExitCounts, CouldComputeBECount, MaxBECount);
}
ScalarEvolution::ComputeExitLimit(const Loop *L, BasicBlock *ExitingBlock) {
// Okay, we've chosen an exiting block. See what condition causes us to
- // exit at this block.
- //
- // FIXME: we should be able to handle switch instructions (with a single exit)
- BranchInst *ExitBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator());
- if (ExitBr == 0) return getCouldNotCompute();
- assert(ExitBr->isConditional() && "If unconditional, it can't be in loop!");
+ // exit at this block and remember the exit block and whether all other targets
+ // lead to the loop header.
+ bool MustExecuteLoopHeader = true;
+ BasicBlock *Exit = 0;
+ for (succ_iterator SI = succ_begin(ExitingBlock), SE = succ_end(ExitingBlock);
+ SI != SE; ++SI)
+ if (!L->contains(*SI)) {
+ if (Exit) // Multiple exit successors.
+ return getCouldNotCompute();
+ Exit = *SI;
+ } else if (*SI != L->getHeader()) {
+ MustExecuteLoopHeader = false;
+ }
// At this point, we know we have a conditional branch that determines whether
// the loop is exited. However, we don't know if the branch is executed each
//
// More extensive analysis could be done to handle more cases here.
//
- if (ExitBr->getSuccessor(0) != L->getHeader() &&
- ExitBr->getSuccessor(1) != L->getHeader() &&
- ExitBr->getParent() != L->getHeader()) {
+ if (!MustExecuteLoopHeader && ExitingBlock != L->getHeader()) {
// The simple checks failed, try climbing the unique predecessor chain
// up to the header.
bool Ok = false;
- for (BasicBlock *BB = ExitBr->getParent(); BB; ) {
+ for (BasicBlock *BB = ExitingBlock; BB; ) {
BasicBlock *Pred = BB->getUniquePredecessor();
if (!Pred)
return getCouldNotCompute();
return getCouldNotCompute();
}
- // Proceed to the next level to examine the exit condition expression.
- return ComputeExitLimitFromCond(L, ExitBr->getCondition(),
- ExitBr->getSuccessor(0),
- ExitBr->getSuccessor(1),
- /*IsSubExpr=*/false);
+ TerminatorInst *Term = ExitingBlock->getTerminator();
+ if (BranchInst *BI = dyn_cast<BranchInst>(Term)) {
+ assert(BI->isConditional() && "If unconditional, it can't be in loop!");
+ // Proceed to the next level to examine the exit condition expression.
+ return ComputeExitLimitFromCond(L, BI->getCondition(), BI->getSuccessor(0),
+ BI->getSuccessor(1),
+ /*IsSubExpr=*/false);
+ }
+
+ if (SwitchInst *SI = dyn_cast<SwitchInst>(Term))
+ return ComputeExitLimitFromSingleExitSwitch(L, SI, Exit,
+ /*IsSubExpr=*/false);
+
+ return getCouldNotCompute();
}
/// ComputeExitLimitFromCond - Compute the number of times the
IsSubExpr || EitherMayExit);
const SCEV *BECount = getCouldNotCompute();
const SCEV *MaxBECount = getCouldNotCompute();
+ bool MustExit = false;
if (EitherMayExit) {
// Both conditions must be true for the loop to continue executing.
// Choose the less conservative count.
MaxBECount = EL0.Max;
else
MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max);
+ MustExit = EL0.MustExit || EL1.MustExit;
} else {
// Both conditions must be true at the same time for the loop to exit.
// For now, be conservative.
MaxBECount = EL0.Max;
if (EL0.Exact == EL1.Exact)
BECount = EL0.Exact;
+ MustExit = EL0.MustExit && EL1.MustExit;
}
- return ExitLimit(BECount, MaxBECount);
+ return ExitLimit(BECount, MaxBECount, MustExit);
}
if (BO->getOpcode() == Instruction::Or) {
// Recurse on the operands of the or.
IsSubExpr || EitherMayExit);
const SCEV *BECount = getCouldNotCompute();
const SCEV *MaxBECount = getCouldNotCompute();
+ bool MustExit = false;
if (EitherMayExit) {
// Both conditions must be false for the loop to continue executing.
// Choose the less conservative count.
MaxBECount = EL0.Max;
else
MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max);
+ MustExit = EL0.MustExit || EL1.MustExit;
} else {
// Both conditions must be false at the same time for the loop to exit.
// For now, be conservative.
MaxBECount = EL0.Max;
if (EL0.Exact == EL1.Exact)
BECount = EL0.Exact;
+ MustExit = EL0.MustExit && EL1.MustExit;
}
- return ExitLimit(BECount, MaxBECount);
+ return ExitLimit(BECount, MaxBECount, MustExit);
}
}
return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB));
}
+ScalarEvolution::ExitLimit
+ScalarEvolution::ComputeExitLimitFromSingleExitSwitch(const Loop *L,
+ SwitchInst *Switch,
+ BasicBlock *ExitingBlock,
+ bool IsSubExpr) {
+ assert(!L->contains(ExitingBlock) && "Not an exiting block!");
+
+ // Give up if the exit is the default dest of a switch.
+ if (Switch->getDefaultDest() == ExitingBlock)
+ return getCouldNotCompute();
+
+ assert(L->contains(Switch->getDefaultDest()) &&
+ "Default case must not exit the loop!");
+ const SCEV *LHS = getSCEVAtScope(Switch->getCondition(), L);
+ const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock));
+
+ // while (X != Y) --> while (X-Y != 0)
+ ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, IsSubExpr);
+ if (EL.hasAnyInfo())
+ return EL;
+
+ return getCouldNotCompute();
+}
+
static ConstantInt *
EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
ScalarEvolution &SE) {
/// reason, return null.
static Constant *EvaluateExpression(Value *V, const Loop *L,
DenseMap<Instruction *, Constant *> &Vals,
- const DataLayout *TD,
+ const DataLayout *DL,
const TargetLibraryInfo *TLI) {
// Convenient constant check, but redundant for recursive calls.
if (Constant *C = dyn_cast<Constant>(V)) return C;
if (!Operands[i]) return 0;
continue;
}
- Constant *C = EvaluateExpression(Operand, L, Vals, TD, TLI);
+ Constant *C = EvaluateExpression(Operand, L, Vals, DL, TLI);
Vals[Operand] = C;
if (!C) return 0;
Operands[i] = C;
if (CmpInst *CI = dyn_cast<CmpInst>(I))
return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0],
- Operands[1], TD, TLI);
+ Operands[1], DL, TLI);
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
if (!LI->isVolatile())
- return ConstantFoldLoadFromConstPtr(Operands[0], TD);
+ return ConstantFoldLoadFromConstPtr(Operands[0], DL);
}
- return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Operands, TD,
+ return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Operands, DL,
TLI);
}
// Compute the value of the PHIs for the next iteration.
// EvaluateExpression adds non-phi values to the CurrentIterVals map.
DenseMap<Instruction *, Constant *> NextIterVals;
- Constant *NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, TD,
+ Constant *NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL,
TLI);
if (NextPHI == 0)
return 0; // Couldn't evaluate!
Constant *&NextPHI = NextIterVals[PHI];
if (!NextPHI) { // Not already computed.
Value *BEValue = PHI->getIncomingValue(SecondIsBackedge);
- NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, TD, TLI);
+ NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI);
}
if (NextPHI != I->second)
StoppedEvolving = false;
for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
ConstantInt *CondVal =
dyn_cast_or_null<ConstantInt>(EvaluateExpression(Cond, L, CurrentIterVals,
- TD, TLI));
+ DL, TLI));
// Couldn't symbolically evaluate.
if (!CondVal) return getCouldNotCompute();
if (NextPHI) continue; // Already computed!
Value *BEValue = PHI->getIncomingValue(SecondIsBackedge);
- NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, TD, TLI);
+ NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI);
}
CurrentIterVals.swap(NextIterVals);
}
/// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
/// Returns NULL if the SCEV isn't representable as a Constant.
static Constant *BuildConstantFromSCEV(const SCEV *V) {
- switch (V->getSCEVType()) {
- default: // TODO: smax, umax.
+ switch (static_cast<SCEVTypes>(V->getSCEVType())) {
case scCouldNotCompute:
case scAddRecExpr:
break;
return ConstantExpr::getUDiv(LHS, RHS);
break;
}
+ case scSMaxExpr:
+ case scUMaxExpr:
+ break; // TODO: smax, umax.
}
return 0;
}
Constant *C = 0;
if (const CmpInst *CI = dyn_cast<CmpInst>(I))
C = ConstantFoldCompareInstOperands(CI->getPredicate(),
- Operands[0], Operands[1], TD,
+ Operands[0], Operands[1], DL,
TLI);
else if (const LoadInst *LI = dyn_cast<LoadInst>(I)) {
if (!LI->isVolatile())
- C = ConstantFoldLoadFromConstPtr(Operands[0], TD);
+ C = ConstantFoldLoadFromConstPtr(Operands[0], DL);
} else
C = ConstantFoldInstOperands(I->getOpcode(), I->getType(),
- Operands, TD, TLI);
+ Operands, DL, TLI);
if (!C) return V;
return getSCEV(C);
}
else
MaxBECount = getConstant(CountDown ? CR.getUnsignedMax()
: -CR.getUnsignedMin());
- return ExitLimit(Distance, MaxBECount);
+ return ExitLimit(Distance, MaxBECount, /*MustExit=*/true);
}
// If the recurrence is known not to wraparound, unsigned divide computes the
// that the value will either become zero (and thus the loop terminates), that
// the loop will terminate through some other exit condition first, or that
// the loop has undefined behavior. This means we can't "miss" the exit
- // value, even with nonunit stride.
+ // value, even with nonunit stride, and exit later via the same branch. Note
+ // that we can skip this exit if loop later exits via a different
+ // branch. Hence MustExit=false.
//
// This is only valid for expressions that directly compute the loop exit. It
// is invalid for subexpressions in which the loop may exit through this
// branch even if this subexpression is false. In that case, the trip count
// computed by this udiv could be smaller than the number of well-defined
// iterations.
- if (!IsSubExpr && AddRec->getNoWrapFlags(SCEV::FlagNW))
- return getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step);
-
+ if (!IsSubExpr && AddRec->getNoWrapFlags(SCEV::FlagNW)) {
+ const SCEV *Exact =
+ getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step);
+ return ExitLimit(Exact, Exact, /*MustExit=*/false);
+ }
// Then, try to solve the above equation provided that Start is constant.
if (const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start))
return SolveLinEquationWithOverflow(StepC->getValue()->getValue(),
// LHS' type is checked for above.
if (getTypeSizeInBits(LHS->getType()) >
getTypeSizeInBits(FoundLHS->getType())) {
- if (CmpInst::isSigned(Pred)) {
+ if (CmpInst::isSigned(FoundPred)) {
FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType());
FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType());
} else {
if (isa<SCEVCouldNotCompute>(MaxBECount))
MaxBECount = BECount;
- return ExitLimit(BECount, MaxBECount);
+ return ExitLimit(BECount, MaxBECount, /*MustExit=*/true);
}
ScalarEvolution::ExitLimit
if (isa<SCEVCouldNotCompute>(MaxBECount))
MaxBECount = BECount;
- return ExitLimit(BECount, MaxBECount);
+ return ExitLimit(BECount, MaxBECount, /*MustExit=*/true);
}
/// getNumIterationsInRange - Return the number of iterations of this loop that
return SE.getCouldNotCompute();
}
-static const APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) {
- APInt A = C1->getValue()->getValue().abs();
- APInt B = C2->getValue()->getValue().abs();
- uint32_t ABW = A.getBitWidth();
- uint32_t BBW = B.getBitWidth();
-
- if (ABW > BBW)
- B.zext(ABW);
- else if (ABW < BBW)
- A.zext(BBW);
-
- return APIntOps::GreatestCommonDivisor(A, B);
-}
-
static const APInt srem(const SCEVConstant *C1, const SCEVConstant *C2) {
APInt A = C1->getValue()->getValue();
APInt B = C2->getValue()->getValue();
uint32_t BBW = B.getBitWidth();
if (ABW > BBW)
- B.sext(ABW);
+ B = B.sext(ABW);
else if (ABW < BBW)
- A.sext(BBW);
+ A = A.sext(BBW);
return APIntOps::srem(A, B);
}
uint32_t BBW = B.getBitWidth();
if (ABW > BBW)
- B.sext(ABW);
+ B = B.sext(ABW);
else if (ABW < BBW)
- A.sext(BBW);
+ A = A.sext(BBW);
return APIntOps::sdiv(A, B);
}
/// Splits the SCEV into two vectors of SCEVs representing the subscripts and
/// sizes of an array access. Returns the remainder of the delinearization that
-/// is the offset start of the array. For example
-/// delinearize ({(((-4 + (3 * %m)))),+,(%m)}<%for.i>) {
-/// IV: {0,+,1}<%for.i>
-/// Start: -4 + (3 * %m)
-/// Step: %m
-/// SCEVUDiv (Start, Step) = 3 remainder -4
-/// rem = delinearize (3) = 3
-/// Subscripts.push_back(IV + rem)
-/// Sizes.push_back(Step)
-/// return remainder -4
-/// }
-/// When delinearize fails, it returns the SCEV unchanged.
+/// is the offset start of the array. The SCEV->delinearize algorithm computes
+/// the multiples of SCEV coefficients: that is a pattern matching of sub
+/// expressions in the stride and base of a SCEV corresponding to the
+/// computation of a GCD (greatest common divisor) of base and stride. When
+/// SCEV->delinearize fails, it returns the SCEV unchanged.
+///
+/// For example: when analyzing the memory access A[i][j][k] in this loop nest
+///
+/// void foo(long n, long m, long o, double A[n][m][o]) {
+///
+/// for (long i = 0; i < n; i++)
+/// for (long j = 0; j < m; j++)
+/// for (long k = 0; k < o; k++)
+/// A[i][j][k] = 1.0;
+/// }
+///
+/// the delinearization input is the following AddRec SCEV:
+///
+/// AddRec: {{{%A,+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k>
+///
+/// From this SCEV, we are able to say that the base offset of the access is %A
+/// because it appears as an offset that does not divide any of the strides in
+/// the loops:
+///
+/// CHECK: Base offset: %A
+///
+/// and then SCEV->delinearize determines the size of some of the dimensions of
+/// the array as these are the multiples by which the strides are happening:
+///
+/// CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of sizeof(double) bytes.
+///
+/// Note that the outermost dimension remains of UnknownSize because there are
+/// no strides that would help identifying the size of the last dimension: when
+/// the array has been statically allocated, one could compute the size of that
+/// dimension by dividing the overall size of the array by the size of the known
+/// dimensions: %m * %o * 8.
+///
+/// Finally delinearize provides the access functions for the array reference
+/// that does correspond to A[i][j][k] of the above C testcase:
+///
+/// CHECK: ArrayRef[{0,+,1}<%for.i>][{0,+,1}<%for.j>][{0,+,1}<%for.k>]
+///
+/// The testcases are checking the output of a function pass:
+/// DelinearizationPass that walks through all loads and stores of a function
+/// asking for the SCEV of the memory access with respect to all enclosing
+/// loops, calling SCEV->delinearize on that and printing the results.
+
const SCEV *
SCEVAddRecExpr::delinearize(ScalarEvolution &SE,
SmallVectorImpl<const SCEV *> &Subscripts,
SmallVectorImpl<const SCEV *> &Sizes) const {
+ // Early exit in case this SCEV is not an affine multivariate function.
if (!this->isAffine())
return this;
const SCEV *Start = this->getStart();
const SCEV *Step = this->getStepRecurrence(SE);
+
+ // Build the SCEV representation of the canonical induction variable in the
+ // loop of this SCEV.
const SCEV *Zero = SE.getConstant(this->getType(), 0);
const SCEV *One = SE.getConstant(this->getType(), 1);
const SCEV *IV =
DEBUG(dbgs() << "(delinearize: " << *this << "\n");
- if (Step == One) {
- DEBUG(dbgs() << "failed to delinearize " << *this << "\n)\n");
- return this;
- }
+ // When the stride of this SCEV is 1, do not compute the GCD: the size of this
+ // subscript is 1, and this same SCEV for the access function.
+ const SCEV *Remainder = Zero;
+ const SCEV *GCD = One;
- const SCEV *Remainder = NULL;
- const SCEV *GCD = SCEVGCD::findGCD(SE, Start, Step, &Remainder);
+ // Find the GCD and Remainder of the Start and Step coefficients of this SCEV.
+ if (Step != One && !Step->isAllOnesValue())
+ GCD = SCEVGCD::findGCD(SE, Start, Step, &Remainder);
DEBUG(dbgs() << "GCD: " << *GCD << "\n");
DEBUG(dbgs() << "Remainder: " << *Remainder << "\n");
- if (GCD == One) {
- DEBUG(dbgs() << "failed to delinearize " << *this << "\n)\n");
- return this;
- }
+ const SCEV *Quotient = Start;
+ if (GCD != One && !GCD->isAllOnesValue())
+ // As findGCD computed Remainder, GCD divides "Start - Remainder." The
+ // Quotient is then this SCEV without Remainder, scaled down by the GCD. The
+ // Quotient is what will be used in the next subscript delinearization.
+ Quotient = SCEVDivision::divide(SE, SE.getMinusSCEV(Start, Remainder), GCD);
- const SCEV *Quotient =
- SCEVDivision::divide(SE, SE.getMinusSCEV(Start, Remainder), GCD);
DEBUG(dbgs() << "Quotient: " << *Quotient << "\n");
- const SCEV *Rem;
+ const SCEV *Rem = Quotient;
if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Quotient))
+ // Recursively call delinearize on the Quotient until there are no more
+ // multiples that can be recognized.
Rem = AR->delinearize(SE, Subscripts, Sizes);
- else
- Rem = Quotient;
- if (Step != GCD) {
+ // Scale up the canonical induction variable IV by whatever remains from the
+ // Step after division by the GCD: the GCD is the size of all the sub-array.
+ if (Step != One && !Step->isAllOnesValue() && GCD != One &&
+ !GCD->isAllOnesValue() && Step != GCD) {
Step = SCEVDivision::divide(SE, Step, GCD);
IV = SE.getMulExpr(IV, Step);
}
+ // The access function in the current subscript is computed as the canonical
+ // induction variable IV (potentially scaled up by the step) and offset by
+ // Rem, the offset of delinearization in the sub-array.
const SCEV *Index = SE.getAddExpr(IV, Rem);
+ // Record the access function and the size of the current subscript.
Subscripts.push_back(Index);
Sizes.push_back(GCD);
bool ScalarEvolution::runOnFunction(Function &F) {
this->F = &F;
LI = &getAnalysis<LoopInfo>();
- TD = getAnalysisIfAvailable<DataLayout>();
+ DataLayoutPass *DLP = getAnalysisIfAvailable<DataLayoutPass>();
+ DL = DLP ? &DLP->getDataLayout() : 0;
TLI = &getAnalysis<TargetLibraryInfo>();
- DT = &getAnalysis<DominatorTree>();
+ DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
return false;
}
void ScalarEvolution::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
AU.addRequiredTransitive<LoopInfo>();
- AU.addRequiredTransitive<DominatorTree>();
+ AU.addRequiredTransitive<DominatorTreeWrapperPass>();
AU.addRequired<TargetLibraryInfo>();
}
PrintLoopInfo(OS, SE, *I);
OS << "Loop ";
- WriteAsOperand(OS, L->getHeader(), /*PrintType=*/false);
+ L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
OS << ": ";
SmallVector<BasicBlock *, 8> ExitBlocks;
OS << "\n"
"Loop ";
- WriteAsOperand(OS, L->getHeader(), /*PrintType=*/false);
+ L->getHeader()->printAsOperand(OS, /*PrintType=*/false);
OS << ": ";
if (!isa<SCEVCouldNotCompute>(SE->getMaxBackedgeTakenCount(L))) {
ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
OS << "Classifying expressions for: ";
- WriteAsOperand(OS, F, /*PrintType=*/false);
+ F->printAsOperand(OS, /*PrintType=*/false);
OS << "\n";
for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I)
if (isSCEVable(I->getType()) && !isa<CmpInst>(*I)) {
}
OS << "Determining loop execution counts for: ";
- WriteAsOperand(OS, F, /*PrintType=*/false);
+ F->printAsOperand(OS, /*PrintType=*/false);
OS << "\n";
for (LoopInfo::iterator I = LI->begin(), E = LI->end(); I != E; ++I)
PrintLoopInfo(OS, &SE, *I);
ScalarEvolution::LoopDisposition
ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
- switch (S->getSCEVType()) {
+ switch (static_cast<SCEVTypes>(S->getSCEVType())) {
case scConstant:
return LoopInvariant;
case scTruncate:
return LoopInvariant;
case scCouldNotCompute:
llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
- default: llvm_unreachable("Unknown SCEV kind!");
}
+ llvm_unreachable("Unknown SCEV kind!");
}
bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) {
ScalarEvolution::BlockDisposition
ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
- switch (S->getSCEVType()) {
+ switch (static_cast<SCEVTypes>(S->getSCEVType())) {
case scConstant:
return ProperlyDominatesBlock;
case scTruncate:
return ProperlyDominatesBlock;
case scCouldNotCompute:
llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
- default:
- llvm_unreachable("Unknown SCEV kind!");
}
+ llvm_unreachable("Unknown SCEV kind!");
}
bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
typedef DenseMap<const Loop *, std::string> VerifyMap;
-/// replaceSubString - Replaces all occurences of From in Str with To.
+/// replaceSubString - Replaces all occurrences of From in Str with To.
static void replaceSubString(std::string &Str, StringRef From, StringRef To) {
size_t Pos = 0;
while ((Pos = Str.find(From, Pos)) != std::string::npos) {
projects / oota-llvm.git / blobdiff