X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FAnalysis%2FScalarEvolution.cpp;h=a64a7c183305100c6f7c0522630587dc87db0e77;hb=dcfd404e3ccc66844632aa601bf52522dae41512;hp=b734d00f7fe05587d601154ceb97c78fb384b9e3;hpb=714b5290b04e08570dae4304c1c92d30c06d3c99;p=oota-llvm.git diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp index b734d00f7fe..a64a7c18330 100644 --- a/lib/Analysis/ScalarEvolution.cpp +++ b/lib/Analysis/ScalarEvolution.cpp @@ -157,6 +157,13 @@ void SCEV::print(raw_ostream &OS) const { for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i) OS << ",+," << *AR->getOperand(i); OS << "}<"; + if (AR->getNoWrapFlags(FlagNUW)) + OS << "nuw><"; + if (AR->getNoWrapFlags(FlagNSW)) + OS << "nsw><"; + if (AR->getNoWrapFlags(FlagNW) && + !AR->getNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW))) + OS << "nw><"; WriteAsOperand(OS, AR->getLoop()->getHeader(), /*PrintType=*/false); OS << ">"; return; @@ -166,7 +173,7 @@ void SCEV::print(raw_ostream &OS) const { case scUMaxExpr: case scSMaxExpr: { const SCEVNAryExpr *NAry = cast(this); - const char *OpStr; + const char *OpStr = 0; switch (NAry->getSCEVType()) { case scAddExpr: OpStr = " + "; break; case scMulExpr: OpStr = " * "; break; @@ -199,7 +206,7 @@ void SCEV::print(raw_ostream &OS) const { OS << "alignof(" << *AllocTy << ")"; return; } - + const Type *CTy; Constant *FieldNo; if (U->isOffsetOf(CTy, FieldNo)) { @@ -208,7 +215,7 @@ void SCEV::print(raw_ostream &OS) const { OS << ")"; return; } - + // Otherwise just print it normally. WriteAsOperand(OS, U->getValue(), false); return; @@ -325,10 +332,7 @@ SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, void SCEVUnknown::deleted() { // Clear this SCEVUnknown from various maps. - SE->ValuesAtScopes.erase(this); - SE->LoopDispositions.erase(this); - SE->UnsignedRanges.erase(this); - SE->SignedRanges.erase(this); + SE->forgetMemoizedResults(this); // Remove this SCEVUnknown from the uniquing map. SE->UniqueSCEVs.RemoveNode(this); @@ -339,10 +343,7 @@ void SCEVUnknown::deleted() { void SCEVUnknown::allUsesReplacedWith(Value *New) { // Clear this SCEVUnknown from various maps. - SE->ValuesAtScopes.erase(this); - SE->LoopDispositions.erase(this); - SE->UnsignedRanges.erase(this); - SE->SignedRanges.erase(this); + SE->forgetMemoizedResults(this); // Remove this SCEVUnknown from the uniquing map. SE->UniqueSCEVs.RemoveNode(this); @@ -821,12 +822,42 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, if (const SCEVZeroExtendExpr *SZ = dyn_cast(Op)) return getTruncateOrZeroExtend(SZ->getOperand(), Ty); + // trunc(x1+x2+...+xN) --> trunc(x1)+trunc(x2)+...+trunc(xN) if we can + // eliminate all the truncates. + if (const SCEVAddExpr *SA = dyn_cast(Op)) { + SmallVector Operands; + bool hasTrunc = false; + for (unsigned i = 0, e = SA->getNumOperands(); i != e && !hasTrunc; ++i) { + const SCEV *S = getTruncateExpr(SA->getOperand(i), Ty); + hasTrunc = isa(S); + Operands.push_back(S); + } + if (!hasTrunc) + return getAddExpr(Operands); + UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. + } + + // trunc(x1*x2*...*xN) --> trunc(x1)*trunc(x2)*...*trunc(xN) if we can + // eliminate all the truncates. + if (const SCEVMulExpr *SM = dyn_cast(Op)) { + SmallVector Operands; + bool hasTrunc = false; + for (unsigned i = 0, e = SM->getNumOperands(); i != e && !hasTrunc; ++i) { + const SCEV *S = getTruncateExpr(SM->getOperand(i), Ty); + hasTrunc = isa(S); + Operands.push_back(S); + } + if (!hasTrunc) + return getMulExpr(Operands); + UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. + } + // If the input value is a chrec scev, truncate the chrec's operands. if (const SCEVAddRecExpr *AddRec = dyn_cast(Op)) { SmallVector Operands; for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) Operands.push_back(getTruncateExpr(AddRec->getOperand(i), Ty)); - return getAddRecExpr(Operands, AddRec->getLoop()); + return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap); } // As a special case, fold trunc(undef) to undef. We don't want to @@ -872,6 +903,19 @@ const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, void *IP = 0; if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; + // zext(trunc(x)) --> zext(x) or x or trunc(x) + if (const SCEVTruncateExpr *ST = dyn_cast(Op)) { + // It's possible the bits taken off by the truncate were all zero bits. If + // so, we should be able to simplify this further. + const SCEV *X = ST->getOperand(); + ConstantRange CR = getUnsignedRange(X); + unsigned TruncBits = getTypeSizeInBits(ST->getType()); + unsigned NewBits = getTypeSizeInBits(Ty); + if (CR.truncate(TruncBits).zeroExtend(NewBits).contains( + CR.zextOrTrunc(NewBits))) + return getTruncateOrZeroExtend(X, Ty); + } + // If the input value is a chrec scev, and we can prove that the value // did not overflow the old, smaller, value, we can zero extend all of the // operands (often constants). This allows analysis of something like @@ -885,10 +929,11 @@ const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, // If we have special knowledge that this addrec won't overflow, // we don't need to do any further analysis. - if (AR->hasNoUnsignedWrap()) + if (AR->getNoWrapFlags(SCEV::FlagNUW)) return getAddRecExpr(getZeroExtendExpr(Start, Ty), getZeroExtendExpr(Step, Ty), - L); + // FIXME: Can use SCEV::FlagNUW + L, SCEV::FlagAnyWrap); // Check whether the backedge-taken count is SCEVCouldNotCompute. // Note that this serves two purposes: It filters out loops that are @@ -922,7 +967,8 @@ const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, // Return the expression with the addrec on the outside. return getAddRecExpr(getZeroExtendExpr(Start, Ty), getZeroExtendExpr(Step, Ty), - L); + // FIXME: can use FlagNUW + L, SCEV::FlagAnyWrap); // Similar to above, only this time treat the step value as signed. // This covers loops that count down. @@ -936,7 +982,8 @@ const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, // Return the expression with the addrec on the outside. return getAddRecExpr(getZeroExtendExpr(Start, Ty), getSignExtendExpr(Step, Ty), - L); + // FIXME: can use FlagNW + L, SCEV::FlagAnyWrap); } // If the backedge is guarded by a comparison with the pre-inc value @@ -953,7 +1000,8 @@ const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, // Return the expression with the addrec on the outside. return getAddRecExpr(getZeroExtendExpr(Start, Ty), getZeroExtendExpr(Step, Ty), - L); + // FIXME: can use FlagNUW + L, SCEV::FlagAnyWrap); } else if (isKnownNegative(Step)) { const SCEV *N = getConstant(APInt::getMaxValue(BitWidth) - getSignedRange(Step).getSignedMin()); @@ -961,10 +1009,12 @@ const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_UGT, Start, N) && isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR->getPostIncExpr(*this), N))) - // Return the expression with the addrec on the outside. + // Return the expression with the addrec on the outside. The + // negative step causes unsigned wrap, but it still can't self-wrap. return getAddRecExpr(getZeroExtendExpr(Start, Ty), getSignExtendExpr(Step, Ty), - L); + // FIXME: can use FlagNW + L, SCEV::FlagAnyWrap); } } } @@ -996,6 +1046,10 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, if (const SCEVSignExtendExpr *SS = dyn_cast(Op)) return getSignExtendExpr(SS->getOperand(), Ty); + // sext(zext(x)) --> zext(x) + if (const SCEVZeroExtendExpr *SZ = dyn_cast(Op)) + return getZeroExtendExpr(SZ->getOperand(), Ty); + // Before doing any expensive analysis, check to see if we've already // computed a SCEV for this Op and Ty. FoldingSetNodeID ID; @@ -1005,6 +1059,23 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, void *IP = 0; if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; + // If the input value is provably positive, build a zext instead. + if (isKnownNonNegative(Op)) + return getZeroExtendExpr(Op, Ty); + + // sext(trunc(x)) --> sext(x) or x or trunc(x) + if (const SCEVTruncateExpr *ST = dyn_cast(Op)) { + // It's possible the bits taken off by the truncate were all sign bits. If + // so, we should be able to simplify this further. + const SCEV *X = ST->getOperand(); + ConstantRange CR = getSignedRange(X); + unsigned TruncBits = getTypeSizeInBits(ST->getType()); + unsigned NewBits = getTypeSizeInBits(Ty); + if (CR.truncate(TruncBits).signExtend(NewBits).contains( + CR.sextOrTrunc(NewBits))) + return getTruncateOrSignExtend(X, Ty); + } + // If the input value is a chrec scev, and we can prove that the value // did not overflow the old, smaller, value, we can sign extend all of the // operands (often constants). This allows analysis of something like @@ -1018,10 +1089,11 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, // If we have special knowledge that this addrec won't overflow, // we don't need to do any further analysis. - if (AR->hasNoSignedWrap()) + if (AR->getNoWrapFlags(SCEV::FlagNSW)) return getAddRecExpr(getSignExtendExpr(Start, Ty), getSignExtendExpr(Step, Ty), - L); + // FIXME: can use SCEV::FlagNSW + L, SCEV::FlagAnyWrap); // Check whether the backedge-taken count is SCEVCouldNotCompute. // Note that this serves two purposes: It filters out loops that are @@ -1055,7 +1127,8 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, // Return the expression with the addrec on the outside. return getAddRecExpr(getSignExtendExpr(Start, Ty), getSignExtendExpr(Step, Ty), - L); + // FIXME: can use SCEV::FlagNSW + L, SCEV::FlagAnyWrap); // Similar to above, only this time treat the step value as unsigned. // This covers loops that count up with an unsigned step. @@ -1069,7 +1142,8 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, // Return the expression with the addrec on the outside. return getAddRecExpr(getSignExtendExpr(Start, Ty), getZeroExtendExpr(Step, Ty), - L); + // FIXME: can use SCEV::FlagNSW + L, SCEV::FlagAnyWrap); } // If the backedge is guarded by a comparison with the pre-inc value @@ -1086,7 +1160,8 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, // Return the expression with the addrec on the outside. return getAddRecExpr(getSignExtendExpr(Start, Ty), getSignExtendExpr(Step, Ty), - L); + // FIXME: can use SCEV::FlagNSW + L, SCEV::FlagAnyWrap); } else if (isKnownNegative(Step)) { const SCEV *N = getConstant(APInt::getSignedMaxValue(BitWidth) - getSignedRange(Step).getSignedMin()); @@ -1097,7 +1172,8 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, // Return the expression with the addrec on the outside. return getAddRecExpr(getSignExtendExpr(Start, Ty), getSignExtendExpr(Step, Ty), - L); + // FIXME: can use SCEV::FlagNSW + L, SCEV::FlagAnyWrap); } } } @@ -1151,7 +1227,8 @@ const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op, for (SCEVAddRecExpr::op_iterator I = AR->op_begin(), E = AR->op_end(); I != E; ++I) Ops.push_back(getAnyExtendExpr(*I, Ty)); - return getAddRecExpr(Ops, AR->getLoop()); + // FIXME: can use AR->getNoWrapFlags(SCEV::FlagNW) + return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagAnyWrap); } // As a special case, fold anyext(undef) to undef. We don't want to @@ -1272,7 +1349,9 @@ namespace { /// getAddExpr - Get a canonical add expression, or something simpler if /// possible. const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, - bool HasNUW, bool HasNSW) { + SCEV::NoWrapFlags Flags) { + assert(!(Flags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) && + "only nuw or nsw allowed"); assert(!Ops.empty() && "Cannot get empty add!"); if (Ops.size() == 1) return Ops[0]; #ifndef NDEBUG @@ -1282,8 +1361,8 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, "SCEVAddExpr operand types don't match!"); #endif - // If HasNSW is true and all the operands are non-negative, infer HasNUW. - if (!HasNUW && HasNSW) { + // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. + if (!(Flags & SCEV::FlagNUW) && (Flags & SCEV::FlagNSW)) { bool All = true; for (SmallVectorImpl::const_iterator I = Ops.begin(), E = Ops.end(); I != E; ++I) @@ -1291,7 +1370,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, All = false; break; } - if (All) HasNUW = true; + if (All) Flags = setFlags(Flags, SCEV::FlagNUW); } // Sort by complexity, this groups all similar expression types together. @@ -1342,7 +1421,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, FoundMatch = true; } if (FoundMatch) - return getAddExpr(Ops, HasNUW, HasNSW); + return getAddExpr(Ops, Flags); // Check for truncates. If all the operands are truncated from the same // type, see if factoring out the truncate would permit the result to be @@ -1392,7 +1471,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, } if (Ok) { // Evaluate the expression in the larger type. - const SCEV *Fold = getAddExpr(LargeOps, HasNUW, HasNSW); + const SCEV *Fold = getAddExpr(LargeOps, Flags); // If it folds to something simple, use it. Otherwise, don't. if (isa(Fold) || isa(Fold)) return getTruncateExpr(Fold, DstType); @@ -1563,9 +1642,10 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, // Build the new addrec. Propagate the NUW and NSW flags if both the // outer add and the inner addrec are guaranteed to have no overflow. - const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, - HasNUW && AddRec->hasNoUnsignedWrap(), - HasNSW && AddRec->hasNoSignedWrap()); + // FIXME: Always propagate NW + // AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW)) + Flags = AddRec->getNoWrapFlags(Flags); + const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags); // If all of the other operands were loop invariant, we are done. if (Ops.size() == 1) return NewRec; @@ -1606,7 +1686,8 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, } Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; } - Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop); + // Step size has changed, so we cannot guarantee no self-wraparound. + Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap); return getAddExpr(Ops); } @@ -1630,15 +1711,16 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, O, Ops.size()); UniqueSCEVs.InsertNode(S, IP); } - if (HasNUW) S->setHasNoUnsignedWrap(true); - if (HasNSW) S->setHasNoSignedWrap(true); + S->setNoWrapFlags(Flags); return S; } /// getMulExpr - Get a canonical multiply expression, or something simpler if /// possible. const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, - bool HasNUW, bool HasNSW) { + SCEV::NoWrapFlags Flags) { + assert(Flags == maskFlags(Flags, SCEV::FlagNUW | SCEV::FlagNSW) && + "only nuw or nsw allowed"); assert(!Ops.empty() && "Cannot get empty mul!"); if (Ops.size() == 1) return Ops[0]; #ifndef NDEBUG @@ -1648,8 +1730,8 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, "SCEVMulExpr operand types don't match!"); #endif - // If HasNSW is true and all the operands are non-negative, infer HasNUW. - if (!HasNUW && HasNSW) { + // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. + if (!(Flags & SCEV::FlagNUW) && (Flags & SCEV::FlagNSW)) { bool All = true; for (SmallVectorImpl::const_iterator I = Ops.begin(), E = Ops.end(); I != E; ++I) @@ -1657,7 +1739,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, All = false; break; } - if (All) HasNUW = true; + if (All) Flags = setFlags(Flags, SCEV::FlagNUW); } // Sort by complexity, this groups all similar expression types together. @@ -1697,12 +1779,12 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, } else if (Ops[0]->isAllOnesValue()) { // If we have a mul by -1 of an add, try distributing the -1 among the // add operands. - if (Ops.size() == 2) + if (Ops.size() == 2) { if (const SCEVAddExpr *Add = dyn_cast(Ops[1])) { SmallVector NewOps; bool AnyFolded = false; - for (SCEVAddRecExpr::op_iterator I = Add->op_begin(), E = Add->op_end(); - I != E; ++I) { + for (SCEVAddRecExpr::op_iterator I = Add->op_begin(), + E = Add->op_end(); I != E; ++I) { const SCEV *Mul = getMulExpr(Ops[0], *I); if (!isa(Mul)) AnyFolded = true; NewOps.push_back(Mul); @@ -1710,6 +1792,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, if (AnyFolded) return getAddExpr(NewOps); } + } } if (Ops.size() == 1) @@ -1769,9 +1852,11 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, // Build the new addrec. Propagate the NUW and NSW flags if both the // outer mul and the inner addrec are guaranteed to have no overflow. - const SCEV *NewRec = getAddRecExpr(NewOps, AddRecLoop, - HasNUW && AddRec->hasNoUnsignedWrap(), - HasNSW && AddRec->hasNoSignedWrap()); + // + // No self-wrap cannot be guaranteed after changing the step size, but + // will be infered if either NUW or NSW is true. + Flags = AddRec->getNoWrapFlags(clearFlags(Flags, SCEV::FlagNW)); + const SCEV *NewRec = getAddRecExpr(NewOps, AddRecLoop, Flags); // If all of the other operands were loop invariant, we are done. if (Ops.size() == 1) return NewRec; @@ -1807,7 +1892,8 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, getMulExpr(G, B), getMulExpr(B, D)); const SCEV *NewAddRec = getAddRecExpr(NewStart, NewStep, - F->getLoop()); + F->getLoop(), + SCEV::FlagAnyWrap); if (Ops.size() == 2) return NewAddRec; Ops[Idx] = AddRec = cast(NewAddRec); Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; @@ -1835,8 +1921,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, O, Ops.size()); UniqueSCEVs.InsertNode(S, IP); } - if (HasNUW) S->setHasNoUnsignedWrap(true); - if (HasNSW) S->setHasNoSignedWrap(true); + S->setNoWrapFlags(Flags); return S; } @@ -1876,11 +1961,13 @@ const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS, getZeroExtendExpr(AR, ExtTy) == getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy), getZeroExtendExpr(Step, ExtTy), - AR->getLoop())) { + AR->getLoop(), SCEV::FlagAnyWrap)) { SmallVector Operands; for (unsigned i = 0, e = AR->getNumOperands(); i != e; ++i) Operands.push_back(getUDivExpr(AR->getOperand(i), RHS)); - return getAddRecExpr(Operands, AR->getLoop()); + return getAddRecExpr(Operands, AR->getLoop(), + // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) + SCEV::FlagAnyWrap); } // (A*B)/C --> A*(B/C) if safe and B/C can be folded. if (const SCEVMulExpr *M = dyn_cast(LHS)) { @@ -1944,27 +2031,27 @@ const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS, /// getAddRecExpr - Get an add recurrence expression for the specified loop. /// Simplify the expression as much as possible. -const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, - const SCEV *Step, const Loop *L, - bool HasNUW, bool HasNSW) { +const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step, + const Loop *L, + SCEV::NoWrapFlags Flags) { SmallVector Operands; Operands.push_back(Start); if (const SCEVAddRecExpr *StepChrec = dyn_cast(Step)) if (StepChrec->getLoop() == L) { Operands.append(StepChrec->op_begin(), StepChrec->op_end()); - return getAddRecExpr(Operands, L); + // FIXME: can use maskFlags(Flags, SCEV::FlagNW) + return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); } Operands.push_back(Step); - return getAddRecExpr(Operands, L, HasNUW, HasNSW); + return getAddRecExpr(Operands, L, Flags); } /// getAddRecExpr - Get an add recurrence expression for the specified loop. /// Simplify the expression as much as possible. const SCEV * ScalarEvolution::getAddRecExpr(SmallVectorImpl &Operands, - const Loop *L, - bool HasNUW, bool HasNSW) { + const Loop *L, SCEV::NoWrapFlags Flags) { if (Operands.size() == 1) return Operands[0]; #ifndef NDEBUG const Type *ETy = getEffectiveSCEVType(Operands[0]->getType()); @@ -1978,7 +2065,7 @@ ScalarEvolution::getAddRecExpr(SmallVectorImpl &Operands, if (Operands.back()->isZero()) { Operands.pop_back(); - return getAddRecExpr(Operands, L, HasNUW, HasNSW); // {X,+,0} --> X + return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0} --> X } // It's tempting to want to call getMaxBackedgeTakenCount count here and @@ -1987,8 +2074,8 @@ ScalarEvolution::getAddRecExpr(SmallVectorImpl &Operands, // meaningful BE count at this point (and if we don't, we'd be stuck // with a SCEVCouldNotCompute as the cached BE count). - // If HasNSW is true and all the operands are non-negative, infer HasNUW. - if (!HasNUW && HasNSW) { + // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. + if (!(Flags & SCEV::FlagNUW) && (Flags & SCEV::FlagNSW)) { bool All = true; for (SmallVectorImpl::const_iterator I = Operands.begin(), E = Operands.end(); I != E; ++I) @@ -1996,7 +2083,7 @@ ScalarEvolution::getAddRecExpr(SmallVectorImpl &Operands, All = false; break; } - if (All) HasNUW = true; + if (All) Flags = setFlags(Flags, SCEV::FlagNUW); } // Canonicalize nested AddRecs in by nesting them in order of loop depth. @@ -2019,16 +2106,31 @@ ScalarEvolution::getAddRecExpr(SmallVectorImpl &Operands, break; } if (AllInvariant) { - NestedOperands[0] = getAddRecExpr(Operands, L); + // Create a recurrence for the outer loop with the same step size. + // + // FIXME: + // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the + // inner recurrence has the same property. + // maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags()); + SCEV::NoWrapFlags OuterFlags = SCEV::FlagAnyWrap; + + NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags); AllInvariant = true; for (unsigned i = 0, e = NestedOperands.size(); i != e; ++i) if (!isLoopInvariant(NestedOperands[i], NestedLoop)) { AllInvariant = false; break; } - if (AllInvariant) + if (AllInvariant) { // Ok, both add recurrences are valid after the transformation. - return getAddRecExpr(NestedOperands, NestedLoop, HasNUW, HasNSW); + // + // FIXME: + // The inner recurrence keeps its NW flag but only keeps NUW/NSW if + // the outer recurrence has the same property. + // maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags); + SCEV::NoWrapFlags InnerFlags = SCEV::FlagAnyWrap; + return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags); + } } // Reset Operands to its original state. Operands[0] = NestedAR; @@ -2052,8 +2154,7 @@ ScalarEvolution::getAddRecExpr(SmallVectorImpl &Operands, O, Operands.size(), L); UniqueSCEVs.InsertNode(S, IP); } - if (HasNUW) S->setHasNoUnsignedWrap(true); - if (HasNSW) S->setHasNoSignedWrap(true); + S->setNoWrapFlags(Flags); return S; } @@ -2448,24 +2549,24 @@ const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) { return getMinusSCEV(AllOnes, V); } -/// getMinusSCEV - Return a SCEV corresponding to LHS - RHS. +/// getMinusSCEV - Return LHS-RHS. Minus is represented in SCEV as A+B*-1. /// -const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, - const SCEV *RHS) { +/// FIXME: prohibit FlagNUW here, as soon as getMinusSCEVForExitTest goes. +const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS, + SCEV::NoWrapFlags Flags) { // Fast path: X - X --> 0. if (LHS == RHS) return getConstant(LHS->getType(), 0); // X - Y --> X + -Y - return getAddExpr(LHS, getNegativeSCEV(RHS)); + return getAddExpr(LHS, getNegativeSCEV(RHS), Flags); } /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the /// input value to the specified type. If the type must be extended, it is zero /// extended. const SCEV * -ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, - const Type *Ty) { +ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, const Type *Ty) { const Type *SrcTy = V->getType(); assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && (Ty->isIntegerTy() || Ty->isPointerTy()) && @@ -2636,10 +2737,7 @@ ScalarEvolution::ForgetSymbolicName(Instruction *PN, const SCEV *SymName) { if (!isa(I) || !isa(Old) || (I != PN && Old == SymName)) { - ValuesAtScopes.erase(Old); - LoopDispositions.erase(Old); - UnsignedRanges.erase(Old); - SignedRanges.erase(Old); + forgetMemoizedResults(Old); ValueExprMap.erase(It); } } @@ -2714,27 +2812,35 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { if (isLoopInvariant(Accum, L) || (isa(Accum) && cast(Accum)->getLoop() == L)) { - bool HasNUW = false; - bool HasNSW = false; + SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; // If the increment doesn't overflow, then neither the addrec nor // the post-increment will overflow. if (const AddOperator *OBO = dyn_cast(BEValueV)) { if (OBO->hasNoUnsignedWrap()) - HasNUW = true; + Flags = setFlags(Flags, SCEV::FlagNUW); if (OBO->hasNoSignedWrap()) - HasNSW = true; + Flags = setFlags(Flags, SCEV::FlagNSW); + } else if (const GEPOperator *GEP = + dyn_cast(BEValueV)) { + // If the increment is an inbounds GEP, then we know the address + // space cannot be wrapped around. We cannot make any guarantee + // about signed or unsigned overflow because pointers are + // unsigned but we may have a negative index from the base + // pointer. + if (GEP->isInBounds()) + // FIXME: should be SCEV::FlagNW + Flags = setFlags(Flags, SCEV::FlagNSW); } const SCEV *StartVal = getSCEV(StartValueV); - const SCEV *PHISCEV = - getAddRecExpr(StartVal, Accum, L, HasNUW, HasNSW); + const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags); // Since the no-wrap flags are on the increment, they apply to the // post-incremented value as well. if (isLoopInvariant(Accum, L)) (void)getAddRecExpr(getAddExpr(StartVal, Accum), - Accum, L, HasNUW, HasNSW); + Accum, L, Flags); // Okay, for the entire analysis of this edge we assumed the PHI // to be symbolic. We now need to go back and purge all of the @@ -2758,8 +2864,11 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { // initial step of the addrec evolution. if (StartVal == getMinusSCEV(AddRec->getOperand(0), AddRec->getOperand(1))) { + // FIXME: For constant StartVal, we should be able to infer + // no-wrap flags. const SCEV *PHISCEV = - getAddRecExpr(StartVal, AddRec->getOperand(1), L); + getAddRecExpr(StartVal, AddRec->getOperand(1), L, + SCEV::FlagAnyWrap); // Okay, for the entire analysis of this edge we assumed the PHI // to be symbolic. We now need to go back and purge all of the @@ -2777,25 +2886,10 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { // PHI's incoming blocks are in a different loop, in which case doing so // risks breaking LCSSA form. Instcombine would normally zap these, but // it doesn't have DominatorTree information, so it may miss cases. - if (Value *V = SimplifyInstruction(PN, TD, DT)) { - Instruction *I = dyn_cast(V); - // Only instructions are problematic for preserving LCSSA form. - if (!I) + if (Value *V = SimplifyInstruction(PN, TD, DT)) + if (LI->replacementPreservesLCSSAForm(PN, V)) return getSCEV(V); - // If the instruction is not defined in a loop, then it can be used freely. - Loop *ILoop = LI->getLoopFor(I->getParent()); - if (!ILoop) - return getSCEV(I); - - // If the instruction is defined in the same loop as the phi node, or in a - // loop that contains the phi node loop as an inner loop, then using it as - // a replacement for the phi node will not break LCSSA form. - Loop *PNLoop = LI->getLoopFor(PN->getParent()); - if (ILoop->contains(PNLoop)) - return getSCEV(I); - } - // If it's not a loop phi, we can't handle it yet. return getUnknown(PN); } @@ -2809,6 +2903,7 @@ const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) { // Add expression, because the Instruction may be guarded by control flow // and the no-overflow bits may not be valid for the expression in any // context. + bool isInBounds = GEP->isInBounds(); const Type *IntPtrTy = getEffectiveSCEVType(GEP->getType()); Value *Base = GEP->getOperand(0); @@ -2837,7 +2932,9 @@ const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) { IndexS = getTruncateOrSignExtend(IndexS, IntPtrTy); // Multiply the index by the element size to compute the element offset. - const SCEV *LocalOffset = getMulExpr(IndexS, ElementSize); + const SCEV *LocalOffset = getMulExpr(IndexS, ElementSize, + isInBounds ? SCEV::FlagNSW : + SCEV::FlagAnyWrap); // Add the element offset to the running total offset. TotalOffset = getAddExpr(TotalOffset, LocalOffset); @@ -2848,7 +2945,8 @@ const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) { const SCEV *BaseS = getSCEV(Base); // Add the total offset from all the GEP indices to the base. - return getAddExpr(BaseS, TotalOffset); + return getAddExpr(BaseS, TotalOffset, + isInBounds ? SCEV::FlagNSW : SCEV::FlagAnyWrap); } /// GetMinTrailingZeros - Determine the minimum number of zero bits that S is @@ -3010,7 +3108,8 @@ ScalarEvolution::getUnsignedRange(const SCEV *S) { if (const SCEVAddRecExpr *AddRec = dyn_cast(S)) { // If there's no unsigned wrap, the value will never be less than its // initial value. - if (AddRec->hasNoUnsignedWrap()) + // FIXME: can broaden to FlagNW? + if (AddRec->getNoWrapFlags(SCEV::FlagNUW)) if (const SCEVConstant *C = dyn_cast(AddRec->getStart())) if (!C->getValue()->isZero()) ConservativeResult = @@ -3078,6 +3177,7 @@ ScalarEvolution::getUnsignedRange(const SCEV *S) { /// ConstantRange ScalarEvolution::getSignedRange(const SCEV *S) { + // See if we've computed this range already. DenseMap::iterator I = SignedRanges.find(S); if (I != SignedRanges.end()) return I->second; @@ -3151,7 +3251,7 @@ ScalarEvolution::getSignedRange(const SCEV *S) { if (const SCEVAddRecExpr *AddRec = dyn_cast(S)) { // If there's no signed wrap, and all the operands have the same sign or // zero, the value won't ever change sign. - if (AddRec->hasNoSignedWrap()) { + if (AddRec->getNoWrapFlags(SCEV::FlagNSW)) { bool AllNonNeg = true; bool AllNonPos = true; for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) { @@ -3284,7 +3384,7 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) { SmallVector MulOps; MulOps.push_back(getSCEV(U->getOperand(1))); for (Value *Op = U->getOperand(0); - Op->getValueID() == Instruction::Mul + Value::InstructionVal; + Op->getValueID() == Instruction::Mul + Value::InstructionVal; Op = U->getOperand(0)) { U = cast(Op); MulOps.push_back(getSCEV(U->getOperand(1))); @@ -3346,10 +3446,8 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) { // transfer the no-wrap flags, since an or won't introduce a wrap. if (const SCEVAddRecExpr *NewAR = dyn_cast(S)) { const SCEVAddRecExpr *OldAR = cast(LHS); - if (OldAR->hasNoUnsignedWrap()) - const_cast(NewAR)->setHasNoUnsignedWrap(true); - if (OldAR->hasNoSignedWrap()) - const_cast(NewAR)->setHasNoSignedWrap(true); + const_cast(NewAR)->setNoWrapFlags( + OldAR->getNoWrapFlags()); } return S; } @@ -3391,8 +3489,8 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) { // If C is a single bit, it may be in the sign-bit position // before the zero-extend. In this case, represent the xor // using an add, which is equivalent, and re-apply the zext. - APInt Trunc = APInt(CI->getValue()).trunc(Z0TySize); - if (APInt(Trunc).zext(getTypeSizeInBits(UTy)) == CI->getValue() && + APInt Trunc = CI->getValue().trunc(Z0TySize); + if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() && Trunc.isSignBit()) return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)), UTy); @@ -3632,63 +3730,61 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { // backedge-taken count, which could result in infinite recursion. std::pair::iterator, bool> Pair = BackedgeTakenCounts.insert(std::make_pair(L, getCouldNotCompute())); - if (Pair.second) { - BackedgeTakenInfo BECount = ComputeBackedgeTakenCount(L); - if (BECount.Exact != getCouldNotCompute()) { - assert(isLoopInvariant(BECount.Exact, L) && - isLoopInvariant(BECount.Max, L) && - "Computed backedge-taken count isn't loop invariant for loop!"); - ++NumTripCountsComputed; + if (!Pair.second) + return Pair.first->second; + + BackedgeTakenInfo BECount = ComputeBackedgeTakenCount(L); + if (BECount.Exact != getCouldNotCompute()) { + assert(isLoopInvariant(BECount.Exact, L) && + isLoopInvariant(BECount.Max, L) && + "Computed backedge-taken count isn't loop invariant for loop!"); + ++NumTripCountsComputed; + // Update the value in the map. + Pair.first->second = BECount; + } else { + if (BECount.Max != getCouldNotCompute()) // Update the value in the map. Pair.first->second = BECount; - } else { - if (BECount.Max != getCouldNotCompute()) - // Update the value in the map. - Pair.first->second = BECount; - if (isa(L->getHeader()->begin())) - // Only count loops that have phi nodes as not being computable. - ++NumTripCountsNotComputed; - } - - // Now that we know more about the trip count for this loop, forget any - // existing SCEV values for PHI nodes in this loop since they are only - // conservative estimates made without the benefit of trip count - // information. This is similar to the code in forgetLoop, except that - // it handles SCEVUnknown PHI nodes specially. - if (BECount.hasAnyInfo()) { - SmallVector Worklist; - PushLoopPHIs(L, Worklist); - - SmallPtrSet Visited; - while (!Worklist.empty()) { - Instruction *I = Worklist.pop_back_val(); - if (!Visited.insert(I)) continue; - - ValueExprMapType::iterator It = - ValueExprMap.find(static_cast(I)); - if (It != ValueExprMap.end()) { - const SCEV *Old = It->second; - - // SCEVUnknown for a PHI either means that it has an unrecognized - // structure, or it's a PHI that's in the progress of being computed - // by createNodeForPHI. In the former case, additional loop trip - // count information isn't going to change anything. In the later - // case, createNodeForPHI will perform the necessary updates on its - // own when it gets to that point. - if (!isa(I) || !isa(Old)) { - ValuesAtScopes.erase(Old); - LoopDispositions.erase(Old); - UnsignedRanges.erase(Old); - SignedRanges.erase(Old); - ValueExprMap.erase(It); - } - if (PHINode *PN = dyn_cast(I)) - ConstantEvolutionLoopExitValue.erase(PN); + if (isa(L->getHeader()->begin())) + // Only count loops that have phi nodes as not being computable. + ++NumTripCountsNotComputed; + } + + // Now that we know more about the trip count for this loop, forget any + // existing SCEV values for PHI nodes in this loop since they are only + // conservative estimates made without the benefit of trip count + // information. This is similar to the code in forgetLoop, except that + // it handles SCEVUnknown PHI nodes specially. + if (BECount.hasAnyInfo()) { + SmallVector Worklist; + PushLoopPHIs(L, Worklist); + + SmallPtrSet Visited; + while (!Worklist.empty()) { + Instruction *I = Worklist.pop_back_val(); + if (!Visited.insert(I)) continue; + + ValueExprMapType::iterator It = + ValueExprMap.find(static_cast(I)); + if (It != ValueExprMap.end()) { + const SCEV *Old = It->second; + + // SCEVUnknown for a PHI either means that it has an unrecognized + // structure, or it's a PHI that's in the progress of being computed + // by createNodeForPHI. In the former case, additional loop trip + // count information isn't going to change anything. In the later + // case, createNodeForPHI will perform the necessary updates on its + // own when it gets to that point. + if (!isa(I) || !isa(Old)) { + forgetMemoizedResults(Old); + ValueExprMap.erase(It); } - - PushDefUseChildren(I, Worklist); + if (PHINode *PN = dyn_cast(I)) + ConstantEvolutionLoopExitValue.erase(PN); } + + PushDefUseChildren(I, Worklist); } } return Pair.first->second; @@ -3712,11 +3808,7 @@ void ScalarEvolution::forgetLoop(const Loop *L) { ValueExprMapType::iterator It = ValueExprMap.find(static_cast(I)); if (It != ValueExprMap.end()) { - const SCEV *Old = It->second; - ValuesAtScopes.erase(Old); - LoopDispositions.erase(Old); - UnsignedRanges.erase(Old); - SignedRanges.erase(Old); + forgetMemoizedResults(It->second); ValueExprMap.erase(It); if (PHINode *PN = dyn_cast(I)) ConstantEvolutionLoopExitValue.erase(PN); @@ -3749,11 +3841,7 @@ void ScalarEvolution::forgetValue(Value *V) { ValueExprMapType::iterator It = ValueExprMap.find(static_cast(I)); if (It != ValueExprMap.end()) { - const SCEV *Old = It->second; - ValuesAtScopes.erase(Old); - LoopDispositions.erase(Old); - UnsignedRanges.erase(Old); - SignedRanges.erase(Old); + forgetMemoizedResults(It->second); ValueExprMap.erase(It); if (PHINode *PN = dyn_cast(I)) ConstantEvolutionLoopExitValue.erase(PN); @@ -3967,6 +4055,106 @@ ScalarEvolution::ComputeBackedgeTakenCountFromExitCond(const Loop *L, return ComputeBackedgeTakenCountExhaustively(L, ExitCond, !L->contains(TBB)); } +static const SCEVAddRecExpr * +isSimpleUnwrappingAddRec(const SCEV *S, const Loop *L) { + const SCEVAddRecExpr *SA = dyn_cast(S); + + // The SCEV must be an addrec of this loop. + if (!SA || SA->getLoop() != L || !SA->isAffine()) + return 0; + + // The SCEV must be known to not wrap in some way to be interesting. + if (!SA->getNoWrapFlags(SCEV::FlagNW)) + return 0; + + // The stride must be a constant so that we know if it is striding up or down. + if (!isa(SA->getOperand(1))) + return 0; + return SA; +} + +/// getMinusSCEVForExitTest - When considering an exit test for a loop with a +/// "x != y" exit test, we turn this into a computation that evaluates x-y != 0, +/// and this function returns the expression to use for x-y. We know and take +/// advantage of the fact that this subtraction is only being used in a +/// comparison by zero context. +/// +/// FIXME: this can be completely removed once AddRec FlagNWs are propagated. +static const SCEV *getMinusSCEVForExitTest(const SCEV *LHS, const SCEV *RHS, + const Loop *L, ScalarEvolution &SE) { + // If either LHS or RHS is an AddRec SCEV (of this loop) that is known to not + // self-wrap, then we know that the value will either become the other one + // (and thus the loop terminates), that the loop will terminate through some + // other exit condition first, or that the loop has undefined behavior. This + // information is useful when the addrec has a stride that is != 1 or -1, + // because it means we can't "miss" the exit value. + // + // In any of these three cases, it is safe to turn the exit condition into a + // "counting down" AddRec (to zero) by subtracting the two inputs as normal, + // but since we know that the "end cannot be missed" we can force the + // resulting AddRec to be a NUW addrec. Since it is counting down, this means + // that the AddRec *cannot* pass zero. + + // See if LHS and RHS are addrec's we can handle. + const SCEVAddRecExpr *LHSA = isSimpleUnwrappingAddRec(LHS, L); + const SCEVAddRecExpr *RHSA = isSimpleUnwrappingAddRec(RHS, L); + + // If neither addrec is interesting, just return a minus. + if (RHSA == 0 && LHSA == 0) + return SE.getMinusSCEV(LHS, RHS); + + // If only one of LHS and RHS are an AddRec of this loop, make sure it is LHS. + if (RHSA && LHSA == 0) { + // Safe because a-b === b-a for comparisons against zero. + std::swap(LHS, RHS); + std::swap(LHSA, RHSA); + } + + // Handle the case when only one is advancing in a non-overflowing way. + if (RHSA == 0) { + // If RHS is loop varying, then we can't predict when LHS will cross it. + if (!SE.isLoopInvariant(RHS, L)) + return SE.getMinusSCEV(LHS, RHS); + + // If LHS has a positive stride, then we compute RHS-LHS, because the loop + // is counting up until it crosses RHS (which must be larger than LHS). If + // it is negative, we compute LHS-RHS because we're counting down to RHS. + const ConstantInt *Stride = + cast(LHSA->getOperand(1))->getValue(); + if (Stride->getValue().isNegative()) + std::swap(LHS, RHS); + + return SE.getMinusSCEV(RHS, LHS, SCEV::FlagNUW); + } + + // If both LHS and RHS are interesting, we have something like: + // a+i*4 != b+i*8. + const ConstantInt *LHSStride = + cast(LHSA->getOperand(1))->getValue(); + const ConstantInt *RHSStride = + cast(RHSA->getOperand(1))->getValue(); + + // If the strides are equal, then this is just a (complex) loop invariant + // comparison of a and b. + if (LHSStride == RHSStride) + return SE.getMinusSCEV(LHSA->getStart(), RHSA->getStart()); + + // If the signs of the strides differ, then the negative stride is counting + // down to the positive stride. + if (LHSStride->getValue().isNegative() != RHSStride->getValue().isNegative()){ + if (RHSStride->getValue().isNegative()) + std::swap(LHS, RHS); + } else { + // If LHS's stride is smaller than RHS's stride, then "b" must be less than + // "a" and "b" is RHS is counting up (catching up) to LHS. This is true + // whether the strides are positive or negative. + if (RHSStride->getValue().slt(LHSStride->getValue())) + std::swap(LHS, RHS); + } + + return SE.getMinusSCEV(LHS, RHS, SCEV::FlagNUW); +} + /// ComputeBackedgeTakenCountFromExitCondICmp - Compute the number of times the /// backedge of the specified loop will execute if its exit condition /// were a conditional branch of the ICmpInst ExitCond, TBB, and FBB. @@ -4026,7 +4214,10 @@ ScalarEvolution::ComputeBackedgeTakenCountFromExitCondICmp(const Loop *L, switch (Cond) { case ICmpInst::ICMP_NE: { // while (X != Y) // Convert to: while (X-Y != 0) - BackedgeTakenInfo BTI = HowFarToZero(getMinusSCEV(LHS, RHS), L); + // FIXME: Once AddRec FlagNW are propagated, should be: + // BackedgeTakenInfo BTI = HowFarToZero(getMinusSCEV(LHS, RHS), L); + BackedgeTakenInfo BTI = HowFarToZero(getMinusSCEVForExitTest(LHS, RHS, L, + *this), L); if (BTI.hasAnyInfo()) return BTI; break; } @@ -4551,7 +4742,10 @@ const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) { for (++i; i != e; ++i) NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L)); - AddRec = cast(getAddRecExpr(NewOps, AddRec->getLoop())); + AddRec = cast( + getAddRecExpr(NewOps, AddRec->getLoop(), + // FIXME: AddRec->getNoWrapFlags(SCEV::FlagNW) + SCEV::FlagAnyWrap)); break; } @@ -4637,7 +4831,7 @@ static const SCEV *SolveLinEquationWithOverflow(const APInt &A, const APInt &B, // bit width during computations. APInt AD = A.lshr(Mult2).zext(BW + 1); // AD = A / D APInt Mod(BW + 1, 0); - Mod.set(BW - Mult2); // Mod = N / D + Mod.setBit(BW - Mult2); // Mod = N / D APInt I = AD.multiplicativeInverse(Mod); // 4. Compute the minimum unsigned root of the equation: @@ -4716,6 +4910,11 @@ SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) { /// HowFarToZero - Return the number of times a backedge comparing the specified /// value to zero will execute. If not computable, return CouldNotCompute. +/// +/// This is only used for loops with a "x != y" exit test. The exit condition is +/// now expressed as a single expression, V = x-y. So the exit test is +/// effectively V != 0. We know and take advantage of the fact that this +/// expression only being used in a comparison by zero context. ScalarEvolution::BackedgeTakenInfo ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L) { // If the value is a constant @@ -4729,55 +4928,23 @@ ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L) { if (!AddRec || AddRec->getLoop() != L) return getCouldNotCompute(); - if (AddRec->isAffine()) { - // If this is an affine expression, the execution count of this branch is - // the minimum unsigned root of the following equation: - // - // Start + Step*N = 0 (mod 2^BW) - // - // equivalent to: - // - // Step*N = -Start (mod 2^BW) - // - // where BW is the common bit width of Start and Step. - - // Get the initial value for the loop. - const SCEV *Start = getSCEVAtScope(AddRec->getStart(), - L->getParentLoop()); - const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), - L->getParentLoop()); - - if (const SCEVConstant *StepC = dyn_cast(Step)) { - // For now we handle only constant steps. - - // First, handle unitary steps. - if (StepC->getValue()->equalsInt(1)) // 1*N = -Start (mod 2^BW), so: - return getNegativeSCEV(Start); // N = -Start (as unsigned) - if (StepC->getValue()->isAllOnesValue()) // -1*N = -Start (mod 2^BW), so: - return Start; // N = Start (as unsigned) - - // Then, try to solve the above equation provided that Start is constant. - if (const SCEVConstant *StartC = dyn_cast(Start)) - return SolveLinEquationWithOverflow(StepC->getValue()->getValue(), - -StartC->getValue()->getValue(), - *this); - } - } else if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) { - // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of - // the quadratic equation to solve it. - std::pair Roots = SolveQuadraticEquation(AddRec, - *this); + // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of + // the quadratic equation to solve it. + if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) { + std::pair Roots = + SolveQuadraticEquation(AddRec, *this); const SCEVConstant *R1 = dyn_cast(Roots.first); const SCEVConstant *R2 = dyn_cast(Roots.second); - if (R1) { + if (R1 && R2) { #if 0 dbgs() << "HFTZ: " << *V << " - sol#1: " << *R1 << " sol#2: " << *R2 << "\n"; #endif // Pick the smallest positive root value. if (ConstantInt *CB = - dyn_cast(ConstantExpr::getICmp(ICmpInst::ICMP_ULT, - R1->getValue(), R2->getValue()))) { + dyn_cast(ConstantExpr::getICmp(CmpInst::ICMP_ULT, + R1->getValue(), + R2->getValue()))) { if (CB->getZExtValue() == false) std::swap(R1, R2); // R1 is the minimum root now. @@ -4789,8 +4956,71 @@ ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L) { return R1; // We found a quadratic root! } } + return getCouldNotCompute(); } + // Otherwise we can only handle this if it is affine. + if (!AddRec->isAffine()) + return getCouldNotCompute(); + + // If this is an affine expression, the execution count of this branch is + // the minimum unsigned root of the following equation: + // + // Start + Step*N = 0 (mod 2^BW) + // + // equivalent to: + // + // Step*N = -Start (mod 2^BW) + // + // where BW is the common bit width of Start and Step. + + // Get the initial value for the loop. + const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop()); + const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop()); + + // For now we handle only constant steps. + // + // TODO: Handle a nonconstant Step given AddRec. If the + // AddRec is NUW, then (in an unsigned sense) it cannot be counting up to wrap + // to 0, it must be counting down to equal 0. Consequently, N = Start / -Step. + // We have not yet seen any such cases. + const SCEVConstant *StepC = dyn_cast(Step); + if (StepC == 0) + return getCouldNotCompute(); + + // For positive steps (counting up until unsigned overflow): + // N = -Start/Step (as unsigned) + // For negative steps (counting down to zero): + // N = Start/-Step + // First compute the unsigned distance from zero in the direction of Step. + bool CountDown = StepC->getValue()->getValue().isNegative(); + const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start); + + // Handle unitary steps, which cannot wraparound. + // 1*N = -Start; -1*N = Start (mod 2^BW), so: + // N = Distance (as unsigned) + if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) + return Distance; + + // If the recurrence is known not to wraparound, unsigned divide computes the + // back edge count. We know that the value will either become zero (and thus + // the loop terminates), that the loop will terminate through some other exit + // condition first, or that the loop has undefined behavior. This means + // we can't "miss" the exit value, even with nonunit stride. + // + // FIXME: Prove that loops always exhibits *acceptable* undefined + // behavior. Loops must exhibit defined behavior until a wrapped value is + // actually used. So the trip count computed by udiv could be smaller than the + // number of well-defined iterations. + if (AddRec->getNoWrapFlags(SCEV::FlagNW)) + // FIXME: We really want an "isexact" bit for udiv. + return getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step); + + // Then, try to solve the above equation provided that Start is constant. + if (const SCEVConstant *StartC = dyn_cast(Start)) + return SolveLinEquationWithOverflow(StepC->getValue()->getValue(), + -StartC->getValue()->getValue(), + *this); return getCouldNotCompute(); } @@ -5051,12 +5281,12 @@ bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, case ICmpInst::ICMP_SLE: if (!getSignedRange(RHS).getSignedMax().isMaxSignedValue()) { RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, - /*HasNUW=*/false, /*HasNSW=*/true); + SCEV::FlagNSW); Pred = ICmpInst::ICMP_SLT; Changed = true; } else if (!getSignedRange(LHS).getSignedMin().isMinSignedValue()) { LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS, - /*HasNUW=*/false, /*HasNSW=*/true); + SCEV::FlagNSW); Pred = ICmpInst::ICMP_SLT; Changed = true; } @@ -5064,12 +5294,12 @@ bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, case ICmpInst::ICMP_SGE: if (!getSignedRange(RHS).getSignedMin().isMinSignedValue()) { RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS, - /*HasNUW=*/false, /*HasNSW=*/true); + SCEV::FlagNSW); Pred = ICmpInst::ICMP_SGT; Changed = true; } else if (!getSignedRange(LHS).getSignedMax().isMaxSignedValue()) { LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, - /*HasNUW=*/false, /*HasNSW=*/true); + SCEV::FlagNSW); Pred = ICmpInst::ICMP_SGT; Changed = true; } @@ -5077,12 +5307,12 @@ bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, case ICmpInst::ICMP_ULE: if (!getUnsignedRange(RHS).getUnsignedMax().isMaxValue()) { RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, - /*HasNUW=*/true, /*HasNSW=*/false); + SCEV::FlagNUW); Pred = ICmpInst::ICMP_ULT; Changed = true; } else if (!getUnsignedRange(LHS).getUnsignedMin().isMinValue()) { LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS, - /*HasNUW=*/true, /*HasNSW=*/false); + SCEV::FlagNUW); Pred = ICmpInst::ICMP_ULT; Changed = true; } @@ -5090,12 +5320,12 @@ bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, case ICmpInst::ICMP_UGE: if (!getUnsignedRange(RHS).getUnsignedMin().isMinValue()) { RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS, - /*HasNUW=*/true, /*HasNSW=*/false); + SCEV::FlagNUW); Pred = ICmpInst::ICMP_UGT; Changed = true; } else if (!getUnsignedRange(LHS).getUnsignedMax().isMaxValue()) { LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, - /*HasNUW=*/true, /*HasNSW=*/false); + SCEV::FlagNUW); Pred = ICmpInst::ICMP_UGT; Changed = true; } @@ -5110,13 +5340,13 @@ bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, trivially_true: // Return 0 == 0. - LHS = RHS = getConstant(Type::getInt1Ty(getContext()), 0); + LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); Pred = ICmpInst::ICMP_EQ; return true; trivially_false: // Return 0 != 0. - LHS = RHS = getConstant(Type::getInt1Ty(getContext()), 0); + LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); Pred = ICmpInst::ICMP_NE; return true; } @@ -5477,6 +5707,13 @@ const SCEV *ScalarEvolution::getBECount(const SCEV *Start, "This code doesn't handle negative strides yet!"); const Type *Ty = Start->getType(); + + // When Start == End, we have an exact BECount == 0. Short-circuit this case + // here because SCEV may not be able to determine that the unsigned division + // after rounding is zero. + if (Start == End) + return getConstant(Ty, 0); + const SCEV *NegOne = getConstant(Ty, (uint64_t)-1); const SCEV *Diff = getMinusSCEV(End, Start); const SCEV *RoundUp = getAddExpr(Step, NegOne); @@ -5514,8 +5751,8 @@ ScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS, return getCouldNotCompute(); // Check to see if we have a flag which makes analysis easy. - bool NoWrap = isSigned ? AddRec->hasNoSignedWrap() : - AddRec->hasNoUnsignedWrap(); + bool NoWrap = isSigned ? AddRec->getNoWrapFlags(SCEV::FlagNSW) : + AddRec->getNoWrapFlags(SCEV::FlagNUW); if (AddRec->isAffine()) { unsigned BitWidth = getTypeSizeInBits(AddRec->getType()); @@ -5599,7 +5836,16 @@ ScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS, // The maximum backedge count is similar, except using the minimum start // value and the maximum end value. - const SCEV *MaxBECount = getBECount(MinStart, MaxEnd, Step, NoWrap); + // If we already have an exact constant BECount, use it instead. + const SCEV *MaxBECount = isa(BECount) ? BECount + : getBECount(MinStart, MaxEnd, Step, NoWrap); + + // If the stride is nonconstant, and NoWrap == true, then + // getBECount(MinStart, MaxEnd) may not compute. This would result in an + // exact BECount and invalid MaxBECount, which should be avoided to catch + // more optimization opportunities. + if (isa(MaxBECount)) + MaxBECount = BECount; return BackedgeTakenInfo(BECount, MaxBECount); } @@ -5622,7 +5868,9 @@ const SCEV *SCEVAddRecExpr::getNumIterationsInRange(ConstantRange Range, if (!SC->getValue()->isZero()) { SmallVector Operands(op_begin(), op_end()); Operands[0] = SE.getConstant(SC->getType(), 0); - const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop()); + const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop(), + // FIXME: getNoWrapFlags(FlagNW) + FlagAnyWrap); if (const SCEVAddRecExpr *ShiftedAddRec = dyn_cast(Shifted)) return ShiftedAddRec->getNumIterationsInRange( @@ -5683,7 +5931,9 @@ const SCEV *SCEVAddRecExpr::getNumIterationsInRange(ConstantRange Range, // Range.getUpper() is crossed. SmallVector NewOps(op_begin(), op_end()); NewOps[0] = SE.getNegativeSCEV(SE.getConstant(Range.getUpper())); - const SCEV *NewAddRec = SE.getAddRecExpr(NewOps, getLoop()); + const SCEV *NewAddRec = SE.getAddRecExpr(NewOps, getLoop(), + // getNoWrapFlags(FlagNW) + FlagAnyWrap); // Next, solve the constructed addrec std::pair Roots = @@ -5810,6 +6060,7 @@ void ScalarEvolution::releaseMemory() { ConstantEvolutionLoopExitValue.clear(); ValuesAtScopes.clear(); LoopDispositions.clear(); + BlockDispositions.clear(); UnsignedRanges.clear(); SignedRanges.clear(); UniqueSCEVs.clear(); @@ -6010,64 +6261,35 @@ bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) { return getLoopDisposition(S, L) == LoopComputable; } -bool ScalarEvolution::dominates(const SCEV *S, BasicBlock *BB) const { - switch (S->getSCEVType()) { - case scConstant: - return true; - case scTruncate: - case scZeroExtend: - case scSignExtend: - return dominates(cast(S)->getOperand(), BB); - case scAddRecExpr: { - const SCEVAddRecExpr *AR = cast(S); - if (!DT->dominates(AR->getLoop()->getHeader(), BB)) - return false; - } - // FALL THROUGH into SCEVNAryExpr handling. - case scAddExpr: - case scMulExpr: - case scUMaxExpr: - case scSMaxExpr: { - const SCEVNAryExpr *NAry = cast(S); - for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); - I != E; ++I) - if (!dominates(*I, BB)) - return false; - return true; - } - case scUDivExpr: { - const SCEVUDivExpr *UDiv = cast(S); - return dominates(UDiv->getLHS(), BB) && dominates(UDiv->getRHS(), BB); - } - case scUnknown: - if (Instruction *I = - dyn_cast(cast(S)->getValue())) - return DT->dominates(I->getParent(), BB); - return true; - case scCouldNotCompute: - llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); - return false; - default: break; - } - llvm_unreachable("Unknown SCEV kind!"); - return false; +ScalarEvolution::BlockDisposition +ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) { + std::map &Values = BlockDispositions[S]; + std::pair::iterator, bool> + Pair = Values.insert(std::make_pair(BB, DoesNotDominateBlock)); + if (!Pair.second) + return Pair.first->second; + + BlockDisposition D = computeBlockDisposition(S, BB); + return BlockDispositions[S][BB] = D; } -bool ScalarEvolution::properlyDominates(const SCEV *S, BasicBlock *BB) const { +ScalarEvolution::BlockDisposition +ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) { switch (S->getSCEVType()) { case scConstant: - return true; + return ProperlyDominatesBlock; case scTruncate: case scZeroExtend: case scSignExtend: - return properlyDominates(cast(S)->getOperand(), BB); + return getBlockDisposition(cast(S)->getOperand(), BB); case scAddRecExpr: { // This uses a "dominates" query instead of "properly dominates" query - // because the instruction which produces the addrec's value is a PHI, and - // a PHI effectively properly dominates its entire containing block. + // to test for proper dominance too, because the instruction which + // produces the addrec's value is a PHI, and a PHI effectively properly + // dominates its entire containing block. const SCEVAddRecExpr *AR = cast(S); if (!DT->dominates(AR->getLoop()->getHeader(), BB)) - return false; + return DoesNotDominateBlock; } // FALL THROUGH into SCEVNAryExpr handling. case scAddExpr: @@ -6075,29 +6297,54 @@ bool ScalarEvolution::properlyDominates(const SCEV *S, BasicBlock *BB) const { case scUMaxExpr: case scSMaxExpr: { const SCEVNAryExpr *NAry = cast(S); + bool Proper = true; for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); - I != E; ++I) - if (!properlyDominates(*I, BB)) - return false; - return true; + I != E; ++I) { + BlockDisposition D = getBlockDisposition(*I, BB); + if (D == DoesNotDominateBlock) + return DoesNotDominateBlock; + if (D == DominatesBlock) + Proper = false; + } + return Proper ? ProperlyDominatesBlock : DominatesBlock; } case scUDivExpr: { const SCEVUDivExpr *UDiv = cast(S); - return properlyDominates(UDiv->getLHS(), BB) && - properlyDominates(UDiv->getRHS(), BB); + const SCEV *LHS = UDiv->getLHS(), *RHS = UDiv->getRHS(); + BlockDisposition LD = getBlockDisposition(LHS, BB); + if (LD == DoesNotDominateBlock) + return DoesNotDominateBlock; + BlockDisposition RD = getBlockDisposition(RHS, BB); + if (RD == DoesNotDominateBlock) + return DoesNotDominateBlock; + return (LD == ProperlyDominatesBlock && RD == ProperlyDominatesBlock) ? + ProperlyDominatesBlock : DominatesBlock; } case scUnknown: if (Instruction *I = - dyn_cast(cast(S)->getValue())) - return DT->properlyDominates(I->getParent(), BB); - return true; + dyn_cast(cast(S)->getValue())) { + if (I->getParent() == BB) + return DominatesBlock; + if (DT->properlyDominates(I->getParent(), BB)) + return ProperlyDominatesBlock; + return DoesNotDominateBlock; + } + return ProperlyDominatesBlock; case scCouldNotCompute: llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); - return false; + return DoesNotDominateBlock; default: break; } llvm_unreachable("Unknown SCEV kind!"); - return false; + return DoesNotDominateBlock; +} + +bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) { + return getBlockDisposition(S, BB) >= DominatesBlock; +} + +bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) { + return getBlockDisposition(S, BB) == ProperlyDominatesBlock; } bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const { @@ -6141,3 +6388,11 @@ bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const { llvm_unreachable("Unknown SCEV kind!"); return false; } + +void ScalarEvolution::forgetMemoizedResults(const SCEV *S) { + ValuesAtScopes.erase(S); + LoopDispositions.erase(S); + BlockDispositions.erase(S); + UnsignedRanges.erase(S); + SignedRanges.erase(S); +}