#define SV_NAME "slp-vectorizer"
#define DEBUG_TYPE "SLP"
-#include "VecUtils.h"
#include "llvm/Transforms/Vectorize.h"
#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/Verifier.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
#include <map>
using namespace llvm;
"number. (gain = -cost of vectorization)"));
namespace {
+static const unsigned MinVecRegSize = 128;
+
+static const unsigned RecursionMaxDepth = 6;
+
+/// RAII pattern to save the insertion point of the IR builder.
+class BuilderLocGuard {
+public:
+ BuilderLocGuard(IRBuilder<> &B) : Builder(B), Loc(B.GetInsertPoint()) {}
+ ~BuilderLocGuard() { Builder.SetInsertPoint(Loc); }
+
+private:
+ // Prevent copying.
+ BuilderLocGuard(const BuilderLocGuard &);
+ BuilderLocGuard &operator=(const BuilderLocGuard &);
+ IRBuilder<> &Builder;
+ BasicBlock::iterator Loc;
+};
+
+/// A helper class for numbering instructions in multible blocks.
+/// Numbers starts at zero for each basic block.
+struct BlockNumbering {
+
+ BlockNumbering(BasicBlock *Bb) : BB(Bb), Valid(false) {}
+
+ BlockNumbering() : BB(0), Valid(false) {}
+
+ void numberInstructions() {
+ unsigned Loc = 0;
+ InstrIdx.clear();
+ InstrVec.clear();
+ // Number the instructions in the block.
+ for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
+ InstrIdx[it] = Loc++;
+ InstrVec.push_back(it);
+ assert(InstrVec[InstrIdx[it]] == it && "Invalid allocation");
+ }
+ Valid = true;
+ }
+
+ int getIndex(Instruction *I) {
+ if (!Valid)
+ numberInstructions();
+ assert(InstrIdx.count(I) && "Unknown instruction");
+ return InstrIdx[I];
+ }
+
+ Instruction *getInstruction(unsigned loc) {
+ if (!Valid)
+ numberInstructions();
+ assert(InstrVec.size() > loc && "Invalid Index");
+ return InstrVec[loc];
+ }
+
+ void forget() { Valid = false; }
+
+private:
+ /// The block we are numbering.
+ BasicBlock *BB;
+ /// Is the block numbered.
+ bool Valid;
+ /// Maps instructions to numbers and back.
+ SmallDenseMap<Instruction *, int> InstrIdx;
+ /// Maps integers to Instructions.
+ std::vector<Instruction *> InstrVec;
+};
+
+class FuncSLP {
+ typedef SmallVector<Value *, 8> ValueList;
+ typedef SmallVector<Instruction *, 16> InstrList;
+ typedef SmallPtrSet<Value *, 16> ValueSet;
+ typedef SmallVector<StoreInst *, 8> StoreList;
+
+public:
+ static const int MAX_COST = INT_MIN;
+
+ FuncSLP(Function *Func, ScalarEvolution *Se, DataLayout *Dl,
+ TargetTransformInfo *Tti, AliasAnalysis *Aa, LoopInfo *Li)
+ : F(Func), SE(Se), DL(Dl), TTI(Tti), AA(Aa), LI(Li),
+ Builder(Se->getContext()) {
+ for (Function::iterator it = F->begin(), e = F->end(); it != e; ++it) {
+ BasicBlock *BB = it;
+ BlocksNumbers[BB] = BlockNumbering(BB);
+ }
+ }
+
+ /// \brief Take the pointer operand from the Load/Store instruction.
+ /// \returns NULL if this is not a valid Load/Store instruction.
+ static Value *getPointerOperand(Value *I);
+
+ /// \brief Take the address space operand from the Load/Store instruction.
+ /// \returns -1 if this is not a valid Load/Store instruction.
+ static unsigned getAddressSpaceOperand(Value *I);
+
+ /// \returns true if the memory operations A and B are consecutive.
+ bool isConsecutiveAccess(Value *A, Value *B);
+
+ /// \brief Vectorize the tree that starts with the elements in \p VL.
+ /// \returns the vectorized value.
+ Value *vectorizeTree(ArrayRef<Value *> VL);
+
+ /// \returns the vectorization cost of the subtree that starts at \p VL.
+ /// A negative number means that this is profitable.
+ int getTreeCost(ArrayRef<Value *> VL);
+
+ /// \returns the scalarization cost for this list of values. Assuming that
+ /// this subtree gets vectorized, we may need to extract the values from the
+ /// roots. This method calculates the cost of extracting the values.
+ int getGatherCost(ArrayRef<Value *> VL);
+
+ /// \brief Attempts to order and vectorize a sequence of stores. This
+ /// function does a quadratic scan of the given stores.
+ /// \returns true if the basic block was modified.
+ bool vectorizeStores(ArrayRef<StoreInst *> Stores, int costThreshold);
+
+ /// \brief Vectorize a group of scalars into a vector tree.
+ /// \returns the vectorized value.
+ Value *vectorizeArith(ArrayRef<Value *> Operands);
+
+ /// \brief This method contains the recursive part of getTreeCost.
+ int getTreeCost_rec(ArrayRef<Value *> VL, unsigned Depth);
+
+ /// \brief This recursive method looks for vectorization hazards such as
+ /// values that are used by multiple users and checks that values are used
+ /// by only one vector lane. It updates the variables LaneMap, MultiUserVals.
+ void getTreeUses_rec(ArrayRef<Value *> VL, unsigned Depth);
+
+ /// \brief This method contains the recursive part of vectorizeTree.
+ Value *vectorizeTree_rec(ArrayRef<Value *> VL);
+
+ /// \brief Vectorize a sorted sequence of stores.
+ bool vectorizeStoreChain(ArrayRef<Value *> Chain, int CostThreshold);
+
+ /// \returns the scalarization cost for this type. Scalarization in this
+ /// context means the creation of vectors from a group of scalars.
+ int getGatherCost(Type *Ty);
+
+ /// \returns the AA location that is being access by the instruction.
+ AliasAnalysis::Location getLocation(Instruction *I);
+
+ /// \brief Checks if it is possible to sink an instruction from
+ /// \p Src to \p Dst.
+ /// \returns the pointer to the barrier instruction if we can't sink.
+ Value *getSinkBarrier(Instruction *Src, Instruction *Dst);
+
+ /// \returns the index of the last instrucion in the BB from \p VL.
+ int getLastIndex(ArrayRef<Value *> VL);
+
+ /// \returns the Instrucion in the bundle \p VL.
+ Instruction *getLastInstruction(ArrayRef<Value *> VL);
+
+ /// \returns the Instruction at index \p Index which is in Block \p BB.
+ Instruction *getInstructionForIndex(unsigned Index, BasicBlock *BB);
+
+ /// \returns the index of the first User of \p VL.
+ int getFirstUserIndex(ArrayRef<Value *> VL);
+
+ /// \returns a vector from a collection of scalars in \p VL.
+ Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
+
+ /// \brief Try to hoist gather sequences outside of the loop in cases where
+ /// all of the sources are loop invariant.
+ void hoistGatherSequence();
+
+ bool needToGatherAny(ArrayRef<Value *> VL) {
+ for (int i = 0, e = VL.size(); i < e; ++i)
+ if (MustGather.count(VL[i]))
+ return true;
+ return false;
+ }
+
+ /// -- Vectorization State --
+
+ /// Maps values in the tree to the vector lanes that uses them. This map must
+ /// be reset between runs of getCost.
+ std::map<Value *, int> LaneMap;
+ /// A list of instructions to ignore while sinking
+ /// memory instructions. This map must be reset between runs of getCost.
+ ValueSet MemBarrierIgnoreList;
+
+ /// Maps between the first scalar to the vector. This map must be reset
+ /// between runs.
+ DenseMap<Value *, Value *> VectorizedValues;
+
+ /// Contains values that must be gathered because they are used
+ /// by multiple lanes, or by users outside the tree.
+ /// NOTICE: The vectorization methods also use this set.
+ ValueSet MustGather;
+
+ /// Contains a list of values that are used outside the current tree. This
+ /// set must be reset between runs.
+ SetVector<Value *> MultiUserVals;
+
+ /// Holds all of the instructions that we gathered.
+ SetVector<Instruction *> GatherSeq;
+
+ /// Numbers instructions in different blocks.
+ std::map<BasicBlock *, BlockNumbering> BlocksNumbers;
+
+ // Analysis and block reference.
+ Function *F;
+ ScalarEvolution *SE;
+ DataLayout *DL;
+ TargetTransformInfo *TTI;
+ AliasAnalysis *AA;
+ LoopInfo *LI;
+ /// Instruction builder to construct the vectorized tree.
+ IRBuilder<> Builder;
+};
+
+int FuncSLP::getGatherCost(Type *Ty) {
+ int Cost = 0;
+ for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
+ Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
+ return Cost;
+}
+
+int FuncSLP::getGatherCost(ArrayRef<Value *> VL) {
+ // Find the type of the operands in VL.
+ Type *ScalarTy = VL[0]->getType();
+ if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
+ ScalarTy = SI->getValueOperand()->getType();
+ VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
+ // Find the cost of inserting/extracting values from the vector.
+ return getGatherCost(VecTy);
+}
+
+AliasAnalysis::Location FuncSLP::getLocation(Instruction *I) {
+ if (StoreInst *SI = dyn_cast<StoreInst>(I))
+ return AA->getLocation(SI);
+ if (LoadInst *LI = dyn_cast<LoadInst>(I))
+ return AA->getLocation(LI);
+ return AliasAnalysis::Location();
+}
+
+Value *FuncSLP::getPointerOperand(Value *I) {
+ if (LoadInst *LI = dyn_cast<LoadInst>(I))
+ return LI->getPointerOperand();
+ if (StoreInst *SI = dyn_cast<StoreInst>(I))
+ return SI->getPointerOperand();
+ return 0;
+}
+
+unsigned FuncSLP::getAddressSpaceOperand(Value *I) {
+ if (LoadInst *L = dyn_cast<LoadInst>(I))
+ return L->getPointerAddressSpace();
+ if (StoreInst *S = dyn_cast<StoreInst>(I))
+ return S->getPointerAddressSpace();
+ return -1;
+}
+
+bool FuncSLP::isConsecutiveAccess(Value *A, Value *B) {
+ Value *PtrA = getPointerOperand(A);
+ Value *PtrB = getPointerOperand(B);
+ unsigned ASA = getAddressSpaceOperand(A);
+ unsigned ASB = getAddressSpaceOperand(B);
+
+ // Check that the address spaces match and that the pointers are valid.
+ if (!PtrA || !PtrB || (ASA != ASB))
+ return false;
+
+ // Check that A and B are of the same type.
+ if (PtrA->getType() != PtrB->getType())
+ return false;
+
+ // Calculate the distance.
+ const SCEV *PtrSCEVA = SE->getSCEV(PtrA);
+ const SCEV *PtrSCEVB = SE->getSCEV(PtrB);
+ const SCEV *OffsetSCEV = SE->getMinusSCEV(PtrSCEVA, PtrSCEVB);
+ const SCEVConstant *ConstOffSCEV = dyn_cast<SCEVConstant>(OffsetSCEV);
+
+ // Non constant distance.
+ if (!ConstOffSCEV)
+ return false;
+
+ int64_t Offset = ConstOffSCEV->getValue()->getSExtValue();
+ Type *Ty = cast<PointerType>(PtrA->getType())->getElementType();
+ // The Instructions are connsecutive if the size of the first load/store is
+ // the same as the offset.
+ int64_t Sz = DL->getTypeStoreSize(Ty);
+ return ((-Offset) == Sz);
+}
+
+Value *FuncSLP::getSinkBarrier(Instruction *Src, Instruction *Dst) {
+ assert(Src->getParent() == Dst->getParent() && "Not the same BB");
+ BasicBlock::iterator I = Src, E = Dst;
+ /// Scan all of the instruction from SRC to DST and check if
+ /// the source may alias.
+ for (++I; I != E; ++I) {
+ // Ignore store instructions that are marked as 'ignore'.
+ if (MemBarrierIgnoreList.count(I))
+ continue;
+ if (Src->mayWriteToMemory()) /* Write */ {
+ if (!I->mayReadOrWriteMemory())
+ continue;
+ } else /* Read */ {
+ if (!I->mayWriteToMemory())
+ continue;
+ }
+ AliasAnalysis::Location A = getLocation(&*I);
+ AliasAnalysis::Location B = getLocation(Src);
+
+ if (!A.Ptr || !B.Ptr || AA->alias(A, B))
+ return I;
+ }
+ return 0;
+}
+
+static BasicBlock *getSameBlock(ArrayRef<Value *> VL) {
+ BasicBlock *BB = 0;
+ for (int i = 0, e = VL.size(); i < e; i++) {
+ Instruction *I = dyn_cast<Instruction>(VL[i]);
+ if (!I)
+ return 0;
+
+ if (!BB) {
+ BB = I->getParent();
+ continue;
+ }
+
+ if (BB != I->getParent())
+ return 0;
+ }
+ return BB;
+}
+
+static bool allConstant(ArrayRef<Value *> VL) {
+ for (unsigned i = 0, e = VL.size(); i < e; ++i)
+ if (!isa<Constant>(VL[i]))
+ return false;
+ return true;
+}
+
+static bool isSplat(ArrayRef<Value *> VL) {
+ for (unsigned i = 1, e = VL.size(); i < e; ++i)
+ if (VL[i] != VL[0])
+ return false;
+ return true;
+}
+
+static unsigned getSameOpcode(ArrayRef<Value *> VL) {
+ unsigned Opcode = 0;
+ for (int i = 0, e = VL.size(); i < e; i++) {
+ if (Instruction *I = dyn_cast<Instruction>(VL[i])) {
+ if (!Opcode) {
+ Opcode = I->getOpcode();
+ continue;
+ }
+ if (Opcode != I->getOpcode())
+ return 0;
+ }
+ }
+ return Opcode;
+}
+
+static bool CanReuseExtract(ArrayRef<Value *> VL, unsigned VF,
+ VectorType *VecTy) {
+ assert(Instruction::ExtractElement == getSameOpcode(VL) && "Invalid opcode");
+ // Check if all of the extracts come from the same vector and from the
+ // correct offset.
+ Value *VL0 = VL[0];
+ ExtractElementInst *E0 = cast<ExtractElementInst>(VL0);
+ Value *Vec = E0->getOperand(0);
+
+ // We have to extract from the same vector type.
+ if (Vec->getType() != VecTy)
+ return false;
+
+ // Check that all of the indices extract from the correct offset.
+ ConstantInt *CI = dyn_cast<ConstantInt>(E0->getOperand(1));
+ if (!CI || CI->getZExtValue())
+ return false;
+
+ for (unsigned i = 1, e = VF; i < e; ++i) {
+ ExtractElementInst *E = cast<ExtractElementInst>(VL[i]);
+ ConstantInt *CI = dyn_cast<ConstantInt>(E->getOperand(1));
+
+ if (!CI || CI->getZExtValue() != i || E->getOperand(0) != Vec)
+ return false;
+ }
+
+ return true;
+}
+
+void FuncSLP::getTreeUses_rec(ArrayRef<Value *> VL, unsigned Depth) {
+ if (Depth == RecursionMaxDepth)
+ return MustGather.insert(VL.begin(), VL.end());
+
+ // Don't handle vectors.
+ if (VL[0]->getType()->isVectorTy())
+ return;
+
+ if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
+ if (SI->getValueOperand()->getType()->isVectorTy())
+ return;
+
+ // If all of the operands are identical or constant we have a simple solution.
+ if (allConstant(VL) || isSplat(VL) || !getSameBlock(VL))
+ return MustGather.insert(VL.begin(), VL.end());
+
+ // Stop the scan at unknown IR.
+ Instruction *VL0 = dyn_cast<Instruction>(VL[0]);
+ assert(VL0 && "Invalid instruction");
+
+ // Mark instructions with multiple users.
+ for (unsigned i = 0, e = VL.size(); i < e; ++i) {
+ Instruction *I = dyn_cast<Instruction>(VL[i]);
+ // Remember to check if all of the users of this instruction are vectorized
+ // within our tree. At depth zero we have no local users, only external
+ // users that we don't care about.
+ if (Depth && I && I->getNumUses() > 1) {
+ DEBUG(dbgs() << "SLP: Adding to MultiUserVals "
+ "because it has multiple users:" << *I << " \n");
+ MultiUserVals.insert(I);
+ }
+ }
+
+ // Check that the instruction is only used within one lane.
+ for (int i = 0, e = VL.size(); i < e; ++i) {
+ if (LaneMap.count(VL[i]) && LaneMap[VL[i]] != i) {
+ DEBUG(dbgs() << "SLP: Value used by multiple lanes:" << *VL[i] << "\n");
+ return MustGather.insert(VL.begin(), VL.end());
+ }
+ // Make this instruction as 'seen' and remember the lane.
+ LaneMap[VL[i]] = i;
+ }
+
+ unsigned Opcode = getSameOpcode(VL);
+ if (!Opcode)
+ return MustGather.insert(VL.begin(), VL.end());
+
+ switch (Opcode) {
+ case Instruction::ExtractElement: {
+ VectorType *VecTy = VectorType::get(VL[0]->getType(), VL.size());
+ // No need to follow ExtractElements that are going to be optimized away.
+ if (CanReuseExtract(VL, VL.size(), VecTy))
+ return;
+ // Fall through.
+ }
+ case Instruction::Load:
+ return;
+ case Instruction::ZExt:
+ case Instruction::SExt:
+ case Instruction::FPToUI:
+ case Instruction::FPToSI:
+ case Instruction::FPExt:
+ case Instruction::PtrToInt:
+ case Instruction::IntToPtr:
+ case Instruction::SIToFP:
+ case Instruction::UIToFP:
+ case Instruction::Trunc:
+ case Instruction::FPTrunc:
+ case Instruction::BitCast:
+ case Instruction::Select:
+ case Instruction::ICmp:
+ case Instruction::FCmp:
+ case Instruction::Add:
+ case Instruction::FAdd:
+ case Instruction::Sub:
+ case Instruction::FSub:
+ case Instruction::Mul:
+ case Instruction::FMul:
+ case Instruction::UDiv:
+ case Instruction::SDiv:
+ case Instruction::FDiv:
+ case Instruction::URem:
+ case Instruction::SRem:
+ case Instruction::FRem:
+ case Instruction::Shl:
+ case Instruction::LShr:
+ case Instruction::AShr:
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor: {
+ for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
+ ValueList Operands;
+ // Prepare the operand vector.
+ for (unsigned j = 0; j < VL.size(); ++j)
+ Operands.push_back(cast<Instruction>(VL[j])->getOperand(i));
+
+ getTreeUses_rec(Operands, Depth + 1);
+ }
+ return;
+ }
+ case Instruction::Store: {
+ ValueList Operands;
+ for (unsigned j = 0; j < VL.size(); ++j)
+ Operands.push_back(cast<Instruction>(VL[j])->getOperand(0));
+ getTreeUses_rec(Operands, Depth + 1);
+ return;
+ }
+ default:
+ return MustGather.insert(VL.begin(), VL.end());
+ }
+}
+
+int FuncSLP::getLastIndex(ArrayRef<Value *> VL) {
+ BasicBlock *BB = cast<Instruction>(VL[0])->getParent();
+ assert(BB == getSameBlock(VL) && BlocksNumbers.count(BB) && "Invalid block");
+ BlockNumbering &BN = BlocksNumbers[BB];
+
+ int MaxIdx = BN.getIndex(BB->getFirstNonPHI());
+ for (unsigned i = 0, e = VL.size(); i < e; ++i)
+ MaxIdx = std::max(MaxIdx, BN.getIndex(cast<Instruction>(VL[i])));
+ return MaxIdx;
+}
+
+Instruction *FuncSLP::getLastInstruction(ArrayRef<Value *> VL) {
+ BasicBlock *BB = cast<Instruction>(VL[0])->getParent();
+ assert(BB == getSameBlock(VL) && BlocksNumbers.count(BB) && "Invalid block");
+ BlockNumbering &BN = BlocksNumbers[BB];
+
+ int MaxIdx = BN.getIndex(cast<Instruction>(VL[0]));
+ for (unsigned i = 1, e = VL.size(); i < e; ++i)
+ MaxIdx = std::max(MaxIdx, BN.getIndex(cast<Instruction>(VL[i])));
+ return BN.getInstruction(MaxIdx);
+}
+
+Instruction *FuncSLP::getInstructionForIndex(unsigned Index, BasicBlock *BB) {
+ BlockNumbering &BN = BlocksNumbers[BB];
+ return BN.getInstruction(Index);
+}
+
+int FuncSLP::getFirstUserIndex(ArrayRef<Value *> VL) {
+ BasicBlock *BB = getSameBlock(VL);
+ BlockNumbering &BN = BlocksNumbers[BB];
+
+ // Find the first user of the values.
+ int FirstUser = BN.getIndex(BB->getTerminator());
+ for (unsigned i = 0, e = VL.size(); i < e; ++i) {
+ for (Value::use_iterator U = VL[i]->use_begin(), UE = VL[i]->use_end();
+ U != UE; ++U) {
+ Instruction *Instr = dyn_cast<Instruction>(*U);
+
+ if (!Instr || Instr->getParent() != BB)
+ continue;
+
+ FirstUser = std::min(FirstUser, BN.getIndex(Instr));
+ }
+ }
+ return FirstUser;
+}
+
+int FuncSLP::getTreeCost_rec(ArrayRef<Value *> VL, unsigned Depth) {
+ Type *ScalarTy = VL[0]->getType();
+
+ if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
+ ScalarTy = SI->getValueOperand()->getType();
+
+ /// Don't mess with vectors.
+ if (ScalarTy->isVectorTy())
+ return FuncSLP::MAX_COST;
+
+ VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
+
+ if (allConstant(VL))
+ return 0;
+
+ if (isSplat(VL))
+ return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
+
+ if (Depth == RecursionMaxDepth || needToGatherAny(VL))
+ return getGatherCost(VecTy);
+
+ BasicBlock *BB = getSameBlock(VL);
+ unsigned Opcode = getSameOpcode(VL);
+ assert(Opcode && BB && "Invalid Instruction Value");
+
+ // Check if it is safe to sink the loads or the stores.
+ if (Opcode == Instruction::Load || Opcode == Instruction::Store) {
+ int MaxIdx = getLastIndex(VL);
+ Instruction *Last = getInstructionForIndex(MaxIdx, BB);
+
+ for (unsigned i = 0, e = VL.size(); i < e; ++i) {
+ if (VL[i] == Last)
+ continue;
+ Value *Barrier = getSinkBarrier(cast<Instruction>(VL[i]), Last);
+ if (Barrier) {
+ DEBUG(dbgs() << "SLP: Can't sink " << *VL[i] << "\n down to " << *Last
+ << "\n because of " << *Barrier << "\n");
+ return MAX_COST;
+ }
+ }
+ }
+
+ Instruction *VL0 = cast<Instruction>(VL[0]);
+ switch (Opcode) {
+ case Instruction::ExtractElement: {
+ if (CanReuseExtract(VL, VL.size(), VecTy))
+ return 0;
+ return getGatherCost(VecTy);
+ }
+ case Instruction::ZExt:
+ case Instruction::SExt:
+ case Instruction::FPToUI:
+ case Instruction::FPToSI:
+ case Instruction::FPExt:
+ case Instruction::PtrToInt:
+ case Instruction::IntToPtr:
+ case Instruction::SIToFP:
+ case Instruction::UIToFP:
+ case Instruction::Trunc:
+ case Instruction::FPTrunc:
+ case Instruction::BitCast: {
+ ValueList Operands;
+ Type *SrcTy = VL0->getOperand(0)->getType();
+ // Prepare the operand vector.
+ for (unsigned j = 0; j < VL.size(); ++j) {
+ Operands.push_back(cast<Instruction>(VL[j])->getOperand(0));
+ // Check that the casted type is the same for all users.
+ if (cast<Instruction>(VL[j])->getOperand(0)->getType() != SrcTy)
+ return getGatherCost(VecTy);
+ }
+
+ int Cost = getTreeCost_rec(Operands, Depth + 1);
+ if (Cost == FuncSLP::MAX_COST)
+ return Cost;
+
+ // Calculate the cost of this instruction.
+ int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
+ VL0->getType(), SrcTy);
+
+ VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
+ int VecCost = TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy);
+ Cost += (VecCost - ScalarCost);
+ return Cost;
+ }
+ case Instruction::FCmp:
+ case Instruction::ICmp: {
+ // Check that all of the compares have the same predicate.
+ CmpInst::Predicate P0 = dyn_cast<CmpInst>(VL0)->getPredicate();
+ for (unsigned i = 1, e = VL.size(); i < e; ++i) {
+ CmpInst *Cmp = cast<CmpInst>(VL[i]);
+ if (Cmp->getPredicate() != P0)
+ return getGatherCost(VecTy);
+ }
+ // Fall through.
+ }
+ case Instruction::Select:
+ case Instruction::Add:
+ case Instruction::FAdd:
+ case Instruction::Sub:
+ case Instruction::FSub:
+ case Instruction::Mul:
+ case Instruction::FMul:
+ case Instruction::UDiv:
+ case Instruction::SDiv:
+ case Instruction::FDiv:
+ case Instruction::URem:
+ case Instruction::SRem:
+ case Instruction::FRem:
+ case Instruction::Shl:
+ case Instruction::LShr:
+ case Instruction::AShr:
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor: {
+ int TotalCost = 0;
+ // Calculate the cost of all of the operands.
+ for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
+ ValueList Operands;
+ // Prepare the operand vector.
+ for (unsigned j = 0; j < VL.size(); ++j)
+ Operands.push_back(cast<Instruction>(VL[j])->getOperand(i));
+
+ int Cost = getTreeCost_rec(Operands, Depth + 1);
+ if (Cost == MAX_COST)
+ return MAX_COST;
+ TotalCost += TotalCost;
+ }
+
+ // Calculate the cost of this instruction.
+ int ScalarCost = 0;
+ int VecCost = 0;
+ if (Opcode == Instruction::FCmp || Opcode == Instruction::ICmp ||
+ Opcode == Instruction::Select) {
+ VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
+ ScalarCost =
+ VecTy->getNumElements() *
+ TTI->getCmpSelInstrCost(Opcode, ScalarTy, Builder.getInt1Ty());
+ VecCost = TTI->getCmpSelInstrCost(Opcode, VecTy, MaskTy);
+ } else {
+ ScalarCost = VecTy->getNumElements() *
+ TTI->getArithmeticInstrCost(Opcode, ScalarTy);
+ VecCost = TTI->getArithmeticInstrCost(Opcode, VecTy);
+ }
+ TotalCost += (VecCost - ScalarCost);
+ return TotalCost;
+ }
+ case Instruction::Load: {
+ // If we are scalarize the loads, add the cost of forming the vector.
+ for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
+ if (!isConsecutiveAccess(VL[i], VL[i + 1]))
+ return getGatherCost(VecTy);
+
+ // Cost of wide load - cost of scalar loads.
+ int ScalarLdCost = VecTy->getNumElements() *
+ TTI->getMemoryOpCost(Instruction::Load, ScalarTy, 1, 0);
+ int VecLdCost = TTI->getMemoryOpCost(Instruction::Load, ScalarTy, 1, 0);
+ return VecLdCost - ScalarLdCost;
+ }
+ case Instruction::Store: {
+ // We know that we can merge the stores. Calculate the cost.
+ int ScalarStCost = VecTy->getNumElements() *
+ TTI->getMemoryOpCost(Instruction::Store, ScalarTy, 1, 0);
+ int VecStCost = TTI->getMemoryOpCost(Instruction::Store, ScalarTy, 1, 0);
+ int StoreCost = VecStCost - ScalarStCost;
+
+ ValueList Operands;
+ for (unsigned j = 0; j < VL.size(); ++j) {
+ Operands.push_back(cast<Instruction>(VL[j])->getOperand(0));
+ MemBarrierIgnoreList.insert(VL[j]);
+ }
+
+ int Cost = getTreeCost_rec(Operands, Depth + 1);
+ if (Cost == MAX_COST)
+ return MAX_COST;
+
+ int TotalCost = StoreCost + Cost;
+ return TotalCost;
+ }
+ default:
+ // Unable to vectorize unknown instructions.
+ return getGatherCost(VecTy);
+ }
+}
+
+int FuncSLP::getTreeCost(ArrayRef<Value *> VL) {
+ // Get rid of the list of stores that were removed, and from the
+ // lists of instructions with multiple users.
+ MemBarrierIgnoreList.clear();
+ LaneMap.clear();
+ MultiUserVals.clear();
+ MustGather.clear();
+
+ if (!getSameBlock(VL))
+ return MAX_COST;
+
+ // Find the location of the last root.
+ int LastRootIndex = getLastIndex(VL);
+ int FirstUserIndex = getFirstUserIndex(VL);
+
+ // Don't vectorize if there are users of the tree roots inside the tree
+ // itself.
+ if (LastRootIndex > FirstUserIndex)
+ return MAX_COST;
+
+ // Scan the tree and find which value is used by which lane, and which values
+ // must be scalarized.
+ getTreeUses_rec(VL, 0);
+
+ // Check that instructions with multiple users can be vectorized. Mark unsafe
+ // instructions.
+ for (SetVector<Value *>::iterator it = MultiUserVals.begin(),
+ e = MultiUserVals.end();
+ it != e; ++it) {
+ // Check that all of the users of this instr are within the tree.
+ for (Value::use_iterator I = (*it)->use_begin(), E = (*it)->use_end();
+ I != E; ++I) {
+ if (LaneMap.find(*I) == LaneMap.end()) {
+ DEBUG(dbgs() << "SLP: Adding to MustExtract "
+ "because of an out of tree usage.\n");
+ MustGather.insert(*it);
+ continue;
+ }
+ }
+ }
+
+ // Now calculate the cost of vectorizing the tree.
+ return getTreeCost_rec(VL, 0);
+}
+bool FuncSLP::vectorizeStoreChain(ArrayRef<Value *> Chain, int CostThreshold) {
+ unsigned ChainLen = Chain.size();
+ DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLen
+ << "\n");
+ Type *StoreTy = cast<StoreInst>(Chain[0])->getValueOperand()->getType();
+ unsigned Sz = DL->getTypeSizeInBits(StoreTy);
+ unsigned VF = MinVecRegSize / Sz;
+
+ if (!isPowerOf2_32(Sz) || VF < 2)
+ return false;
+
+ bool Changed = false;
+ // Look for profitable vectorizable trees at all offsets, starting at zero.
+ for (unsigned i = 0, e = ChainLen; i < e; ++i) {
+ if (i + VF > e)
+ break;
+ DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << i
+ << "\n");
+ ArrayRef<Value *> Operands = Chain.slice(i, VF);
+
+ int Cost = getTreeCost(Operands);
+ if (Cost == FuncSLP::MAX_COST)
+ continue;
+ DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n");
+ if (Cost < CostThreshold) {
+ DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
+ vectorizeTree(Operands);
+ i += VF - 1;
+ Changed = true;
+ }
+ }
+
+ if (Changed || ChainLen > VF)
+ return Changed;
+
+ // Handle short chains. This helps us catch types such as <3 x float> that
+ // are smaller than vector size.
+ int Cost = getTreeCost(Chain);
+ if (Cost == FuncSLP::MAX_COST)
+ return false;
+ if (Cost < CostThreshold) {
+ DEBUG(dbgs() << "SLP: Found store chain cost = " << Cost
+ << " for size = " << ChainLen << "\n");
+ vectorizeTree(Chain);
+ return true;
+ }
+
+ return false;
+}
+
+bool FuncSLP::vectorizeStores(ArrayRef<StoreInst *> Stores, int costThreshold) {
+ SetVector<Value *> Heads, Tails;
+ SmallDenseMap<Value *, Value *> ConsecutiveChain;
+
+ // We may run into multiple chains that merge into a single chain. We mark the
+ // stores that we vectorized so that we don't visit the same store twice.
+ ValueSet VectorizedStores;
+ bool Changed = false;
+
+ // Do a quadratic search on all of the given stores and find
+ // all of the pairs of loads that follow each other.
+ for (unsigned i = 0, e = Stores.size(); i < e; ++i)
+ for (unsigned j = 0; j < e; ++j) {
+ if (i == j)
+ continue;
+
+ if (isConsecutiveAccess(Stores[i], Stores[j])) {
+ Tails.insert(Stores[j]);
+ Heads.insert(Stores[i]);
+ ConsecutiveChain[Stores[i]] = Stores[j];
+ }
+ }
+
+ // For stores that start but don't end a link in the chain:
+ for (SetVector<Value *>::iterator it = Heads.begin(), e = Heads.end();
+ it != e; ++it) {
+ if (Tails.count(*it))
+ continue;
+
+ // We found a store instr that starts a chain. Now follow the chain and try
+ // to vectorize it.
+ ValueList Operands;
+ Value *I = *it;
+ // Collect the chain into a list.
+ while (Tails.count(I) || Heads.count(I)) {
+ if (VectorizedStores.count(I))
+ break;
+ Operands.push_back(I);
+ // Move to the next value in the chain.
+ I = ConsecutiveChain[I];
+ }
+
+ bool Vectorized = vectorizeStoreChain(Operands, costThreshold);
+
+ // Mark the vectorized stores so that we don't vectorize them again.
+ if (Vectorized)
+ VectorizedStores.insert(Operands.begin(), Operands.end());
+ Changed |= Vectorized;
+ }
+
+ return Changed;
+}
+
+Value *FuncSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
+ Value *Vec = UndefValue::get(Ty);
+ // Generate the 'InsertElement' instruction.
+ for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
+ Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
+ if (Instruction *I = dyn_cast<Instruction>(Vec))
+ GatherSeq.insert(I);
+ }
+
+ VectorizedValues[VL[0]] = Vec;
+ return Vec;
+}
+
+Value *FuncSLP::vectorizeTree_rec(ArrayRef<Value *> VL) {
+ BuilderLocGuard Guard(Builder);
+
+ Type *ScalarTy = VL[0]->getType();
+ if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
+ ScalarTy = SI->getValueOperand()->getType();
+ VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
+
+ if (needToGatherAny(VL))
+ return Gather(VL, VecTy);
+
+ if (VectorizedValues.count(VL[0])) {
+ DEBUG(dbgs() << "SLP: Diamond merged at depth.\n");
+ return VectorizedValues[VL[0]];
+ }
+
+ Instruction *VL0 = cast<Instruction>(VL[0]);
+ unsigned Opcode = VL0->getOpcode();
+ assert(Opcode == getSameOpcode(VL) && "Invalid opcode");
+
+ switch (Opcode) {
+ case Instruction::ExtractElement: {
+ if (CanReuseExtract(VL, VL.size(), VecTy))
+ return VL0->getOperand(0);
+ return Gather(VL, VecTy);
+ }
+ case Instruction::ZExt:
+ case Instruction::SExt:
+ case Instruction::FPToUI:
+ case Instruction::FPToSI:
+ case Instruction::FPExt:
+ case Instruction::PtrToInt:
+ case Instruction::IntToPtr:
+ case Instruction::SIToFP:
+ case Instruction::UIToFP:
+ case Instruction::Trunc:
+ case Instruction::FPTrunc:
+ case Instruction::BitCast: {
+ ValueList INVL;
+ for (int i = 0, e = VL.size(); i < e; ++i)
+ INVL.push_back(cast<Instruction>(VL[i])->getOperand(0));
+
+ Builder.SetInsertPoint(getLastInstruction(VL));
+ Value *InVec = vectorizeTree_rec(INVL);
+ CastInst *CI = dyn_cast<CastInst>(VL0);
+ Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
+ VectorizedValues[VL0] = V;
+ return V;
+ }
+ case Instruction::FCmp:
+ case Instruction::ICmp: {
+ // Check that all of the compares have the same predicate.
+ CmpInst::Predicate P0 = dyn_cast<CmpInst>(VL0)->getPredicate();
+ for (unsigned i = 1, e = VL.size(); i < e; ++i) {
+ CmpInst *Cmp = cast<CmpInst>(VL[i]);
+ if (Cmp->getPredicate() != P0)
+ return Gather(VL, VecTy);
+ }
+
+ ValueList LHSV, RHSV;
+ for (int i = 0, e = VL.size(); i < e; ++i) {
+ LHSV.push_back(cast<Instruction>(VL[i])->getOperand(0));
+ RHSV.push_back(cast<Instruction>(VL[i])->getOperand(1));
+ }
+
+ Builder.SetInsertPoint(getLastInstruction(VL));
+ Value *L = vectorizeTree_rec(LHSV);
+ Value *R = vectorizeTree_rec(RHSV);
+ Value *V;
+
+ if (Opcode == Instruction::FCmp)
+ V = Builder.CreateFCmp(P0, L, R);
+ else
+ V = Builder.CreateICmp(P0, L, R);
+
+ VectorizedValues[VL0] = V;
+ return V;
+ }
+ case Instruction::Select: {
+ ValueList TrueVec, FalseVec, CondVec;
+ for (int i = 0, e = VL.size(); i < e; ++i) {
+ CondVec.push_back(cast<Instruction>(VL[i])->getOperand(0));
+ TrueVec.push_back(cast<Instruction>(VL[i])->getOperand(1));
+ FalseVec.push_back(cast<Instruction>(VL[i])->getOperand(2));
+ }
+
+ Builder.SetInsertPoint(getLastInstruction(VL));
+ Value *True = vectorizeTree_rec(TrueVec);
+ Value *False = vectorizeTree_rec(FalseVec);
+ Value *Cond = vectorizeTree_rec(CondVec);
+ Value *V = Builder.CreateSelect(Cond, True, False);
+ VectorizedValues[VL0] = V;
+ return V;
+ }
+ case Instruction::Add:
+ case Instruction::FAdd:
+ case Instruction::Sub:
+ case Instruction::FSub:
+ case Instruction::Mul:
+ case Instruction::FMul:
+ case Instruction::UDiv:
+ case Instruction::SDiv:
+ case Instruction::FDiv:
+ case Instruction::URem:
+ case Instruction::SRem:
+ case Instruction::FRem:
+ case Instruction::Shl:
+ case Instruction::LShr:
+ case Instruction::AShr:
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor: {
+ ValueList LHSVL, RHSVL;
+ for (int i = 0, e = VL.size(); i < e; ++i) {
+ LHSVL.push_back(cast<Instruction>(VL[i])->getOperand(0));
+ RHSVL.push_back(cast<Instruction>(VL[i])->getOperand(1));
+ }
+
+ Builder.SetInsertPoint(getLastInstruction(VL));
+ Value *LHS = vectorizeTree_rec(LHSVL);
+ Value *RHS = vectorizeTree_rec(RHSVL);
+
+ if (LHS == RHS) {
+ assert((VL0->getOperand(0) == VL0->getOperand(1)) && "Invalid order");
+ }
+
+ BinaryOperator *BinOp = cast<BinaryOperator>(VL0);
+ Value *V = Builder.CreateBinOp(BinOp->getOpcode(), LHS, RHS);
+ VectorizedValues[VL0] = V;
+ return V;
+ }
+ case Instruction::Load: {
+ // Check if all of the loads are consecutive.
+ for (unsigned i = 1, e = VL.size(); i < e; ++i)
+ if (!isConsecutiveAccess(VL[i - 1], VL[i]))
+ return Gather(VL, VecTy);
+
+ // Loads are inserted at the head of the tree because we don't want to
+ // sink them all the way down past store instructions.
+ Builder.SetInsertPoint(getLastInstruction(VL));
+ LoadInst *LI = cast<LoadInst>(VL0);
+ Value *VecPtr =
+ Builder.CreateBitCast(LI->getPointerOperand(), VecTy->getPointerTo());
+ unsigned Alignment = LI->getAlignment();
+ LI = Builder.CreateLoad(VecPtr);
+ LI->setAlignment(Alignment);
+
+ VectorizedValues[VL0] = LI;
+ return LI;
+ }
+ case Instruction::Store: {
+ StoreInst *SI = cast<StoreInst>(VL0);
+ unsigned Alignment = SI->getAlignment();
+
+ ValueList ValueOp;
+ for (int i = 0, e = VL.size(); i < e; ++i)
+ ValueOp.push_back(cast<StoreInst>(VL[i])->getValueOperand());
+
+ Value *VecValue = vectorizeTree_rec(ValueOp);
+
+ Builder.SetInsertPoint(getLastInstruction(VL));
+ Value *VecPtr =
+ Builder.CreateBitCast(SI->getPointerOperand(), VecTy->getPointerTo());
+ Builder.CreateStore(VecValue, VecPtr)->setAlignment(Alignment);
+
+ for (int i = 0, e = VL.size(); i < e; ++i)
+ cast<Instruction>(VL[i])->eraseFromParent();
+ return 0;
+ }
+ default:
+ return Gather(VL, VecTy);
+ }
+}
+
+Value *FuncSLP::vectorizeTree(ArrayRef<Value *> VL) {
+ Builder.SetInsertPoint(getLastInstruction(VL));
+ Value *V = vectorizeTree_rec(VL);
+
+ // We moved some instructions around. We have to number them again
+ // before we can do any analysis.
+ MustGather.clear();
+ VectorizedValues.clear();
+ MemBarrierIgnoreList.clear();
+ for (Function::iterator it = F->begin(), e = F->end(); it != e; ++it)
+ BlocksNumbers[it].forget();
+ return V;
+}
+
+Value *FuncSLP::vectorizeArith(ArrayRef<Value *> Operands) {
+ Value *Vec = vectorizeTree(Operands);
+ // After vectorizing the operands we need to generate extractelement
+ // instructions and replace all of the uses of the scalar values with
+ // the values that we extracted from the vectorized tree.
+ for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
+ Value *S = Builder.CreateExtractElement(Vec, Builder.getInt32(i));
+ Operands[i]->replaceAllUsesWith(S);
+ }
+
+ return Vec;
+}
+
+void FuncSLP::hoistGatherSequence() {
+ for (SetVector<Instruction *>::iterator it = GatherSeq.begin(),
+ e = GatherSeq.end();
+ it != e; ++it) {
+ InsertElementInst *Insert = dyn_cast_or_null<InsertElementInst>(*it);
+
+ // The InsertElement sequence can be simplified into a constant.
+ // Also Ignore NULL pointers because they are only here to separate
+ // sequences.
+ if (!Insert)
+ continue;
+
+ BasicBlock *BB = Insert->getParent();
+
+ // Check if this block is inside a loop.
+ Loop *L = LI->getLoopFor(BB);
+ if (!L)
+ return;
+
+ // Check if it has a preheader.
+ BasicBlock *PreHeader = L->getLoopPreheader();
+ if (!PreHeader)
+ return;
+
+ // If the vector or the element that we insert into it are
+ // instructions that are defined in this basic block then we can't
+ // hoist this instruction.
+ Instruction *CurrVec = dyn_cast<Instruction>(Insert->getOperand(0));
+ Instruction *NewElem = dyn_cast<Instruction>(Insert->getOperand(1));
+ if (CurrVec && L->contains(CurrVec))
+ continue;
+ if (NewElem && L->contains(NewElem))
+ continue;
+
+ // Mark the insertion point for the block.
+ Instruction *Location = PreHeader->getTerminator();
+ // We can hoist this instruction. Move it to the pre-header.
+ Insert->moveBefore(Location);
+ }
+}
+
/// The SLPVectorizer Pass.
struct SLPVectorizer : public FunctionPass {
- typedef MapVector<Value *, BoUpSLP::StoreList> StoreListMap;
+ typedef SmallVector<StoreInst *, 8> StoreList;
+ typedef MapVector<Value *, StoreList> StoreListMap;
/// Pass identification, replacement for typeid
static char ID;
DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n");
+ // Use the bollom up slp vectorizer to construct chains that start with
+ // he store instructions.
+ FuncSLP R(&F, SE, DL, TTI, AA, LI);
+
for (Function::iterator it = F.begin(), e = F.end(); it != e; ++it) {
BasicBlock *BB = it;
- bool BBChanged = false;
-
- // Use the bollom up slp vectorizer to construct chains that start with
- // he store instructions.
- BoUpSLP R(BB, SE, DL, TTI, AA, LI->getLoopFor(BB));
// Vectorize trees that end at reductions.
- BBChanged |= vectorizeChainsInBlock(BB, R);
+ Changed |= vectorizeChainsInBlock(BB, R);
// Vectorize trees that end at stores.
if (unsigned count = collectStores(BB, R)) {
(void)count;
DEBUG(dbgs() << "SLP: Found " << count << " stores to vectorize.\n");
- BBChanged |= vectorizeStoreChains(R);
+ Changed |= vectorizeStoreChains(R);
}
-
- // Try to hoist some of the scalarization code to the preheader.
- if (BBChanged) {
- hoistGatherSequence(LI, BB, R);
- Changed |= vectorizeUsingGatherHints(R.getGatherSeqInstructions());
- }
-
- Changed |= BBChanged;
}
if (Changed) {
+ R.hoistGatherSequence();
DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
DEBUG(verifyFunction(F));
}
/// object. We sort the stores to their base objects to reduce the cost of the
/// quadratic search on the stores. TODO: We can further reduce this cost
/// if we flush the chain creation every time we run into a memory barrier.
- unsigned collectStores(BasicBlock *BB, BoUpSLP &R);
+ unsigned collectStores(BasicBlock *BB, FuncSLP &R);
/// \brief Try to vectorize a chain that starts at two arithmetic instrs.
- bool tryToVectorizePair(Value *A, Value *B, BoUpSLP &R);
+ bool tryToVectorizePair(Value *A, Value *B, FuncSLP &R);
/// \brief Try to vectorize a list of operands. If \p NeedExtracts is true
/// then we calculate the cost of extracting the scalars from the vector.
/// \returns true if a value was vectorized.
- bool tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R, bool NeedExtracts);
+ bool tryToVectorizeList(ArrayRef<Value *> VL, FuncSLP &R, bool NeedExtracts);
/// \brief Try to vectorize a chain that may start at the operands of \V;
- bool tryToVectorize(BinaryOperator *V, BoUpSLP &R);
+ bool tryToVectorize(BinaryOperator *V, FuncSLP &R);
/// \brief Vectorize the stores that were collected in StoreRefs.
- bool vectorizeStoreChains(BoUpSLP &R);
-
- /// \brief Try to hoist gather sequences outside of the loop in cases where
- /// all of the sources are loop invariant.
- void hoistGatherSequence(LoopInfo *LI, BasicBlock *BB, BoUpSLP &R);
-
- /// \brief Try to vectorize additional sequences in different basic blocks
- /// based on values that we gathered in previous blocks. The list \p Gathers
- /// holds the gather InsertElement instructions that were generated during
- /// vectorization.
- /// \returns True if some code was vectorized.
- bool vectorizeUsingGatherHints(BoUpSLP::InstrList &Gathers);
+ bool vectorizeStoreChains(FuncSLP &R);
/// \brief Scan the basic block and look for patterns that are likely to start
/// a vectorization chain.
- bool vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R);
+ bool vectorizeChainsInBlock(BasicBlock *BB, FuncSLP &R);
private:
StoreListMap StoreRefs;
};
-unsigned SLPVectorizer::collectStores(BasicBlock *BB, BoUpSLP &R) {
+unsigned SLPVectorizer::collectStores(BasicBlock *BB, FuncSLP &R) {
unsigned count = 0;
StoreRefs.clear();
for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
return count;
}
-bool SLPVectorizer::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
+bool SLPVectorizer::tryToVectorizePair(Value *A, Value *B, FuncSLP &R) {
if (!A || !B)
return false;
Value *VL[] = { A, B };
return tryToVectorizeList(VL, R, true);
}
-bool SLPVectorizer::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
+bool SLPVectorizer::tryToVectorizeList(ArrayRef<Value *> VL, FuncSLP &R,
bool NeedExtracts) {
if (VL.size() < 2)
return false;
}
int Cost = R.getTreeCost(VL);
- int ExtrCost = NeedExtracts ? R.getScalarizationCost(VL) : 0;
+ if (Cost == FuncSLP::MAX_COST)
+ return false;
+
+ int ExtrCost = NeedExtracts ? R.getGatherCost(VL) : 0;
DEBUG(dbgs() << "SLP: Cost of pair:" << Cost
<< " Cost of extract:" << ExtrCost << ".\n");
if ((Cost + ExtrCost) >= -SLPCostThreshold)
return true;
}
-bool SLPVectorizer::tryToVectorize(BinaryOperator *V, BoUpSLP &R) {
+bool SLPVectorizer::tryToVectorize(BinaryOperator *V, FuncSLP &R) {
if (!V)
return false;
-
+
// Try to vectorize V.
if (tryToVectorizePair(V->getOperand(0), V->getOperand(1), R))
return true;
return 0;
}
-bool SLPVectorizer::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
+bool SLPVectorizer::vectorizeChainsInBlock(BasicBlock *BB, FuncSLP &R) {
bool Changed = false;
for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
if (isa<DbgInfoIntrinsic>(it))
Value *Inst = BI->getOperand(0);
if (Inst == P)
Inst = BI->getOperand(1);
-
+
Changed |= tryToVectorize(dyn_cast<BinaryOperator>(Inst), R);
continue;
}
return Changed;
}
-bool SLPVectorizer::vectorizeStoreChains(BoUpSLP &R) {
+bool SLPVectorizer::vectorizeStoreChains(FuncSLP &R) {
bool Changed = false;
// Attempt to sort and vectorize each of the store-groups.
for (StoreListMap::iterator it = StoreRefs.begin(), e = StoreRefs.end();
return Changed;
}
-bool SLPVectorizer::vectorizeUsingGatherHints(BoUpSLP::InstrList &Gathers) {
- SmallVector<Value *, 4> Seq;
- bool Changed = false;
- for (int i = 0, e = Gathers.size(); i < e; ++i) {
- InsertElementInst *IEI = dyn_cast_or_null<InsertElementInst>(Gathers[i]);
-
- if (IEI) {
- if (Instruction *I = dyn_cast<Instruction>(IEI->getOperand(1)))
- Seq.push_back(I);
- } else {
-
- if (!Seq.size())
- continue;
-
- Instruction *I = cast<Instruction>(Seq[0]);
- BasicBlock *BB = I->getParent();
-
- DEBUG(dbgs() << "SLP: Inspecting a gather list of size " << Seq.size()
- << " in " << BB->getName() << ".\n");
-
- // Check if the gathered values have multiple uses. If they only have one
- // user then we know that the insert/extract pair will go away.
- bool HasMultipleUsers = false;
- for (int i = 0; e = Seq.size(), i < e; ++i) {
- if (!Seq[i]->hasOneUse()) {
- HasMultipleUsers = true;
- break;
- }
- }
-
- BoUpSLP BO(BB, SE, DL, TTI, AA, LI->getLoopFor(BB));
-
- if (tryToVectorizeList(Seq, BO, HasMultipleUsers)) {
- DEBUG(dbgs() << "SLP: Vectorized a gather list of len " << Seq.size()
- << " in " << BB->getName() << ".\n");
- Changed = true;
- }
-
- Seq.clear();
- }
- }
-
- return Changed;
-}
-
-void SLPVectorizer::hoistGatherSequence(LoopInfo *LI, BasicBlock *BB,
- BoUpSLP &R) {
- // Check if this block is inside a loop.
- Loop *L = LI->getLoopFor(BB);
- if (!L)
- return;
-
- // Check if it has a preheader.
- BasicBlock *PreHeader = L->getLoopPreheader();
- if (!PreHeader)
- return;
-
- // Mark the insertion point for the block.
- Instruction *Location = PreHeader->getTerminator();
-
- BoUpSLP::InstrList &Gathers = R.getGatherSeqInstructions();
- for (BoUpSLP::InstrList::iterator it = Gathers.begin(), e = Gathers.end();
- it != e; ++it) {
- InsertElementInst *Insert = dyn_cast_or_null<InsertElementInst>(*it);
-
- // The InsertElement sequence can be simplified into a constant.
- // Also Ignore NULL pointers because they are only here to separate
- // sequences.
- if (!Insert)
- continue;
-
- // If the vector or the element that we insert into it are
- // instructions that are defined in this basic block then we can't
- // hoist this instruction.
- Instruction *CurrVec = dyn_cast<Instruction>(Insert->getOperand(0));
- Instruction *NewElem = dyn_cast<Instruction>(Insert->getOperand(1));
- if (CurrVec && L->contains(CurrVec))
- continue;
- if (NewElem && L->contains(NewElem))
- continue;
-
- // We can hoist this instruction. Move it to the pre-header.
- Insert->moveBefore(Location);
- }
-}
-
} // end anonymous namespace
char SLPVectorizer::ID = 0;
+++ /dev/null
-//===- VecUtils.cpp --- Vectorization Utilities ---------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-#define DEBUG_TYPE "SLP"
-
-#include "VecUtils.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Analysis/ScalarEvolution.h"
-#include "llvm/Analysis/ScalarEvolutionExpressions.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/Analysis/Verifier.h"
-#include "llvm/Analysis/LoopInfo.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Type.h"
-#include "llvm/IR/Value.h"
-#include "llvm/Pass.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetLibraryInfo.h"
-#include "llvm/Transforms/Scalar.h"
-#include "llvm/Transforms/Utils/Local.h"
-#include <algorithm>
-#include <map>
-
-using namespace llvm;
-
-static const unsigned MinVecRegSize = 128;
-
-static const unsigned RecursionMaxDepth = 6;
-
-namespace llvm {
-
-BoUpSLP::BoUpSLP(BasicBlock *Bb, ScalarEvolution *S, DataLayout *Dl,
- TargetTransformInfo *Tti, AliasAnalysis *Aa, Loop *Lp)
- : Builder(S->getContext()), BB(Bb), SE(S), DL(Dl), TTI(Tti), AA(Aa), L(Lp) {
- numberInstructions();
-}
-
-void BoUpSLP::numberInstructions() {
- int Loc = 0;
- InstrIdx.clear();
- InstrVec.clear();
- // Number the instructions in the block.
- for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
- InstrIdx[it] = Loc++;
- InstrVec.push_back(it);
- assert(InstrVec[InstrIdx[it]] == it && "Invalid allocation");
- }
-}
-
-Value *BoUpSLP::getPointerOperand(Value *I) {
- if (LoadInst *LI = dyn_cast<LoadInst>(I))
- return LI->getPointerOperand();
- if (StoreInst *SI = dyn_cast<StoreInst>(I))
- return SI->getPointerOperand();
- return 0;
-}
-
-unsigned BoUpSLP::getAddressSpaceOperand(Value *I) {
- if (LoadInst *L = dyn_cast<LoadInst>(I))
- return L->getPointerAddressSpace();
- if (StoreInst *S = dyn_cast<StoreInst>(I))
- return S->getPointerAddressSpace();
- return -1;
-}
-
-bool BoUpSLP::isConsecutiveAccess(Value *A, Value *B) {
- Value *PtrA = getPointerOperand(A);
- Value *PtrB = getPointerOperand(B);
- unsigned ASA = getAddressSpaceOperand(A);
- unsigned ASB = getAddressSpaceOperand(B);
-
- // Check that the address spaces match and that the pointers are valid.
- if (!PtrA || !PtrB || (ASA != ASB))
- return false;
-
- // Check that A and B are of the same type.
- if (PtrA->getType() != PtrB->getType())
- return false;
-
- // Calculate the distance.
- const SCEV *PtrSCEVA = SE->getSCEV(PtrA);
- const SCEV *PtrSCEVB = SE->getSCEV(PtrB);
- const SCEV *OffsetSCEV = SE->getMinusSCEV(PtrSCEVA, PtrSCEVB);
- const SCEVConstant *ConstOffSCEV = dyn_cast<SCEVConstant>(OffsetSCEV);
-
- // Non constant distance.
- if (!ConstOffSCEV)
- return false;
-
- int64_t Offset = ConstOffSCEV->getValue()->getSExtValue();
- Type *Ty = cast<PointerType>(PtrA->getType())->getElementType();
- // The Instructions are connsecutive if the size of the first load/store is
- // the same as the offset.
- int64_t Sz = DL->getTypeStoreSize(Ty);
- return ((-Offset) == Sz);
-}
-
-bool BoUpSLP::vectorizeStoreChain(ArrayRef<Value *> Chain, int CostThreshold) {
- unsigned ChainLen = Chain.size();
- DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLen
- << "\n");
- Type *StoreTy = cast<StoreInst>(Chain[0])->getValueOperand()->getType();
- unsigned Sz = DL->getTypeSizeInBits(StoreTy);
- unsigned VF = MinVecRegSize / Sz;
-
- if (!isPowerOf2_32(Sz) || VF < 2)
- return false;
-
- bool Changed = false;
- // Look for profitable vectorizable trees at all offsets, starting at zero.
- for (unsigned i = 0, e = ChainLen; i < e; ++i) {
- if (i + VF > e)
- break;
- DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << i
- << "\n");
- ArrayRef<Value *> Operands = Chain.slice(i, VF);
-
- int Cost = getTreeCost(Operands);
- DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n");
- if (Cost < CostThreshold) {
- DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
- Builder.SetInsertPoint(getInsertionPoint(getLastIndex(Operands, VF)));
- vectorizeTree(Operands, VF);
- i += VF - 1;
- Changed = true;
- }
- }
-
- if (Changed || ChainLen > VF)
- return Changed;
-
- // Handle short chains. This helps us catch types such as <3 x float> that
- // are smaller than vector size.
- int Cost = getTreeCost(Chain);
- if (Cost < CostThreshold) {
- DEBUG(dbgs() << "SLP: Found store chain cost = " << Cost
- << " for size = " << ChainLen << "\n");
- Builder.SetInsertPoint(getInsertionPoint(getLastIndex(Chain, ChainLen)));
- vectorizeTree(Chain, ChainLen);
- return true;
- }
-
- return false;
-}
-
-bool BoUpSLP::vectorizeStores(ArrayRef<StoreInst *> Stores, int costThreshold) {
- SetVector<Value *> Heads, Tails;
- SmallDenseMap<Value *, Value *> ConsecutiveChain;
-
- // We may run into multiple chains that merge into a single chain. We mark the
- // stores that we vectorized so that we don't visit the same store twice.
- ValueSet VectorizedStores;
- bool Changed = false;
-
- // Do a quadratic search on all of the given stores and find
- // all of the pairs of loads that follow each other.
- for (unsigned i = 0, e = Stores.size(); i < e; ++i)
- for (unsigned j = 0; j < e; ++j) {
- if (i == j)
- continue;
-
- if (isConsecutiveAccess(Stores[i], Stores[j])) {
- Tails.insert(Stores[j]);
- Heads.insert(Stores[i]);
- ConsecutiveChain[Stores[i]] = Stores[j];
- }
- }
-
- // For stores that start but don't end a link in the chain:
- for (SetVector<Value *>::iterator it = Heads.begin(), e = Heads.end();
- it != e; ++it) {
- if (Tails.count(*it))
- continue;
-
- // We found a store instr that starts a chain. Now follow the chain and try
- // to vectorize it.
- ValueList Operands;
- Value *I = *it;
- // Collect the chain into a list.
- while (Tails.count(I) || Heads.count(I)) {
- if (VectorizedStores.count(I))
- break;
- Operands.push_back(I);
- // Move to the next value in the chain.
- I = ConsecutiveChain[I];
- }
-
- bool Vectorized = vectorizeStoreChain(Operands, costThreshold);
-
- // Mark the vectorized stores so that we don't vectorize them again.
- if (Vectorized)
- VectorizedStores.insert(Operands.begin(), Operands.end());
- Changed |= Vectorized;
- }
-
- return Changed;
-}
-
-int BoUpSLP::getScalarizationCost(ArrayRef<Value *> VL) {
- // Find the type of the operands in VL.
- Type *ScalarTy = VL[0]->getType();
- if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
- ScalarTy = SI->getValueOperand()->getType();
- VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
- // Find the cost of inserting/extracting values from the vector.
- return getScalarizationCost(VecTy);
-}
-
-int BoUpSLP::getScalarizationCost(Type *Ty) {
- int Cost = 0;
- for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
- Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
- return Cost;
-}
-
-AliasAnalysis::Location BoUpSLP::getLocation(Instruction *I) {
- if (StoreInst *SI = dyn_cast<StoreInst>(I))
- return AA->getLocation(SI);
- if (LoadInst *LI = dyn_cast<LoadInst>(I))
- return AA->getLocation(LI);
- return AliasAnalysis::Location();
-}
-
-Value *BoUpSLP::isUnsafeToSink(Instruction *Src, Instruction *Dst) {
- assert(Src->getParent() == Dst->getParent() && "Not the same BB");
- BasicBlock::iterator I = Src, E = Dst;
- /// Scan all of the instruction from SRC to DST and check if
- /// the source may alias.
- for (++I; I != E; ++I) {
- // Ignore store instructions that are marked as 'ignore'.
- if (MemBarrierIgnoreList.count(I))
- continue;
- if (Src->mayWriteToMemory()) /* Write */ {
- if (!I->mayReadOrWriteMemory())
- continue;
- } else /* Read */ {
- if (!I->mayWriteToMemory())
- continue;
- }
- AliasAnalysis::Location A = getLocation(&*I);
- AliasAnalysis::Location B = getLocation(Src);
-
- if (!A.Ptr || !B.Ptr || AA->alias(A, B))
- return I;
- }
- return 0;
-}
-
-Value *BoUpSLP::vectorizeArith(ArrayRef<Value *> Operands) {
- int LastIdx = getLastIndex(Operands, Operands.size());
- Instruction *Loc = getInsertionPoint(LastIdx);
- Builder.SetInsertPoint(Loc);
-
- assert(getFirstUserIndex(Operands, Operands.size()) > LastIdx &&
- "Vectorizing with in-tree users");
-
- Value *Vec = vectorizeTree(Operands, Operands.size());
- // After vectorizing the operands we need to generate extractelement
- // instructions and replace all of the uses of the scalar values with
- // the values that we extracted from the vectorized tree.
- for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
- Value *S = Builder.CreateExtractElement(Vec, Builder.getInt32(i));
- Operands[i]->replaceAllUsesWith(S);
- }
-
- return Vec;
-}
-
-int BoUpSLP::getTreeCost(ArrayRef<Value *> VL) {
- // Get rid of the list of stores that were removed, and from the
- // lists of instructions with multiple users.
- MemBarrierIgnoreList.clear();
- LaneMap.clear();
- MultiUserVals.clear();
- MustScalarize.clear();
- MustExtract.clear();
-
- // Find the location of the last root.
- int LastRootIndex = getLastIndex(VL, VL.size());
- int FirstUserIndex = getFirstUserIndex(VL, VL.size());
-
- // Don't vectorize if there are users of the tree roots inside the tree
- // itself.
- if (LastRootIndex > FirstUserIndex)
- return max_cost;
-
- // Scan the tree and find which value is used by which lane, and which values
- // must be scalarized.
- getTreeUses_rec(VL, 0);
-
- // Check that instructions with multiple users can be vectorized. Mark unsafe
- // instructions.
- for (SetVector<Value *>::iterator it = MultiUserVals.begin(),
- e = MultiUserVals.end();
- it != e; ++it) {
- // Check that all of the users of this instr are within the tree
- // and that they are all from the same lane.
- int Lane = -1;
- for (Value::use_iterator I = (*it)->use_begin(), E = (*it)->use_end();
- I != E; ++I) {
- if (LaneMap.find(*I) == LaneMap.end()) {
- DEBUG(dbgs() << "SLP: Instr " << **it << " has multiple users.\n");
-
- // We don't have an ordering problem if the user is not in this basic
- // block.
- Instruction *Inst = cast<Instruction>(*I);
- if (Inst->getParent() != BB) {
- MustExtract.insert(*it);
- continue;
- }
-
- // We don't have an ordering problem if the user is after the last root.
- int Idx = InstrIdx[Inst];
- if (Idx < LastRootIndex) {
- MustScalarize.insert(*it);
- DEBUG(dbgs() << "SLP: Adding to MustScalarize "
- "because of an unsafe out of tree usage.\n");
- break;
- }
-
- DEBUG(dbgs() << "SLP: Adding to MustExtract "
- "because of a safe out of tree usage.\n");
- MustExtract.insert(*it);
- continue;
- }
- if (Lane == -1)
- Lane = LaneMap[*I];
- if (Lane != LaneMap[*I]) {
- MustScalarize.insert(*it);
- DEBUG(dbgs() << "SLP: Adding " << **it
- << " to MustScalarize because multiple lane use it: "
- << Lane << " and " << LaneMap[*I] << ".\n");
- break;
- }
- }
- }
-
- // Now calculate the cost of vectorizing the tree.
- return getTreeCost_rec(VL, 0);
-}
-
-static bool CanReuseExtract(ArrayRef<Value *> VL, unsigned VF,
- VectorType *VecTy) {
- // Check if all of the extracts come from the same vector and from the
- // correct offset.
- Value *VL0 = VL[0];
- ExtractElementInst *E0 = cast<ExtractElementInst>(VL0);
- Value *Vec = E0->getOperand(0);
-
- // We have to extract from the same vector type.
- if (Vec->getType() != VecTy)
- return false;
-
- // Check that all of the indices extract from the correct offset.
- ConstantInt *CI = dyn_cast<ConstantInt>(E0->getOperand(1));
- if (!CI || CI->getZExtValue())
- return false;
-
- for (unsigned i = 1, e = VF; i < e; ++i) {
- ExtractElementInst *E = cast<ExtractElementInst>(VL[i]);
- ConstantInt *CI = dyn_cast<ConstantInt>(E->getOperand(1));
-
- if (!CI || CI->getZExtValue() != i || E->getOperand(0) != Vec)
- return false;
- }
-
- return true;
-}
-
-void BoUpSLP::getTreeUses_rec(ArrayRef<Value *> VL, unsigned Depth) {
- if (Depth == RecursionMaxDepth)
- return;
-
- // Don't handle vectors.
- if (VL[0]->getType()->isVectorTy())
- return;
-
- if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
- if (SI->getValueOperand()->getType()->isVectorTy())
- return;
-
- // Check if all of the operands are constants.
- bool AllConst = true;
- bool AllSameScalar = true;
- for (unsigned i = 0, e = VL.size(); i < e; ++i) {
- AllConst &= isa<Constant>(VL[i]);
- AllSameScalar &= (VL[0] == VL[i]);
- Instruction *I = dyn_cast<Instruction>(VL[i]);
- // If one of the instructions is out of this BB, we need to scalarize all.
- if (I && I->getParent() != BB)
- return;
- }
-
- // If all of the operands are identical or constant we have a simple solution.
- if (AllConst || AllSameScalar)
- return;
-
- // Scalarize unknown structures.
- Instruction *VL0 = dyn_cast<Instruction>(VL[0]);
- if (!VL0)
- return;
-
- unsigned Opcode = VL0->getOpcode();
- for (unsigned i = 0, e = VL.size(); i < e; ++i) {
- Instruction *I = dyn_cast<Instruction>(VL[i]);
- // If not all of the instructions are identical then we have to scalarize.
- if (!I || Opcode != I->getOpcode())
- return;
- }
-
- for (int i = 0, e = VL.size(); i < e; ++i) {
- // Check that the instruction is only used within
- // one lane.
- if (LaneMap.count(VL[i]) && LaneMap[VL[i]] != i)
- return;
- // Make this instruction as 'seen' and remember the lane.
- LaneMap[VL[i]] = i;
- }
-
- // Mark instructions with multiple users.
- for (unsigned i = 0, e = VL.size(); i < e; ++i) {
- Instruction *I = dyn_cast<Instruction>(VL[i]);
- // Remember to check if all of the users of this instr are vectorized
- // within our tree. At depth zero we have no local users, only external
- // users that we don't care about.
- if (Depth && I && I->getNumUses() > 1) {
- DEBUG(dbgs() << "SLP: Adding to MultiUserVals "
- "because it has multiple users:" << *I << " \n");
- MultiUserVals.insert(I);
- }
- }
-
- switch (Opcode) {
- case Instruction::ExtractElement: {
- VectorType *VecTy = VectorType::get(VL[0]->getType(), VL.size());
- // No need to follow ExtractElements that are going to be optimized away.
- if (CanReuseExtract(VL, VL.size(), VecTy))
- return;
- // Fall through.
- }
- case Instruction::ZExt:
- case Instruction::SExt:
- case Instruction::FPToUI:
- case Instruction::FPToSI:
- case Instruction::FPExt:
- case Instruction::PtrToInt:
- case Instruction::IntToPtr:
- case Instruction::SIToFP:
- case Instruction::UIToFP:
- case Instruction::Trunc:
- case Instruction::FPTrunc:
- case Instruction::BitCast:
- case Instruction::Select:
- case Instruction::ICmp:
- case Instruction::FCmp:
- case Instruction::Add:
- case Instruction::FAdd:
- case Instruction::Sub:
- case Instruction::FSub:
- case Instruction::Mul:
- case Instruction::FMul:
- case Instruction::UDiv:
- case Instruction::SDiv:
- case Instruction::FDiv:
- case Instruction::URem:
- case Instruction::SRem:
- case Instruction::FRem:
- case Instruction::Shl:
- case Instruction::LShr:
- case Instruction::AShr:
- case Instruction::And:
- case Instruction::Or:
- case Instruction::Xor: {
- for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
- ValueList Operands;
- // Prepare the operand vector.
- for (unsigned j = 0; j < VL.size(); ++j)
- Operands.push_back(cast<Instruction>(VL[j])->getOperand(i));
-
- getTreeUses_rec(Operands, Depth + 1);
- }
- return;
- }
- case Instruction::Store: {
- ValueList Operands;
- for (unsigned j = 0; j < VL.size(); ++j)
- Operands.push_back(cast<Instruction>(VL[j])->getOperand(0));
- getTreeUses_rec(Operands, Depth + 1);
- return;
- }
- default:
- return;
- }
-}
-
-int BoUpSLP::getTreeCost_rec(ArrayRef<Value *> VL, unsigned Depth) {
- Type *ScalarTy = VL[0]->getType();
-
- if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
- ScalarTy = SI->getValueOperand()->getType();
-
- /// Don't mess with vectors.
- if (ScalarTy->isVectorTy())
- return max_cost;
-
- VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
-
- if (Depth == RecursionMaxDepth)
- return getScalarizationCost(VecTy);
-
- // Check if all of the operands are constants.
- bool AllConst = true;
- bool AllSameScalar = true;
- bool MustScalarizeFlag = false;
- for (unsigned i = 0, e = VL.size(); i < e; ++i) {
- AllConst &= isa<Constant>(VL[i]);
- AllSameScalar &= (VL[0] == VL[i]);
- // Must have a single use.
- Instruction *I = dyn_cast<Instruction>(VL[i]);
- MustScalarizeFlag |= MustScalarize.count(VL[i]);
- // This instruction is outside the basic block.
- if (I && I->getParent() != BB)
- return getScalarizationCost(VecTy);
- }
-
- // Is this a simple vector constant.
- if (AllConst)
- return 0;
-
- // If all of the operands are identical we can broadcast them.
- Instruction *VL0 = dyn_cast<Instruction>(VL[0]);
- if (AllSameScalar) {
- // If we are in a loop, and this is not an instruction (e.g. constant or
- // argument) or the instruction is defined outside the loop then assume
- // that the cost is zero.
- if (L && (!VL0 || !L->contains(VL0)))
- return 0;
-
- // We need to broadcast the scalar.
- return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
- }
-
- // If this is not a constant, or a scalar from outside the loop then we
- // need to scalarize it.
- if (MustScalarizeFlag)
- return getScalarizationCost(VecTy);
-
- if (!VL0)
- return getScalarizationCost(VecTy);
- assert(VL0->getParent() == BB && "Wrong BB");
-
- unsigned Opcode = VL0->getOpcode();
- for (unsigned i = 0, e = VL.size(); i < e; ++i) {
- Instruction *I = dyn_cast<Instruction>(VL[i]);
- // If not all of the instructions are identical then we have to scalarize.
- if (!I || Opcode != I->getOpcode())
- return getScalarizationCost(VecTy);
- }
-
- // Check if it is safe to sink the loads or the stores.
- if (Opcode == Instruction::Load || Opcode == Instruction::Store) {
- int MaxIdx = getLastIndex(VL, VL.size());
- Instruction *Last = InstrVec[MaxIdx];
-
- for (unsigned i = 0, e = VL.size(); i < e; ++i) {
- if (VL[i] == Last)
- continue;
- Value *Barrier = isUnsafeToSink(cast<Instruction>(VL[i]), Last);
- if (Barrier) {
- DEBUG(dbgs() << "SLP: Can't sink " << *VL[i] << "\n down to " << *Last
- << "\n because of " << *Barrier << "\n");
- return max_cost;
- }
- }
- }
-
- // Calculate the extract cost.
- unsigned ExternalUserExtractCost = 0;
- for (unsigned i = 0, e = VL.size(); i < e; ++i)
- if (MustExtract.count(VL[i]))
- ExternalUserExtractCost +=
- TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
-
- switch (Opcode) {
- case Instruction::ExtractElement: {
- if (CanReuseExtract(VL, VL.size(), VecTy))
- return 0;
- return getScalarizationCost(VecTy);
- }
- case Instruction::ZExt:
- case Instruction::SExt:
- case Instruction::FPToUI:
- case Instruction::FPToSI:
- case Instruction::FPExt:
- case Instruction::PtrToInt:
- case Instruction::IntToPtr:
- case Instruction::SIToFP:
- case Instruction::UIToFP:
- case Instruction::Trunc:
- case Instruction::FPTrunc:
- case Instruction::BitCast: {
- int Cost = ExternalUserExtractCost;
- ValueList Operands;
- Type *SrcTy = VL0->getOperand(0)->getType();
- // Prepare the operand vector.
- for (unsigned j = 0; j < VL.size(); ++j) {
- Operands.push_back(cast<Instruction>(VL[j])->getOperand(0));
- // Check that the casted type is the same for all users.
- if (cast<Instruction>(VL[j])->getOperand(0)->getType() != SrcTy)
- return getScalarizationCost(VecTy);
- }
-
- Cost += getTreeCost_rec(Operands, Depth + 1);
- if (Cost >= max_cost)
- return max_cost;
-
- // Calculate the cost of this instruction.
- int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
- VL0->getType(), SrcTy);
-
- VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
- int VecCost = TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy);
- Cost += (VecCost - ScalarCost);
- return Cost;
- }
- case Instruction::FCmp:
- case Instruction::ICmp: {
- // Check that all of the compares have the same predicate.
- CmpInst::Predicate P0 = dyn_cast<CmpInst>(VL0)->getPredicate();
- for (unsigned i = 1, e = VL.size(); i < e; ++i) {
- CmpInst *Cmp = cast<CmpInst>(VL[i]);
- if (Cmp->getPredicate() != P0)
- return getScalarizationCost(VecTy);
- }
- // Fall through.
- }
- case Instruction::Select:
- case Instruction::Add:
- case Instruction::FAdd:
- case Instruction::Sub:
- case Instruction::FSub:
- case Instruction::Mul:
- case Instruction::FMul:
- case Instruction::UDiv:
- case Instruction::SDiv:
- case Instruction::FDiv:
- case Instruction::URem:
- case Instruction::SRem:
- case Instruction::FRem:
- case Instruction::Shl:
- case Instruction::LShr:
- case Instruction::AShr:
- case Instruction::And:
- case Instruction::Or:
- case Instruction::Xor: {
- int Cost = ExternalUserExtractCost;
- // Calculate the cost of all of the operands.
- for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
- ValueList Operands;
- // Prepare the operand vector.
- for (unsigned j = 0; j < VL.size(); ++j)
- Operands.push_back(cast<Instruction>(VL[j])->getOperand(i));
-
- Cost += getTreeCost_rec(Operands, Depth + 1);
- if (Cost >= max_cost)
- return max_cost;
- }
-
- // Calculate the cost of this instruction.
- int ScalarCost = 0;
- int VecCost = 0;
- if (Opcode == Instruction::FCmp || Opcode == Instruction::ICmp ||
- Opcode == Instruction::Select) {
- VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
- ScalarCost =
- VecTy->getNumElements() *
- TTI->getCmpSelInstrCost(Opcode, ScalarTy, Builder.getInt1Ty());
- VecCost = TTI->getCmpSelInstrCost(Opcode, VecTy, MaskTy);
- } else {
- ScalarCost = VecTy->getNumElements() *
- TTI->getArithmeticInstrCost(Opcode, ScalarTy);
- VecCost = TTI->getArithmeticInstrCost(Opcode, VecTy);
- }
- Cost += (VecCost - ScalarCost);
- return Cost;
- }
- case Instruction::Load: {
- // If we are scalarize the loads, add the cost of forming the vector.
- for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
- if (!isConsecutiveAccess(VL[i], VL[i + 1]))
- return getScalarizationCost(VecTy);
-
- // Cost of wide load - cost of scalar loads.
- int ScalarLdCost = VecTy->getNumElements() *
- TTI->getMemoryOpCost(Instruction::Load, ScalarTy, 1, 0);
- int VecLdCost = TTI->getMemoryOpCost(Instruction::Load, ScalarTy, 1, 0);
- return VecLdCost - ScalarLdCost + ExternalUserExtractCost;
- }
- case Instruction::Store: {
- // We know that we can merge the stores. Calculate the cost.
- int ScalarStCost = VecTy->getNumElements() *
- TTI->getMemoryOpCost(Instruction::Store, ScalarTy, 1, 0);
- int VecStCost = TTI->getMemoryOpCost(Instruction::Store, ScalarTy, 1, 0);
- int StoreCost = VecStCost - ScalarStCost;
-
- ValueList Operands;
- for (unsigned j = 0; j < VL.size(); ++j) {
- Operands.push_back(cast<Instruction>(VL[j])->getOperand(0));
- MemBarrierIgnoreList.insert(VL[j]);
- }
-
- int TotalCost = StoreCost + getTreeCost_rec(Operands, Depth + 1);
- return TotalCost + ExternalUserExtractCost;
- }
- default:
- // Unable to vectorize unknown instructions.
- return getScalarizationCost(VecTy);
- }
-}
-
-int BoUpSLP::getLastIndex(ArrayRef<Value *> VL, unsigned VF) {
- int MaxIdx = InstrIdx[BB->getFirstNonPHI()];
- for (unsigned i = 0; i < VF; ++i)
- MaxIdx = std::max(MaxIdx, InstrIdx[VL[i]]);
- return MaxIdx;
-}
-
-int BoUpSLP::getFirstUserIndex(ArrayRef<Value *> VL, unsigned VF) {
- // Find the first user of the values.
- int FirstUser = InstrVec.size();
- for (unsigned i = 0; i < VF; ++i) {
- for (Value::use_iterator U = VL[i]->use_begin(), UE = VL[i]->use_end();
- U != UE; ++U) {
- Instruction *Instr = dyn_cast<Instruction>(*U);
- if (!Instr || Instr->getParent() != BB)
- continue;
-
- FirstUser = std::min(FirstUser, InstrIdx[Instr]);
- }
- }
- return FirstUser;
-}
-
-int BoUpSLP::getLastIndex(Instruction *I, Instruction *J) {
- assert(I->getParent() == BB && "Invalid parent for instruction I");
- assert(J->getParent() == BB && "Invalid parent for instruction J");
- return std::max(InstrIdx[I], InstrIdx[J]);
-}
-
-Instruction *BoUpSLP::getInsertionPoint(unsigned Index) {
- return InstrVec[Index + 1];
-}
-
-Value *BoUpSLP::Scalarize(ArrayRef<Value *> VL, VectorType *Ty) {
- Value *Vec = UndefValue::get(Ty);
- for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
- // Generate the 'InsertElement' instruction.
- Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
- // Remember that this instruction is used as part of a 'gather' sequence.
- // The caller of the bottom-up slp vectorizer can try to hoist the sequence
- // if the users are outside of the basic block.
- if (InsertElementInst *IEI = dyn_cast<InsertElementInst>(Vec))
- GatherInstructions.push_back(IEI);
- }
-
- // Mark the end of the gather sequence.
- GatherInstructions.push_back(0);
-
- for (unsigned i = 0; i < Ty->getNumElements(); ++i)
- VectorizedValues[VL[i]] = Vec;
-
- return Vec;
-}
-
-Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL, int VF) {
- Value *V = vectorizeTree_rec(VL, VF);
-
- int LastInstrIdx = getLastIndex(VL, VL.size());
- for (SetVector<Value *>::iterator it = MustExtract.begin(),
- e = MustExtract.end();
- it != e; ++it) {
- Instruction *I = cast<Instruction>(*it);
-
- // This is a scalarized value, so we can use the original value.
- // No need to extract from the vector.
- if (!LaneMap.count(I))
- continue;
-
- Value *Vec = VectorizedValues[I];
- // We decided not to vectorize I because one of its users was not
- // vectorizerd. This is okay.
- if (!Vec)
- continue;
-
- Value *Idx = Builder.getInt32(LaneMap[I]);
- Value *Extract = Builder.CreateExtractElement(Vec, Idx);
- bool Replaced = false;
- for (Value::use_iterator U = I->use_begin(), UE = I->use_end(); U != UE;
- ++U) {
- Instruction *UI = cast<Instruction>(*U);
- if (UI->getParent() != I->getParent() || InstrIdx[UI] > LastInstrIdx)
- UI->replaceUsesOfWith(I, Extract);
- Replaced = true;
- }
- assert(Replaced && "Must replace at least one outside user");
- (void)Replaced;
- }
-
- // We moved some instructions around. We have to number them again
- // before we can do any analysis.
- numberInstructions();
- MustScalarize.clear();
- MustExtract.clear();
- VectorizedValues.clear();
- return V;
-}
-
-Value *BoUpSLP::vectorizeTree_rec(ArrayRef<Value *> VL, int VF) {
- Type *ScalarTy = VL[0]->getType();
- if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
- ScalarTy = SI->getValueOperand()->getType();
- VectorType *VecTy = VectorType::get(ScalarTy, VF);
-
- // Check if all of the operands are constants or identical.
- bool AllConst = true;
- bool AllSameScalar = true;
- for (unsigned i = 0, e = VF; i < e; ++i) {
- AllConst &= isa<Constant>(VL[i]);
- AllSameScalar &= (VL[0] == VL[i]);
- // The instruction must be in the same BB, and it must be vectorizable.
- Instruction *I = dyn_cast<Instruction>(VL[i]);
- if (MustScalarize.count(VL[i]) || (I && I->getParent() != BB))
- return Scalarize(VL, VecTy);
- }
-
- // Check that this is a simple vector constant.
- if (AllConst || AllSameScalar)
- return Scalarize(VL, VecTy);
-
- // Scalarize unknown structures.
- Instruction *VL0 = dyn_cast<Instruction>(VL[0]);
- if (!VL0)
- return Scalarize(VL, VecTy);
-
- if (VectorizedValues.count(VL0)) {
- Value *Vec = VectorizedValues[VL0];
- for (int i = 0; i < VF; ++i)
- VectorizedValues[VL[i]] = Vec;
- return Vec;
- }
-
- unsigned Opcode = VL0->getOpcode();
- for (unsigned i = 0, e = VF; i < e; ++i) {
- Instruction *I = dyn_cast<Instruction>(VL[i]);
- // If not all of the instructions are identical then we have to scalarize.
- if (!I || Opcode != I->getOpcode())
- return Scalarize(VL, VecTy);
- }
-
- switch (Opcode) {
- case Instruction::ExtractElement: {
- if (CanReuseExtract(VL, VL.size(), VecTy))
- return VL0->getOperand(0);
- return Scalarize(VL, VecTy);
- }
- case Instruction::ZExt:
- case Instruction::SExt:
- case Instruction::FPToUI:
- case Instruction::FPToSI:
- case Instruction::FPExt:
- case Instruction::PtrToInt:
- case Instruction::IntToPtr:
- case Instruction::SIToFP:
- case Instruction::UIToFP:
- case Instruction::Trunc:
- case Instruction::FPTrunc:
- case Instruction::BitCast: {
- ValueList INVL;
- for (int i = 0; i < VF; ++i)
- INVL.push_back(cast<Instruction>(VL[i])->getOperand(0));
- Value *InVec = vectorizeTree_rec(INVL, VF);
- CastInst *CI = dyn_cast<CastInst>(VL0);
- Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
-
- for (int i = 0; i < VF; ++i)
- VectorizedValues[VL[i]] = V;
-
- return V;
- }
- case Instruction::FCmp:
- case Instruction::ICmp: {
- // Check that all of the compares have the same predicate.
- CmpInst::Predicate P0 = dyn_cast<CmpInst>(VL0)->getPredicate();
- for (unsigned i = 1, e = VF; i < e; ++i) {
- CmpInst *Cmp = cast<CmpInst>(VL[i]);
- if (Cmp->getPredicate() != P0)
- return Scalarize(VL, VecTy);
- }
-
- ValueList LHSV, RHSV;
- for (int i = 0; i < VF; ++i) {
- LHSV.push_back(cast<Instruction>(VL[i])->getOperand(0));
- RHSV.push_back(cast<Instruction>(VL[i])->getOperand(1));
- }
-
- Value *L = vectorizeTree_rec(LHSV, VF);
- Value *R = vectorizeTree_rec(RHSV, VF);
- Value *V;
- if (VL0->getOpcode() == Instruction::FCmp)
- V = Builder.CreateFCmp(P0, L, R);
- else
- V = Builder.CreateICmp(P0, L, R);
-
- for (int i = 0; i < VF; ++i)
- VectorizedValues[VL[i]] = V;
-
- return V;
- }
- case Instruction::Select: {
- ValueList TrueVec, FalseVec, CondVec;
- for (int i = 0; i < VF; ++i) {
- CondVec.push_back(cast<Instruction>(VL[i])->getOperand(0));
- TrueVec.push_back(cast<Instruction>(VL[i])->getOperand(1));
- FalseVec.push_back(cast<Instruction>(VL[i])->getOperand(2));
- }
-
- Value *True = vectorizeTree_rec(TrueVec, VF);
- Value *False = vectorizeTree_rec(FalseVec, VF);
- Value *Cond = vectorizeTree_rec(CondVec, VF);
- Value *V = Builder.CreateSelect(Cond, True, False);
-
- for (int i = 0; i < VF; ++i)
- VectorizedValues[VL[i]] = V;
-
- return V;
- }
- case Instruction::Add:
- case Instruction::FAdd:
- case Instruction::Sub:
- case Instruction::FSub:
- case Instruction::Mul:
- case Instruction::FMul:
- case Instruction::UDiv:
- case Instruction::SDiv:
- case Instruction::FDiv:
- case Instruction::URem:
- case Instruction::SRem:
- case Instruction::FRem:
- case Instruction::Shl:
- case Instruction::LShr:
- case Instruction::AShr:
- case Instruction::And:
- case Instruction::Or:
- case Instruction::Xor: {
- ValueList LHSVL, RHSVL;
- for (int i = 0; i < VF; ++i) {
- LHSVL.push_back(cast<Instruction>(VL[i])->getOperand(0));
- RHSVL.push_back(cast<Instruction>(VL[i])->getOperand(1));
- }
-
- Value *LHS = vectorizeTree_rec(LHSVL, VF);
- Value *RHS = vectorizeTree_rec(RHSVL, VF);
- BinaryOperator *BinOp = cast<BinaryOperator>(VL0);
- Value *V = Builder.CreateBinOp(BinOp->getOpcode(), LHS, RHS);
-
- for (int i = 0; i < VF; ++i)
- VectorizedValues[VL[i]] = V;
-
- return V;
- }
- case Instruction::Load: {
- LoadInst *LI = cast<LoadInst>(VL0);
- unsigned Alignment = LI->getAlignment();
-
- // Check if all of the loads are consecutive.
- for (unsigned i = 1, e = VF; i < e; ++i)
- if (!isConsecutiveAccess(VL[i - 1], VL[i]))
- return Scalarize(VL, VecTy);
-
- // Loads are inserted at the head of the tree because we don't want to sink
- // them all the way down past store instructions.
- Instruction *Loc = getInsertionPoint(getLastIndex(VL, VL.size()));
- IRBuilder<> LoadBuilder(Loc);
- Value *VecPtr = LoadBuilder.CreateBitCast(LI->getPointerOperand(),
- VecTy->getPointerTo());
- LI = LoadBuilder.CreateLoad(VecPtr);
- LI->setAlignment(Alignment);
-
- for (int i = 0; i < VF; ++i)
- VectorizedValues[VL[i]] = LI;
-
- return LI;
- }
- case Instruction::Store: {
- StoreInst *SI = cast<StoreInst>(VL0);
- unsigned Alignment = SI->getAlignment();
-
- ValueList ValueOp;
- for (int i = 0; i < VF; ++i)
- ValueOp.push_back(cast<StoreInst>(VL[i])->getValueOperand());
-
- Value *VecValue = vectorizeTree_rec(ValueOp, VF);
- Value *VecPtr =
- Builder.CreateBitCast(SI->getPointerOperand(), VecTy->getPointerTo());
- Builder.CreateStore(VecValue, VecPtr)->setAlignment(Alignment);
-
- for (int i = 0; i < VF; ++i)
- cast<Instruction>(VL[i])->eraseFromParent();
- return 0;
- }
- default:
- return Scalarize(VL, VecTy);
- }
-}
-
-} // end of namespace
projects / oota-llvm.git / commitdiff