1 //===- CostModel.cpp ------ Cost Model Analysis ---------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the cost model analysis. It provides a very basic cost
11 // estimation for LLVM-IR. This analysis uses the services of the codegen
12 // to approximate the cost of any IR instruction when lowered to machine
13 // instructions. The cost results are unit-less and the cost number represents
14 // the throughput of the machine assuming that all loads hit the cache, all
15 // branches are predicted, etc. The cost numbers can be added in order to
16 // compare two or more transformation alternatives.
18 //===----------------------------------------------------------------------===//
20 #define CM_NAME "cost-model"
21 #define DEBUG_TYPE CM_NAME
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/Analysis/Passes.h"
24 #include "llvm/Analysis/TargetTransformInfo.h"
25 #include "llvm/IR/Function.h"
26 #include "llvm/IR/Instructions.h"
27 #include "llvm/IR/IntrinsicInst.h"
28 #include "llvm/IR/Value.h"
29 #include "llvm/Pass.h"
30 #include "llvm/Support/CommandLine.h"
31 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/raw_ostream.h"
35 static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false),
37 cl::desc("Recognize reduction patterns."));
40 class CostModelAnalysis : public FunctionPass {
43 static char ID; // Class identification, replacement for typeinfo
44 CostModelAnalysis() : FunctionPass(ID), F(0), TTI(0) {
45 initializeCostModelAnalysisPass(
46 *PassRegistry::getPassRegistry());
49 /// Returns the expected cost of the instruction.
50 /// Returns -1 if the cost is unknown.
51 /// Note, this method does not cache the cost calculation and it
52 /// can be expensive in some cases.
53 unsigned getInstructionCost(const Instruction *I) const;
56 virtual void getAnalysisUsage(AnalysisUsage &AU) const;
57 virtual bool runOnFunction(Function &F);
58 virtual void print(raw_ostream &OS, const Module*) const;
60 /// The function that we analyze.
62 /// Target information.
63 const TargetTransformInfo *TTI;
65 } // End of anonymous namespace
67 // Register this pass.
68 char CostModelAnalysis::ID = 0;
69 static const char cm_name[] = "Cost Model Analysis";
70 INITIALIZE_PASS_BEGIN(CostModelAnalysis, CM_NAME, cm_name, false, true)
71 INITIALIZE_PASS_END (CostModelAnalysis, CM_NAME, cm_name, false, true)
73 FunctionPass *llvm::createCostModelAnalysisPass() {
74 return new CostModelAnalysis();
78 CostModelAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
83 CostModelAnalysis::runOnFunction(Function &F) {
85 TTI = getAnalysisIfAvailable<TargetTransformInfo>();
90 static bool isReverseVectorMask(SmallVectorImpl<int> &Mask) {
91 for (unsigned i = 0, MaskSize = Mask.size(); i < MaskSize; ++i)
92 if (Mask[i] > 0 && Mask[i] != (int)(MaskSize - 1 - i))
97 static TargetTransformInfo::OperandValueKind getOperandInfo(Value *V) {
98 TargetTransformInfo::OperandValueKind OpInfo =
99 TargetTransformInfo::OK_AnyValue;
101 // Check for a splat of a constant.
102 ConstantDataVector *CDV = 0;
103 if ((CDV = dyn_cast<ConstantDataVector>(V)))
104 if (CDV->getSplatValue() != NULL)
105 OpInfo = TargetTransformInfo::OK_UniformConstantValue;
106 ConstantVector *CV = 0;
107 if ((CV = dyn_cast<ConstantVector>(V)))
108 if (CV->getSplatValue() != NULL)
109 OpInfo = TargetTransformInfo::OK_UniformConstantValue;
114 static bool matchMask(SmallVectorImpl<int> &M1, SmallVectorImpl<int> &M2) {
115 if (M1.size() != M2.size())
118 for (unsigned i = 0, e = M1.size(); i != e; ++i)
125 static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft,
127 // We don't need a shuffle if we just want to have element 0 in position 0 of
129 if (!SI && Level == 0 && IsLeft)
134 SmallVector<int, 32> Mask(SI->getType()->getVectorNumElements(), -1);
136 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right) depending on whether
137 // we look at the left or right side.
138 for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
141 SmallVector<int, 16> ActualMask = SI->getShuffleMask();
142 if (!matchMask(Mask, ActualMask))
148 static bool matchPairwiseReductionAtLevel(const BinaryOperator *BinOp,
149 unsigned Level, unsigned NumLevels) {
150 // Match one level of pairwise operations.
151 // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
152 // <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
153 // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef,
154 // <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
155 // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
159 assert(BinOp->getType()->isVectorTy() && "Expecting a vector type");
161 unsigned Opcode = BinOp->getOpcode();
162 Value *L = BinOp->getOperand(0);
163 Value *R = BinOp->getOperand(1);
165 ShuffleVectorInst *LS = dyn_cast<ShuffleVectorInst>(L);
168 ShuffleVectorInst *RS = dyn_cast<ShuffleVectorInst>(R);
172 // On level 0 we can omit one shufflevector instruction.
173 if (!Level && !RS && !LS)
176 // Shuffle inputs must match.
177 Value *NextLevelOpL = LS ? LS->getOperand(0) : 0;
178 Value *NextLevelOpR = RS ? RS->getOperand(0) : 0;
179 Value *NextLevelOp = 0;
180 if (NextLevelOpR && NextLevelOpL) {
181 // If we have two shuffles their operands must match.
182 if (NextLevelOpL != NextLevelOpR)
185 NextLevelOp = NextLevelOpL;
186 } else if (Level == 0 && (NextLevelOpR || NextLevelOpL)) {
187 // On the first level we can omit the shufflevector <0, undef,...>. So the
188 // input to the other shufflevector <1, undef> must match with one of the
189 // inputs to the current binary operation.
191 // %NextLevelOpL = shufflevector %R, <1, undef ...>
192 // %BinOp = fadd %NextLevelOpL, %R
193 if (NextLevelOpL && NextLevelOpL != R)
195 else if (NextLevelOpR && NextLevelOpR != L)
198 NextLevelOp = NextLevelOpL ? R : L;
202 // Check that the next levels binary operation exists and matches with the
204 BinaryOperator *NextLevelBinOp = 0;
205 if (Level + 1 != NumLevels) {
206 if (!(NextLevelBinOp = dyn_cast<BinaryOperator>(NextLevelOp)))
208 else if (NextLevelBinOp->getOpcode() != Opcode)
212 // Shuffle mask for pairwise operation must match.
213 if (matchPairwiseShuffleMask(LS, true, Level)) {
214 if (!matchPairwiseShuffleMask(RS, false, Level))
216 } else if (matchPairwiseShuffleMask(RS, true, Level)) {
217 if (!matchPairwiseShuffleMask(LS, false, Level))
222 if (++Level == NumLevels)
226 return matchPairwiseReductionAtLevel(NextLevelBinOp, Level, NumLevels);
229 static bool matchPairwiseReduction(const ExtractElementInst *ReduxRoot,
230 unsigned &Opcode, Type *&Ty) {
231 if (!EnableReduxCost)
234 // Need to extract the first element.
235 ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
238 Idx = CI->getZExtValue();
242 BinaryOperator *RdxStart = dyn_cast<BinaryOperator>(ReduxRoot->getOperand(0));
246 Type *VecTy = ReduxRoot->getOperand(0)->getType();
247 unsigned NumVecElems = VecTy->getVectorNumElements();
248 if (!isPowerOf2_32(NumVecElems))
251 // We look for a sequence of shuffle,shuffle,add triples like the following
252 // that builds a pairwise reduction tree.
255 // (X0 + X1, X2 + X3, undef, undef)
256 // ((X0 + X1) + (X2 + X3), undef, undef, undef)
258 // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
259 // <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
260 // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef,
261 // <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
262 // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
263 // %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef,
264 // <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
265 // %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef,
266 // <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
267 // %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
268 // %r = extractelement <4 x float> %bin.rdx8, i32 0
269 if (!matchPairwiseReductionAtLevel(RdxStart, 0, Log2_32(NumVecElems)))
272 Opcode = RdxStart->getOpcode();
278 static std::pair<Value *, ShuffleVectorInst *>
279 getShuffleAndOtherOprd(BinaryOperator *B) {
281 Value *L = B->getOperand(0);
282 Value *R = B->getOperand(1);
283 ShuffleVectorInst *S = 0;
285 if ((S = dyn_cast<ShuffleVectorInst>(L)))
286 return std::make_pair(R, S);
288 S = dyn_cast<ShuffleVectorInst>(R);
289 return std::make_pair(L, S);
292 static bool matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot,
293 unsigned &Opcode, Type *&Ty) {
294 if (!EnableReduxCost)
297 // Need to extract the first element.
298 ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
301 Idx = CI->getZExtValue();
305 BinaryOperator *RdxStart = dyn_cast<BinaryOperator>(ReduxRoot->getOperand(0));
308 unsigned RdxOpcode = RdxStart->getOpcode();
310 Type *VecTy = ReduxRoot->getOperand(0)->getType();
311 unsigned NumVecElems = VecTy->getVectorNumElements();
312 if (!isPowerOf2_32(NumVecElems))
315 // We look for a sequence of shuffles and adds like the following matching one
316 // fadd, shuffle vector pair at a time.
318 // %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef,
319 // <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
320 // %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
321 // %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef,
322 // <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
323 // %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
324 // %r = extractelement <4 x float> %bin.rdx8, i32 0
326 unsigned MaskStart = 1;
327 Value *RdxOp = RdxStart;
328 SmallVector<int, 32> ShuffleMask(NumVecElems, 0);
329 unsigned NumVecElemsRemain = NumVecElems;
330 while (NumVecElemsRemain - 1) {
331 // Check for the right reduction operation.
332 BinaryOperator *BinOp;
333 if (!(BinOp = dyn_cast<BinaryOperator>(RdxOp)))
335 if (BinOp->getOpcode() != RdxOpcode)
339 ShuffleVectorInst *Shuffle;
340 tie(NextRdxOp, Shuffle) = getShuffleAndOtherOprd(BinOp);
342 // Check the current reduction operation and the shuffle use the same value.
345 if (Shuffle->getOperand(0) != NextRdxOp)
348 // Check that shuffle masks matches.
349 for (unsigned j = 0; j != MaskStart; ++j)
350 ShuffleMask[j] = MaskStart + j;
351 // Fill the rest of the mask with -1 for undef.
352 std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
354 SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
355 if (!matchMask(ShuffleMask, Mask))
359 NumVecElemsRemain /= 2;
368 unsigned CostModelAnalysis::getInstructionCost(const Instruction *I) const {
372 switch (I->getOpcode()) {
373 case Instruction::GetElementPtr:{
374 Type *ValTy = I->getOperand(0)->getType()->getPointerElementType();
375 return TTI->getAddressComputationCost(ValTy);
378 case Instruction::Ret:
379 case Instruction::PHI:
380 case Instruction::Br: {
381 return TTI->getCFInstrCost(I->getOpcode());
383 case Instruction::Add:
384 case Instruction::FAdd:
385 case Instruction::Sub:
386 case Instruction::FSub:
387 case Instruction::Mul:
388 case Instruction::FMul:
389 case Instruction::UDiv:
390 case Instruction::SDiv:
391 case Instruction::FDiv:
392 case Instruction::URem:
393 case Instruction::SRem:
394 case Instruction::FRem:
395 case Instruction::Shl:
396 case Instruction::LShr:
397 case Instruction::AShr:
398 case Instruction::And:
399 case Instruction::Or:
400 case Instruction::Xor: {
401 TargetTransformInfo::OperandValueKind Op1VK =
402 getOperandInfo(I->getOperand(0));
403 TargetTransformInfo::OperandValueKind Op2VK =
404 getOperandInfo(I->getOperand(1));
405 return TTI->getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK,
408 case Instruction::Select: {
409 const SelectInst *SI = cast<SelectInst>(I);
410 Type *CondTy = SI->getCondition()->getType();
411 return TTI->getCmpSelInstrCost(I->getOpcode(), I->getType(), CondTy);
413 case Instruction::ICmp:
414 case Instruction::FCmp: {
415 Type *ValTy = I->getOperand(0)->getType();
416 return TTI->getCmpSelInstrCost(I->getOpcode(), ValTy);
418 case Instruction::Store: {
419 const StoreInst *SI = cast<StoreInst>(I);
420 Type *ValTy = SI->getValueOperand()->getType();
421 return TTI->getMemoryOpCost(I->getOpcode(), ValTy,
423 SI->getPointerAddressSpace());
425 case Instruction::Load: {
426 const LoadInst *LI = cast<LoadInst>(I);
427 return TTI->getMemoryOpCost(I->getOpcode(), I->getType(),
429 LI->getPointerAddressSpace());
431 case Instruction::ZExt:
432 case Instruction::SExt:
433 case Instruction::FPToUI:
434 case Instruction::FPToSI:
435 case Instruction::FPExt:
436 case Instruction::PtrToInt:
437 case Instruction::IntToPtr:
438 case Instruction::SIToFP:
439 case Instruction::UIToFP:
440 case Instruction::Trunc:
441 case Instruction::FPTrunc:
442 case Instruction::BitCast: {
443 Type *SrcTy = I->getOperand(0)->getType();
444 return TTI->getCastInstrCost(I->getOpcode(), I->getType(), SrcTy);
446 case Instruction::ExtractElement: {
447 const ExtractElementInst * EEI = cast<ExtractElementInst>(I);
448 ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1));
451 Idx = CI->getZExtValue();
453 // Try to match a reduction sequence (series of shufflevector and vector
454 // adds followed by a extractelement).
455 unsigned ReduxOpCode;
458 if (matchVectorSplittingReduction(EEI, ReduxOpCode, ReduxType))
459 return TTI->getReductionCost(ReduxOpCode, ReduxType, false);
460 else if (matchPairwiseReduction(EEI, ReduxOpCode, ReduxType))
461 return TTI->getReductionCost(ReduxOpCode, ReduxType, true);
463 return TTI->getVectorInstrCost(I->getOpcode(),
464 EEI->getOperand(0)->getType(), Idx);
466 case Instruction::InsertElement: {
467 const InsertElementInst * IE = cast<InsertElementInst>(I);
468 ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
471 Idx = CI->getZExtValue();
472 return TTI->getVectorInstrCost(I->getOpcode(),
475 case Instruction::ShuffleVector: {
476 const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I);
477 Type *VecTypOp0 = Shuffle->getOperand(0)->getType();
478 unsigned NumVecElems = VecTypOp0->getVectorNumElements();
479 SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
481 if (NumVecElems == Mask.size() && isReverseVectorMask(Mask))
482 return TTI->getShuffleCost(TargetTransformInfo::SK_Reverse, VecTypOp0, 0,
486 case Instruction::Call:
487 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
488 SmallVector<Type*, 4> Tys;
489 for (unsigned J = 0, JE = II->getNumArgOperands(); J != JE; ++J)
490 Tys.push_back(II->getArgOperand(J)->getType());
492 return TTI->getIntrinsicInstrCost(II->getIntrinsicID(), II->getType(),
497 // We don't have any information on this instruction.
502 void CostModelAnalysis::print(raw_ostream &OS, const Module*) const {
506 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
507 for (BasicBlock::iterator it = B->begin(), e = B->end(); it != e; ++it) {
508 Instruction *Inst = it;
509 unsigned Cost = getInstructionCost(Inst);
510 if (Cost != (unsigned)-1)
511 OS << "Cost Model: Found an estimated cost of " << Cost;
513 OS << "Cost Model: Unknown cost";
515 OS << " for instruction: "<< *Inst << "\n";