X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FAnalysis%2FInlineCost.cpp;h=3e550f35c255362c1353079579c3db64c082526b;hb=00667a54f21f2d58382ac2eed973706332ed5817;hp=4f010b658ef47412e6f055237fbf32fe9c19ffb5;hpb=7b929dad59785f62a66f7c58615082f98441e95e;p=oota-llvm.git diff --git a/lib/Analysis/InlineCost.cpp b/lib/Analysis/InlineCost.cpp index 4f010b658ef..3e550f35c25 100644 --- a/lib/Analysis/InlineCost.cpp +++ b/lib/Analysis/InlineCost.cpp @@ -22,26 +22,32 @@ using namespace llvm; // instructions will be constant folded if the specified value is constant. // unsigned InlineCostAnalyzer::FunctionInfo:: - CountCodeReductionForConstant(Value *V) { +CountCodeReductionForConstant(Value *V) { unsigned Reduction = 0; - for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); UI != E; ++UI) - if (isa(*UI)) - Reduction += 40; // Eliminating a conditional branch is a big win - else if (SwitchInst *SI = dyn_cast(*UI)) - // Eliminating a switch is a big win, proportional to the number of edges - // deleted. - Reduction += (SI->getNumSuccessors()-1) * 40; - else if (CallInst *CI = dyn_cast(*UI)) { + for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); UI != E;++UI){ + User *U = *UI; + if (isa(U) || isa(U)) { + // We will be able to eliminate all but one of the successors. + const TerminatorInst &TI = cast(*U); + const unsigned NumSucc = TI.getNumSuccessors(); + unsigned Instrs = 0; + for (unsigned I = 0; I != NumSucc; ++I) + Instrs += Metrics.NumBBInsts[TI.getSuccessor(I)]; + // We don't know which blocks will be eliminated, so use the average size. + Reduction += InlineConstants::InstrCost*Instrs*(NumSucc-1)/NumSucc; + } else if (CallInst *CI = dyn_cast(U)) { // Turning an indirect call into a direct call is a BIG win - Reduction += CI->getCalledValue() == V ? 500 : 0; - } else if (InvokeInst *II = dyn_cast(*UI)) { + if (CI->getCalledValue() == V) + Reduction += InlineConstants::IndirectCallBonus; + } else if (InvokeInst *II = dyn_cast(U)) { // Turning an indirect call into a direct call is a BIG win - Reduction += II->getCalledValue() == V ? 500 : 0; + if (II->getCalledValue() == V) + Reduction += InlineConstants::IndirectCallBonus; } else { // Figure out if this instruction will be removed due to simple constant // propagation. - Instruction &Inst = cast(**UI); - + Instruction &Inst = cast(*U); + // We can't constant propagate instructions which have effects or // read memory. // @@ -50,7 +56,7 @@ unsigned InlineCostAnalyzer::FunctionInfo:: // Unfortunately, we don't know the pointer that may get propagated here, // so we can't make this decision. if (Inst.mayReadFromMemory() || Inst.mayHaveSideEffects() || - isa(Inst)) + isa(Inst)) continue; bool AllOperandsConstant = true; @@ -62,14 +68,14 @@ unsigned InlineCostAnalyzer::FunctionInfo:: if (AllOperandsConstant) { // We will get to remove this instruction... - Reduction += 7; + Reduction += InlineConstants::InstrCost; // And any other instructions that use it which become constants // themselves. Reduction += CountCodeReductionForConstant(&Inst); } } - + } return Reduction; } @@ -79,16 +85,19 @@ unsigned InlineCostAnalyzer::FunctionInfo:: // unsigned InlineCostAnalyzer::FunctionInfo:: CountCodeReductionForAlloca(Value *V) { - if (!isa(V->getType())) return 0; // Not a pointer + if (!V->getType()->isPointerTy()) return 0; // Not a pointer unsigned Reduction = 0; for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); UI != E;++UI){ Instruction *I = cast(*UI); if (isa(I) || isa(I)) - Reduction += 10; + Reduction += InlineConstants::InstrCost; else if (GetElementPtrInst *GEP = dyn_cast(I)) { // If the GEP has variable indices, we won't be able to do much with it. - if (!GEP->hasAllConstantIndices()) - Reduction += CountCodeReductionForAlloca(GEP)+15; + if (GEP->hasAllConstantIndices()) + Reduction += CountCodeReductionForAlloca(GEP); + } else if (BitCastInst *BCI = dyn_cast(I)) { + // Track pointer through bitcasts. + Reduction += CountCodeReductionForAlloca(BCI); } else { // If there is some other strange instruction, we're not going to be able // to do much if we inline this. @@ -99,11 +108,42 @@ unsigned InlineCostAnalyzer::FunctionInfo:: return Reduction; } +/// callIsSmall - If a call is likely to lower to a single target instruction, +/// or is otherwise deemed small return true. +/// TODO: Perhaps calls like memcpy, strcpy, etc? +bool llvm::callIsSmall(const Function *F) { + if (!F) return false; + + if (F->hasLocalLinkage()) return false; + + if (!F->hasName()) return false; + + StringRef Name = F->getName(); + + // These will all likely lower to a single selection DAG node. + if (Name == "copysign" || Name == "copysignf" || Name == "copysignl" || + Name == "fabs" || Name == "fabsf" || Name == "fabsl" || + Name == "sin" || Name == "sinf" || Name == "sinl" || + Name == "cos" || Name == "cosf" || Name == "cosl" || + Name == "sqrt" || Name == "sqrtf" || Name == "sqrtl" ) + return true; + + // These are all likely to be optimized into something smaller. + if (Name == "pow" || Name == "powf" || Name == "powl" || + Name == "exp2" || Name == "exp2l" || Name == "exp2f" || + Name == "floor" || Name == "floorf" || Name == "ceil" || + Name == "round" || Name == "ffs" || Name == "ffsl" || + Name == "abs" || Name == "labs" || Name == "llabs") + return true; + + return false; +} + /// analyzeBasicBlock - Fill in the current structure with information gleaned /// from the specified block. void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB) { ++NumBlocks; - + unsigned NumInstsBeforeThisBB = NumInsts; for (BasicBlock::const_iterator II = BB->begin(), E = BB->end(); II != E; ++II) { if (isa(II)) continue; // PHI nodes don't count. @@ -112,54 +152,78 @@ void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB) { if (isa(II) || isa(II)) { if (isa(II)) continue; // Debug intrinsics don't count as size. - - CallSite CS = CallSite::get(const_cast(&*II)); - + + ImmutableCallSite CS(cast(II)); + // If this function contains a call to setjmp or _setjmp, never inline // it. This is a hack because we depend on the user marking their local // variables as volatile if they are live across a setjmp call, and they // probably won't do this in callers. - if (Function *F = CS.getCalledFunction()) + if (const Function *F = CS.getCalledFunction()) { if (F->isDeclaration() && (F->getName() == "setjmp" || F->getName() == "_setjmp")) - NeverInline = true; + callsSetJmp = true; + + // If this call is to function itself, then the function is recursive. + // Inlining it into other functions is a bad idea, because this is + // basically just a form of loop peeling, and our metrics aren't useful + // for that case. + if (F == BB->getParent()) + isRecursive = true; + } - // Calls often compile into many machine instructions. Bump up their - // cost to reflect this. - if (!isa(II)) - NumInsts += InlineConstants::CallPenalty; + if (!isa(II) && !callIsSmall(CS.getCalledFunction())) { + // Each argument to a call takes on average one instruction to set up. + NumInsts += CS.arg_size(); + + // We don't want inline asm to count as a call - that would prevent loop + // unrolling. The argument setup cost is still real, though. + if (!isa(CS.getCalledValue())) + ++NumCalls; + } } - // These, too, are calls. - if (isa(II)) - NumInsts += InlineConstants::CallPenalty; - if (const AllocaInst *AI = dyn_cast(II)) { if (!AI->isStaticAlloca()) this->usesDynamicAlloca = true; } - if (isa(II) || isa(II->getType())) + if (isa(II) || II->getType()->isVectorTy()) ++NumVectorInsts; - // Noop casts, including ptr <-> int, don't count. if (const CastInst *CI = dyn_cast(II)) { + // Noop casts, including ptr <-> int, don't count. if (CI->isLosslessCast() || isa(CI) || isa(CI)) continue; - } else if (const GetElementPtrInst *GEPI = - dyn_cast(II)) { + // Result of a cmp instruction is often extended (to be used by other + // cmp instructions, logical or return instructions). These are usually + // nop on most sane targets. + if (isa(CI->getOperand(0))) + continue; + } else if (const GetElementPtrInst *GEPI = dyn_cast(II)){ // If a GEP has all constant indices, it will probably be folded with // a load/store. if (GEPI->hasAllConstantIndices()) continue; } - if (isa(II)) - ++NumRets; - ++NumInsts; } + + if (isa(BB->getTerminator())) + ++NumRets; + + // We never want to inline functions that contain an indirectbr. This is + // incorrect because all the blockaddress's (in static global initializers + // for example) would be referring to the original function, and this indirect + // jump would jump from the inlined copy of the function into the original + // function which is extremely undefined behavior. + if (isa(BB->getTerminator())) + containsIndirectBr = true; + + // Remember NumInsts for this BB. + NumBBInsts[BB] = NumInsts - NumInstsBeforeThisBB; } /// analyzeFunction - Fill in the current structure with information gleaned @@ -181,26 +245,48 @@ void InlineCostAnalyzer::FunctionInfo::analyzeFunction(Function *F) { if (Metrics.NumRets==1) --Metrics.NumInsts; + // Don't bother calculating argument weights if we are never going to inline + // the function anyway. + if (NeverInline()) + return; + // Check out all of the arguments to the function, figuring out how much // code can be eliminated if one of the arguments is a constant. + ArgumentWeights.reserve(F->arg_size()); for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I) ArgumentWeights.push_back(ArgInfo(CountCodeReductionForConstant(I), CountCodeReductionForAlloca(I))); } +/// NeverInline - returns true if the function should never be inlined into +/// any caller +bool InlineCostAnalyzer::FunctionInfo::NeverInline() +{ + return (Metrics.callsSetJmp || Metrics.isRecursive || + Metrics.containsIndirectBr); + +} // getInlineCost - The heuristic used to determine if we should inline the // function call or not. // InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS, - SmallPtrSet &NeverInline) { + SmallPtrSet &NeverInline) { + return getInlineCost(CS, CS.getCalledFunction(), NeverInline); +} + +InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS, + Function *Callee, + SmallPtrSet &NeverInline) { Instruction *TheCall = CS.getInstruction(); - Function *Callee = CS.getCalledFunction(); Function *Caller = TheCall->getParent()->getParent(); + bool isDirectCall = CS.getCalledFunction() == Callee; // Don't inline functions which can be redefined at link-time to mean - // something else. Don't inline functions marked noinline. + // something else. Don't inline functions marked noinline or call sites + // marked noinline. if (Callee->mayBeOverridden() || - Callee->hasFnAttr(Attribute::NoInline) || NeverInline.count(Callee)) + Callee->hasFnAttr(Attribute::NoInline) || NeverInline.count(Callee) || + CS.isNoInline()) return llvm::InlineCost::getNever(); // InlineCost - This value measures how good of an inline candidate this call @@ -208,11 +294,11 @@ InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS, // be inlined. This value may go negative. // int InlineCost = 0; - + // If there is only one call of the function, and it has internal linkage, // make it almost guaranteed to be inlined. // - if (Callee->hasLocalLinkage() && Callee->hasOneUse()) + if (Callee->hasLocalLinkage() && Callee->hasOneUse() && isDirectCall) InlineCost += InlineConstants::LastCallToStaticBonus; // If this function uses the coldcc calling convention, prefer not to inline @@ -229,31 +315,36 @@ InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS, } else if (isa(++BasicBlock::iterator(TheCall))) InlineCost += InlineConstants::NoreturnPenalty; - // Get information about the callee... - FunctionInfo &CalleeFI = CachedFunctionInfo[Callee]; + // Get information about the callee. + FunctionInfo *CalleeFI = &CachedFunctionInfo[Callee]; // If we haven't calculated this information yet, do so now. - if (CalleeFI.Metrics.NumBlocks == 0) - CalleeFI.analyzeFunction(Callee); + if (CalleeFI->Metrics.NumBlocks == 0) + CalleeFI->analyzeFunction(Callee); // If we should never inline this, return a huge cost. - if (CalleeFI.Metrics.NeverInline) + if (CalleeFI->NeverInline()) return InlineCost::getNever(); - // FIXME: It would be nice to kill off CalleeFI.NeverInline. Then we + // FIXME: It would be nice to kill off CalleeFI->NeverInline. Then we // could move this up and avoid computing the FunctionInfo for // things we are going to just return always inline for. This // requires handling setjmp somewhere else, however. if (!Callee->isDeclaration() && Callee->hasFnAttr(Attribute::AlwaysInline)) return InlineCost::getAlways(); - if (CalleeFI.Metrics.usesDynamicAlloca) { - // Get infomation about the caller... + if (CalleeFI->Metrics.usesDynamicAlloca) { + // Get infomation about the caller. FunctionInfo &CallerFI = CachedFunctionInfo[Caller]; // If we haven't calculated this information yet, do so now. - if (CallerFI.Metrics.NumBlocks == 0) + if (CallerFI.Metrics.NumBlocks == 0) { CallerFI.analyzeFunction(Caller); + + // Recompute the CalleeFI pointer, getting Caller could have invalidated + // it. + CalleeFI = &CachedFunctionInfo[Callee]; + } // Don't inline a callee with dynamic alloca into a caller without them. // Functions containing dynamic alloca's are inefficient in various ways; @@ -271,42 +362,35 @@ InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS, for (CallSite::arg_iterator I = CS.arg_begin(), E = CS.arg_end(); I != E; ++I, ++ArgNo) { // Each argument passed in has a cost at both the caller and the callee - // sides. This favors functions that take many arguments over functions - // that take few arguments. - InlineCost -= 20; - - // If this is a function being passed in, it is very likely that we will be - // able to turn an indirect function call into a direct function call. - if (isa(I)) - InlineCost -= 100; - + // sides. Measurements show that each argument costs about the same as an + // instruction. + InlineCost -= InlineConstants::InstrCost; + // If an alloca is passed in, inlining this function is likely to allow // significant future optimization possibilities (like scalar promotion, and // scalarization), so encourage the inlining of the function. // - else if (isa(I)) { - if (ArgNo < CalleeFI.ArgumentWeights.size()) - InlineCost -= CalleeFI.ArgumentWeights[ArgNo].AllocaWeight; - + if (isa(I)) { + if (ArgNo < CalleeFI->ArgumentWeights.size()) + InlineCost -= CalleeFI->ArgumentWeights[ArgNo].AllocaWeight; + // If this is a constant being passed into the function, use the argument // weights calculated for the callee to determine how much will be folded // away with this information. } else if (isa(I)) { - if (ArgNo < CalleeFI.ArgumentWeights.size()) - InlineCost -= CalleeFI.ArgumentWeights[ArgNo].ConstantWeight; + if (ArgNo < CalleeFI->ArgumentWeights.size()) + InlineCost -= CalleeFI->ArgumentWeights[ArgNo].ConstantWeight; } } // Now that we have considered all of the factors that make the call site more // likely to be inlined, look at factors that make us not want to inline it. - - // Don't inline into something too big, which would make it bigger. - // "size" here is the number of basic blocks, not instructions. - // - InlineCost += Caller->size()/15; - + + // Calls usually take a long time, so they make the inlining gain smaller. + InlineCost += CalleeFI->Metrics.NumCalls * InlineConstants::CallPenalty; + // Look at the size of the callee. Each instruction counts as 5. - InlineCost += CalleeFI.Metrics.NumInsts*5; + InlineCost += CalleeFI->Metrics.NumInsts*InlineConstants::InstrCost; return llvm::InlineCost::get(InlineCost); } @@ -316,7 +400,7 @@ InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS, float InlineCostAnalyzer::getInlineFudgeFactor(CallSite CS) { Function *Callee = CS.getCalledFunction(); - // Get information about the callee... + // Get information about the callee. FunctionInfo &CalleeFI = CachedFunctionInfo[Callee]; // If we haven't calculated this information yet, do so now. @@ -336,3 +420,63 @@ float InlineCostAnalyzer::getInlineFudgeFactor(CallSite CS) { Factor += 1.5f; return Factor; } + +/// growCachedCostInfo - update the cached cost info for Caller after Callee has +/// been inlined. +void +InlineCostAnalyzer::growCachedCostInfo(Function *Caller, Function *Callee) { + CodeMetrics &CallerMetrics = CachedFunctionInfo[Caller].Metrics; + + // For small functions we prefer to recalculate the cost for better accuracy. + if (CallerMetrics.NumBlocks < 10 || CallerMetrics.NumInsts < 1000) { + resetCachedCostInfo(Caller); + return; + } + + // For large functions, we can save a lot of computation time by skipping + // recalculations. + if (CallerMetrics.NumCalls > 0) + --CallerMetrics.NumCalls; + + if (Callee == 0) return; + + CodeMetrics &CalleeMetrics = CachedFunctionInfo[Callee].Metrics; + + // If we don't have metrics for the callee, don't recalculate them just to + // update an approximation in the caller. Instead, just recalculate the + // caller info from scratch. + if (CalleeMetrics.NumBlocks == 0) { + resetCachedCostInfo(Caller); + return; + } + + // Since CalleeMetrics were already calculated, we know that the CallerMetrics + // reference isn't invalidated: both were in the DenseMap. + CallerMetrics.usesDynamicAlloca |= CalleeMetrics.usesDynamicAlloca; + + // FIXME: If any of these three are true for the callee, the callee was + // not inlined into the caller, so I think they're redundant here. + CallerMetrics.callsSetJmp |= CalleeMetrics.callsSetJmp; + CallerMetrics.isRecursive |= CalleeMetrics.isRecursive; + CallerMetrics.containsIndirectBr |= CalleeMetrics.containsIndirectBr; + + CallerMetrics.NumInsts += CalleeMetrics.NumInsts; + CallerMetrics.NumBlocks += CalleeMetrics.NumBlocks; + CallerMetrics.NumCalls += CalleeMetrics.NumCalls; + CallerMetrics.NumVectorInsts += CalleeMetrics.NumVectorInsts; + CallerMetrics.NumRets += CalleeMetrics.NumRets; + + // analyzeBasicBlock counts each function argument as an inst. + if (CallerMetrics.NumInsts >= Callee->arg_size()) + CallerMetrics.NumInsts -= Callee->arg_size(); + else + CallerMetrics.NumInsts = 0; + + // We are not updating the argument weights. We have already determined that + // Caller is a fairly large function, so we accept the loss of precision. +} + +/// clear - empty the cache of inline costs +void InlineCostAnalyzer::clear() { + CachedFunctionInfo.clear(); +}